def setUp(self) -> None:
self.tempdir = TemporaryDirectory()
self.session_path = utils.create_fake_session_folder(self.tempdir.name)
utils.create_fake_raw_video_data_folder(self.session_path)
self.eid = 'd3372b15-f696-4279-9be5-98f15783b5bb'
self.qc = CameraQC(self.session_path,
one=self.one,
n_samples=5,
side='left',
stream=False,
download_data=False)
self.qc._type = 'ephys'
def test_incomplete_session(self):
# Verify using local path
session_path = self.incomplete
qc = CameraQC(session_path, 'left',
stream=False, download_data=False, one=ONE(offline=True), n_samples=20)
outcome, extended = qc.run(update=False)
self.assertEqual('FAIL', outcome)
expected = {
'_videoLeft_brightness': 'FAIL',
'_videoLeft_camera_times': ('PASS', 0),
'_videoLeft_dropped_frames': 'NOT_SET',
'_videoLeft_file_headers': 'PASS',
'_videoLeft_focus': 'PASS',
'_videoLeft_framerate': 'NOT_SET',
'_videoLeft_pin_state': 'NOT_SET',
'_videoLeft_position': 'WARNING',
'_videoLeft_resolution': 'PASS',
'_videoLeft_timestamps': 'NOT_SET',
'_videoLeft_wheel_alignment': 'NOT_SET'
}
self.assertEqual(expected, extended)
def test_training_session(self, mock_ext, mock_meta):
"""
Tests the full QC process for a training session. Mock a load of things so we don't need
the ful video file.
:param mock_ext: mock cv.VideoCapture in camera extractor module
:param mock_meta: mock get_video_meta
:return:
"""
n_samples = 100
length = 107913
eid, session_path = self.training
self.frames = np.load(self.data_path / 'camera' / f'{eid}_frame_samples.npy')
mock_meta.return_value = \
Bunch({'length': length, **CameraQC.video_meta['training']['left']})
mock_ext().get.return_value = length
mock_ext().read.side_effect = self.side_effect()
qc = CameraQC(session_path, 'left',
stream=False, n_samples=n_samples, one=ONE(offline=True))
qc.load_data(download_data=False, extract_times=True)
outcome, extended = qc.run(update=False)
self.assertEqual('FAIL', outcome)
expected = {
'_videoLeft_brightness': 'FAIL',
'_videoLeft_camera_times': ('PASS', 0),
'_videoLeft_dropped_frames': ('PASS', 1, 0),
'_videoLeft_file_headers': 'PASS',
'_videoLeft_focus': 'FAIL',
'_videoLeft_framerate': ('FAIL', 32.895),
'_videoLeft_pin_state': ('WARNING', 1151, 0),
'_videoLeft_position': 'FAIL',
'_videoLeft_resolution': 'PASS',
'_videoLeft_timestamps': 'PASS',
'_videoLeft_wheel_alignment': ('FAIL', -95)
}
self.assertEqual(expected, extended)
def _run(self):
# avi to mp4 compression
command = ('ffmpeg -i {file_in} -y -nostdin -codec:v libx264 -preset slow -crf 29 '
'-nostats -codec:a copy {file_out}')
output_files = ffmpeg.iblrig_video_compression(self.session_path, command)
if len(output_files) == 0:
_logger.info('No compressed videos found; skipping timestamp extraction')
return # labels the task as empty if no output
# Video timestamps extraction
data, files = camera.extract_all(self.session_path, save=True, video_path=output_files[0])
output_files.extend(files)
# Video QC
CameraQC(self.session_path, 'left', one=self.one, stream=False).run(update=True)
return output_files
def setUpClass(cls) -> None:
"""Load a few 10 second videos for testing the various video QC checks"""
data_path = base.IntegrationTest.default_data_root()
video_path = data_path.joinpath('camera')
videos = sorted(video_path.rglob('*.mp4'))
# Instantiate using session with a video path to fool constructor.
# To remove once we use ONE cache file
one = ONE(base_url='https://test.alyx.internationalbrainlab.org',
username='******',
password='******')
dummy_id = 'd3372b15-f696-4279-9be5-98f15783b5bb'
qc = CameraQC(dummy_id, 'left',
n_samples=10, stream=False, download_data=False, one=one)
qc.one = None
qc._type = 'ephys' # All videos come from ephys sessions
qcs = OrderedDict()
for video in videos:
qc.video_path = video
qc.label = vidio.label_from_path(video)
qc.n_samples = 10
qc.load_video_data()
qcs[video] = qc.data.copy()
cls.qc = qc
cls.data = qcs
class TestCameraQC(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
cls.one = ONE(
base_url="https://test.alyx.internationalbrainlab.org",
username="******",
password="******",
)
cls.backend = matplotlib.get_backend()
matplotlib.use('Agg')
@classmethod
def tearDownClass(cls) -> None:
matplotlib.use(cls.backend)
def setUp(self) -> None:
self.tempdir = TemporaryDirectory()
self.session_path = utils.create_fake_session_folder(self.tempdir.name)
utils.create_fake_raw_video_data_folder(self.session_path)
self.eid = 'd3372b15-f696-4279-9be5-98f15783b5bb'
self.qc = CameraQC(self.session_path,
one=self.one,
n_samples=5,
side='left',
stream=False,
download_data=False)
self.qc._type = 'ephys'
def tearDown(self) -> None:
self.tempdir.cleanup()
plt.close('all')
def test_check_brightness(self):
self.qc.data['frame_samples'] = self.qc.load_reference_frames('left')
n = len(self.qc.data['frame_samples'])
self.qc.frame_samples_idx = np.linspace(0, 1000, n, dtype=int)
self.assertEqual('PASS', self.qc.check_brightness(display=True))
# Check plots
fig = plt.gcf()
self.assertEqual(3, len(fig.axes))
expected = np.array([58.07007217, 56.55802917, 46.09558182])
np.testing.assert_array_almost_equal(fig.axes[0].lines[0]._y, expected)
# Make frames a third as bright
self.qc.data['frame_samples'] = (self.qc.data['frame_samples'] /
3).astype(np.int32)
self.assertEqual('FAIL', self.qc.check_brightness())
# Change thresholds
self.qc.data['frame_samples'] = self.qc.load_reference_frames('left')
self.assertEqual('FAIL', self.qc.check_brightness(bounds=(10, 20)))
self.assertEqual('FAIL', self.qc.check_brightness(max_std=1e-6))
# Check outcome when no frame samples loaded
self.qc.data['frame_samples'] = None
self.assertEqual('NOT_SET', self.qc.check_brightness())
def test_check_file_headers(self):
self.qc.data['video'] = {'fps': 60.}
self.assertEqual('PASS', self.qc.check_file_headers())
self.qc.data['video']['fps'] = 150
self.assertEqual('FAIL', self.qc.check_file_headers())
self.qc.data['video'] = None
self.assertEqual('NOT_SET', self.qc.check_file_headers())
def test_check_framerate(self):
FPS = 60.
self.qc.data['video'] = {'fps': FPS}
self.qc.data['timestamps'] = np.array([round(1 / FPS, 4)] *
1000).cumsum()
outcome, frate = self.qc.check_framerate()
self.assertEqual('PASS', outcome)
self.assertEqual(59.88, frate)
self.assertEqual('FAIL', self.qc.check_framerate(threshold=1e-2)[0])
self.qc.data['timestamps'] = None
self.assertEqual('NOT_SET', self.qc.check_framerate())
def test_check_pin_state(self):
FPS = 60.
self.assertEqual('NOT_SET', self.qc.check_pin_state())
# Add some dummy data
self.qc.data.timestamps = np.array([round(1 / FPS, 4)] * 5).cumsum()
self.qc.data.pin_state = np.zeros((self.qc.data.timestamps.size, 4),
dtype=bool)
self.qc.data.pin_state[1:-1, -1] = True # Pulse on 4th pin
self.qc.data['video'] = {
'fps': FPS,
'length': len(self.qc.data.timestamps)
}
self.qc.data.audio = self.qc.data.timestamps[[0, -1]] - 10e-3
# Check passes and plots results
outcome, *_ = self.qc.check_pin_state(display=True)
self.assertEqual('PASS', outcome)
a, b = [ln.get_xdata() for ln in plt.gcf().axes[0].lines]
self.assertEqual(a, self.qc.data.timestamps[1])
np.testing.assert_array_equal(b, self.qc.data.audio)
# Fudge some numbers
self.qc.data['video']['length'] = self.qc.data.pin_state.shape[0] - 3
self.assertEqual('WARNING', self.qc.check_pin_state()[0])
self.qc.data['video']['length'] = 10
outcome, *dTTL = self.qc.check_pin_state()
self.assertEqual('FAIL', outcome)
self.assertEqual([0, -5], dTTL)
def test_check_dropped_frames(self):
n = 20
self.qc.data.count = np.arange(n)
self.qc.data.video = {'length': n}
self.assertEqual('PASS', self.qc.check_dropped_frames()[0])
# Drop some frames
dropped = 6
self.qc.data.count = np.append(self.qc.data.count, n + dropped)
outcome, dframe, sz_diff = self.qc.check_dropped_frames()
self.assertEqual('FAIL', outcome)
self.assertEqual(dropped, dframe)
self.assertEqual(1, sz_diff)
# Verify threshold arg; should be warning due to size diff
outcome, *_ = self.qc.check_dropped_frames(threshold=70.)
self.assertEqual('WARNING', outcome)
# Verify critical outcome
self.qc.data.count = np.random.permutation(
self.qc.data.count) # Count out of order
self.assertEqual('CRITICAL', self.qc.check_dropped_frames([0]))
# Verify not set outcome
self.qc.data.video = None
self.assertEqual('NOT_SET', self.qc.check_dropped_frames())
def test_check_focus(self):
self.qc.side = 'left'
self.qc.frame_samples_idx = np.linspace(0, 100, 20, dtype=int)
outcome = self.qc.check_focus(test=True, display=True)
self.assertEqual('FAIL', outcome)
# Verify figures
figs = plt.get_fignums()
self.assertEqual(len(plt.figure(figs[0]).axes), 16)
# Verify Laplacian on blurred images
expected = np.array([
13.19, 14.24, 15.44, 16.64, 18.67, 21.51, 25.99, 31.77, 40.75,
52.52, 71.12, 98.26, 149.85, 229.96, 563.53, 563.53
])
actual = [
round(x, 2)
for x in plt.figure(figs[1]).axes[3].lines[0]._y.tolist()
]
np.testing.assert_array_equal(expected, actual)
# Verify fft on blurred images
expected = np.array([
6.91, 7.2, 7.61, 8.08, 8.76, 9.47, 10.35, 11.22, 11.04, 11.42,
11.35, 11.94, 12.45, 13.22, 13.6, 13.6
])
actual = [
round(x, 2)
for x in plt.figure(figs[2]).axes[3].lines[0]._y.tolist()
]
np.testing.assert_array_equal(expected, actual)
# Verify not set outcome
outcome = self.qc.check_focus()
self.assertEqual('NOT_SET', outcome)
# Verify ROI
self.qc.data.frame_samples = self.qc.load_reference_frames('left')
outcome = self.qc.check_focus(roi=None)
self.assertEqual('PASS', outcome)
def test_check_position(self):
# Verify test mode
outcome = self.qc.check_position(test=True, display=True)
self.assertEqual('PASS', outcome)
# Verify plots
axes = plt.gcf().axes
self.assertEqual(3, len(axes))
expected = np.array([100., 93.74829841, 93.2494463])
np.testing.assert_almost_equal(axes[2].lines[0]._y, expected)
# Verify not set (no frame samples and not in test mode)
outcome = self.qc.check_position()
self.assertEqual('NOT_SET', outcome)
# Verify percent threshold as False
thresh = (75, 80)
outcome = self.qc.check_position(test=True,
pct_thresh=False,
hist_thresh=thresh,
display=True)
self.assertEqual('FAIL', outcome)
fig = plt.get_fignums()[-1]
thr = [ln._y[0] for ln in plt.figure(fig).axes[2].lines[1:]]
self.assertCountEqual(thr, thresh, 'unexpected thresholds in figure')
def test_check_resolution(self):
self.qc.data['video'] = {'width': 1280, 'height': 1024}
self.assertEqual('PASS', self.qc.check_resolution())
self.qc.data['video']['width'] = 150
self.assertEqual('FAIL', self.qc.check_resolution())
self.qc.data['video'] = None
self.assertEqual('NOT_SET', self.qc.check_resolution())
def test_check_timestamps(self):
FPS = 60.
n = 1000
self.qc.data['video'] = Bunch({'fps': FPS, 'length': n})
self.qc.data['timestamps'] = np.array([round(1 / FPS, 4)] * n).cumsum()
# Verify passes
self.assertEqual('PASS', self.qc.check_timestamps())
# Verify fails
self.qc.data['timestamps'] = np.array([round(1 / 30, 4)] *
100).cumsum()
self.assertEqual('FAIL', self.qc.check_timestamps())
# Verify not set
self.qc.data['video'] = None
self.assertEqual('NOT_SET', self.qc.check_timestamps())
def test_check_camera_times(self):
outcome = self.qc.check_camera_times()
self.assertEqual('NOT_SET', outcome)
# Verify passes
self.qc.side = 'body'
ts_path = Path(__file__).parents[1].joinpath('extractors', 'data',
'session_ephys')
ssv_times = load_camera_ssv_times(ts_path, self.qc.side)
self.qc.data.bonsai_times, self.qc.data.camera_times = ssv_times
self.qc.data.video = Bunch({'length': self.qc.data.bonsai_times.size})
outcome, _ = self.qc.check_camera_times()
self.assertEqual('PASS', outcome)
# Verify warning
n_over = 14
self.qc.data.video['length'] -= n_over
outcome, actual = self.qc.check_camera_times()
self.assertEqual('WARNING', outcome)
self.assertEqual(n_over, actual)
def test_check_wheel_alignment(self):
"""This just checks data validation. Integration tests test the MotionAlignment class"""
outcome = self.qc.check_wheel_alignment()
self.assertEqual('NOT_SET', outcome)
# Expect FAIL when no overlapping timestamps between wheel and camera
self.qc.data['wheel'] = {
'timestamps': np.arange(4000),
'position': np.random.random(4000),
'period': np.array([3000, 3050])
}
self.qc.data['timestamps'] = np.arange(5000, 6000)
outcome = self.qc.check_wheel_alignment()
self.assertEqual('FAIL', outcome)
# Expect NOT_SET when some overlapping timestamps but chosen period out of range
self.qc.data['timestamps'] -= 1500
with self.assertLogs(logging.getLogger('ibllib'), logging.WARNING):
outcome = self.qc.check_wheel_alignment()
self.assertEqual('NOT_SET', outcome)
def test_ensure_data(self):
self.qc.eid = self.eid
self.qc.download_data = False
# If data for this session exists locally, overwrite the methods so it is not found
if self.one.path_from_eid(self.eid).exists():
self.qc.one.to_eid = lambda _: self.eid
self.qc.one.download_datasets = lambda _: None
with self.assertRaises(AssertionError):
self.qc.run(update=False)