def compare_wheel_fpga_behaviour(session_path, display=DISPLAY): alf_path = session_path.joinpath('alf') shutil.rmtree(alf_path, ignore_errors=True) sync, chmap = ephys_fpga.get_main_probe_sync(session_path, bin_exists=False) fpga_t, fpga_pos = ephys_fpga.extract_wheel_sync(sync, chmap=chmap) bpod_t, bpod_pos = training_wheel.get_wheel_position(session_path, display=display) data, _ = ephys_fpga.extract_all(session_path) bpod2fpga = scipy.interpolate.interp1d(data['intervals_bpod'][:, 0], data['intervals'][:, 0], fill_value="extrapolate") # resample both traces to the same rate and compute correlation coeff bpod_t = bpod2fpga(bpod_t) tmin = max([np.min(fpga_t), np.min(bpod_t)]) tmax = min([np.max(fpga_t), np.max(bpod_t)]) wheel = {'tscale': np.arange(tmin, tmax, 0.01)} wheel['fpga'] = scipy.interpolate.interp1d(fpga_t, fpga_pos)(wheel['tscale']) wheel['bpod'] = scipy.interpolate.interp1d(bpod_t, bpod_pos)(wheel['tscale']) if display: plt.figure() plt.plot(fpga_t - bpod2fpga(0), fpga_pos, '*') plt.plot(bpod_t - bpod2fpga(0), bpod_pos, '.') raw_wheel = { 'fpga_t': fpga_t, 'fpga_pos': fpga_pos, 'bpod_t': bpod_t, 'bpod_pos': bpod_pos } return raw_wheel, wheel
def _qc_from_path(sess_path, display=True): WHEEL = False sess_path = Path(sess_path) temp_alf_folder = sess_path.joinpath('fpga_test', 'alf') temp_alf_folder.mkdir(parents=True, exist_ok=True) raw_trials = raw_data_loaders.load_data(sess_path) tmax = raw_trials[-1]['behavior_data']['States timestamps']['exit_state'][ 0][-1] + 60 sync, chmap = ephys_fpga._get_main_probe_sync(sess_path, bin_exists=False) _ = ephys_fpga.extract_all(sess_path, output_path=temp_alf_folder, save=True) # check that the output is complete fpga_trials = ephys_fpga.extract_behaviour_sync( sync, output_path=temp_alf_folder, tmax=tmax, chmap=chmap, save=True, display=display) # align with the bpod bpod2fpga = ephys_fpga.align_with_bpod(temp_alf_folder.parent) alf_trials = alf.io.load_object(temp_alf_folder, 'trials') shutil.rmtree(temp_alf_folder) # do the QC qcs, qct = qc_fpga_task(fpga_trials, alf_trials) # do the wheel part if WHEEL: bpod_wheel = training_wheel.get_wheel_data(sess_path, save=False) fpga_wheel = ephys_fpga.extract_wheel_sync(sync, chmap=chmap, save=False) if display: import matplotlib.pyplot as plt t0 = max(np.min(bpod2fpga(bpod_wheel['re_ts'])), np.min(fpga_wheel['re_ts'])) dy = np.interp( t0, fpga_wheel['re_ts'], fpga_wheel['re_pos']) - np.interp( t0, bpod2fpga(bpod_wheel['re_ts']), bpod_wheel['re_pos']) fix, axes = plt.subplots(nrows=2, sharex='all', sharey='all') # axes[0].plot(t, pos), axes[0].title.set_text('Extracted') axes[0].plot(bpod2fpga(bpod_wheel['re_ts']), bpod_wheel['re_pos'] + dy) axes[0].plot(fpga_wheel['re_ts'], fpga_wheel['re_pos']) axes[0].title.set_text('FPGA') axes[1].plot(bpod2fpga(bpod_wheel['re_ts']), bpod_wheel['re_pos'] + dy) axes[1].title.set_text('Bpod') return alf.io.dataframe({**fpga_trials, **alf_trials, **qct})
def validate_ttl_test(ses_path, display=False): """ For a mock session on the Ephys Choice world task, check the sync channels for all device properly connected and perform a synchronization if dual probes to check that all channels are recorded properly :param ses_path: session path :param display: show the probe synchronization plot if several probes :return: True if tests pass, errors otherwise """ def _single_test(assertion, str_ok, str_ko): if assertion: _logger.info(str_ok) return True else: _logger.error(str_ko) return False EXPECTED_RATES_HZ = {'left_camera': 60, 'right_camera': 150, 'body_camera': 30} SYNC_RATE_HZ = 1 MIN_TRIALS_NB = 6 ok = True ses_path = Path(ses_path) if not ses_path.exists(): return False rawsync, sync_map = fpga._get_main_probe_sync(ses_path) last_time = rawsync['times'][-1] # get upgoing fronts for each sync = Bunch({}) for k in sync_map: fronts = fpga._get_sync_fronts(rawsync, sync_map[k]) sync[k] = fronts['times'][fronts['polarities'] == 1] wheel = fpga.extract_wheel_sync(rawsync, chmap=sync_map, save=False) frame_rates = {'right_camera': np.round(1 / np.median(np.diff(sync.right_camera))), 'left_camera': np.round(1 / np.median(np.diff(sync.left_camera))), 'body_camera': np.round(1 / np.median(np.diff(sync.body_camera)))} # check the camera frame rates for lab in frame_rates: expect = EXPECTED_RATES_HZ[lab] ok &= _single_test(assertion=abs((1 - frame_rates[lab] / expect)) < 0.1, str_ok=f'PASS: {lab} frame rate: {frame_rates[lab]} = {expect} Hz', str_ko=f'FAILED: {lab} frame rate: {frame_rates[lab]} != {expect} Hz') # check that the wheel has a minimum rate of activity on both channels re_test = abs(1 - sync.rotary_encoder_1.size / sync.rotary_encoder_0.size) < 0.1 re_test &= len(wheel['re_pos']) / last_time > 5 ok &= _single_test(assertion=re_test, str_ok="PASS: Rotary encoder", str_ko="FAILED: Rotary encoder") # check that the frame 2 ttls has a minimum rate of activity ok &= _single_test(assertion=len(sync.frame2ttl) / last_time > 0.2, str_ok="PASS: Frame2TTL", str_ko="FAILED: Frame2TTL") # the audio has to have at least one event per trial ok &= _single_test(assertion=len(sync.bpod) > len(sync.audio) > MIN_TRIALS_NB, str_ok="PASS: audio", str_ko="FAILED: audio") # the bpod has to have at least twice the amount of min trial pulses ok &= _single_test(assertion=len(sync.bpod) > MIN_TRIALS_NB * 2, str_ok="PASS: Bpod", str_ko="FAILED: Bpod") try: # note: tried to depend as little as possible on the extraction code but for the valve... behaviour = fpga.extract_behaviour_sync(rawsync, save=False, chmap=sync_map) res = behaviour.valve_open.size > 1 except AssertionError: res = False # check that the reward valve is actionned at least once ok &= _single_test(assertion=res, str_ok="PASS: Valve open", str_ko="FAILED: Valve open not detected") _logger.info('ALL CHECKS PASSED !') # the imec sync is for 3B Probes only if sync.get('imec_sync') is not None: ok &= _single_test(assertion=np.all(1 - SYNC_RATE_HZ * np.diff(sync.imec_sync) < 0.1), str_ok="PASS: imec sync", str_ko="FAILED: imec sync") # second step is to test that we can make the sync. Assertions are whithin the synch code if sync.get('imec_sync') is not None: sync_result = sync_probes.version3B(ses_path, display=display) else: sync_result = sync_probes.version3A(ses_path, display=display) ok &= _single_test(assertion=sync_result, str_ok="PASS: synchronisation", str_ko="FAILED: probe synchronizations threshold exceeded") if not ok: raise ValueError('FAILED TTL test') return ok
def load_data(self, download_data: bool = None, extract_times: bool = False, load_video: bool = True) -> None: """Extract the data from raw data files Extracts all the required task data from the raw data files. Data keys: - count (int array): the sequential frame number (n, n+1, n+2...) - pin_state (): the camera GPIO pin; records the audio TTLs; should be one per frame - audio (float array): timestamps of audio TTL fronts - fpga_times (float array): timestamps of camera TTLs recorded by FPGA - timestamps (float array): extracted video timestamps (the camera.times ALF) - bonsai_times (datetime array): system timestamps of video PC; should be one per frame - camera_times (float array): camera frame timestamps extracted from frame headers - wheel (Bunch): rotary encoder timestamps, position and period used for wheel motion - video (Bunch): video meta data, including dimensions and FPS - frame_samples (h x w x n array): array of evenly sampled frames (1 colour channel) :param download_data: if True, any missing raw data is downloaded via ONE. Missing data will raise an AssertionError :param extract_times: if True, the camera.times are re-extracted from the raw data :param load_video: if True, calls the load_video_data method """ assert self.session_path, 'no session path set' if download_data is not None: self.download_data = download_data if self.download_data and self.eid and self.one and not self.one.offline: self.ensure_required_data() _log.info('Gathering data for QC') # Get frame count and pin state self.data['count'], self.data['pin_state'] = \ raw.load_embedded_frame_data(self.session_path, self.label, raw=True) # Load the audio and raw FPGA times if self.type == 'ephys': sync, chmap = ephys_fpga.get_main_probe_sync(self.session_path) audio_ttls = ephys_fpga.get_sync_fronts(sync, chmap['audio']) self.data['audio'] = audio_ttls['times'] # Get rises # Load raw FPGA times cam_ts = extract_camera_sync(sync, chmap) self.data['fpga_times'] = cam_ts[self.label] else: bpod_data = raw.load_data(self.session_path) _, audio_ttls = raw.load_bpod_fronts(self.session_path, bpod_data) self.data['audio'] = audio_ttls['times'] # Load extracted frame times alf_path = self.session_path / 'alf' try: assert not extract_times self.data['timestamps'] = alfio.load_object( alf_path, f'{self.label}Camera', short_keys=True)['times'] except AssertionError: # Re-extract kwargs = dict(video_path=self.video_path, labels=self.label) if self.type == 'ephys': kwargs = {**kwargs, 'sync': sync, 'chmap': chmap} # noqa outputs, _ = extract_all(self.session_path, self.type, save=False, **kwargs) self.data['timestamps'] = outputs[ f'{self.label}_camera_timestamps'] except ALFObjectNotFound: _log.warning('no camera.times ALF found for session') # Get audio and wheel data wheel_keys = ('timestamps', 'position') try: self.data['wheel'] = alfio.load_object(alf_path, 'wheel', short_keys=True) except ALFObjectNotFound: # Extract from raw data if self.type == 'ephys': wheel_data = ephys_fpga.extract_wheel_sync(sync, chmap) else: wheel_data = training_wheel.get_wheel_position( self.session_path) self.data['wheel'] = Bunch(zip(wheel_keys, wheel_data)) # Find short period of wheel motion for motion correlation. if data_for_keys( wheel_keys, self.data['wheel']) and self.data['timestamps'] is not None: self.data['wheel'].period = self.get_active_wheel_period( self.data['wheel']) # Load Bonsai frame timestamps try: ssv_times = raw.load_camera_ssv_times(self.session_path, self.label) self.data['bonsai_times'], self.data['camera_times'] = ssv_times except AssertionError: _log.warning('No Bonsai video timestamps file found') # Gather information from video file if load_video: _log.info('Inspecting video file...') self.load_video_data()