def register_sync(self, root_data_folder, dry=False): flag_files = Path(root_data_folder).glob('**/register_me.flag') for flag_file in flag_files: if dry: print(flag_file) continue file_list = flags.read_flag_file(flag_file) logger_.info('registering ' + str(flag_file.parent)) status_str = self.register_session(flag_file.parent, file_list=file_list) if status_str: error_message = str(flag_file.parent) + ' failed registration' error_message += '\n' + ' ' * 8 + status_str error_message += traceback.format_exc() logger_.error(error_message) err_file = flag_file.parent.joinpath('register_me.error') flag_file.replace(err_file) with open(err_file, 'w+') as f: f.write(error_message) continue flags.write_flag_file(flag_file.parent.joinpath('flatiron.flag'), file_list=file_list) flag_file.unlink() if flag_file.parent.joinpath('create_me.flag').exists(): flag_file.parent.joinpath('create_me.flag').unlink() logger_.info('registered' + '\n')
def test_transfer(self): src_subjects_path = self.root_data_folder / "src" dst_subjects_path = self.root_data_folder / "dst" transfer_rig_data.main(src_subjects_path, dst_subjects_path) gsrc = [x.name for x in list(src_subjects_path.rglob('*.*'))] gdst = [x.name for x in list(dst_subjects_path.rglob('*.*'))] self.assertTrue('extract_me.flag' in gdst) gdst = [x for x in gdst if x != 'extract_me.flag'] self.assertTrue('compress_video.flag' in gdst) gdst = [x for x in gdst if x != 'compress_video.flag'] self.assertTrue('_iblrig_micData.raw.wav' in gdst) gdst = [x for x in gdst if x != '_iblrig_micData.raw.wav'] self.assertTrue('_iblrig_leftCamera.raw.avi' in gdst) gdst = [x for x in gdst if x != '_iblrig_leftCamera.raw.avi'] self.assertEqual(gsrc, gdst) # Test if folder exists not copy because no flag transfer_rig_data.main(src_subjects_path, dst_subjects_path) transfer_rig_data.main(src_subjects_path, dst_subjects_path) # Test if flag exists and folder exists in dst flags.write_flag_file(self.session_path.joinpath("transfer_me.flag")) (self.session_path / "raw_behavior_data" / '_iblrig_micData.raw.wav').touch() (self.session_path / "raw_video_data" / '_iblrig_leftCamera.raw.avi').touch() transfer_rig_data.main(src_subjects_path, dst_subjects_path) # Test transfer w/o video and audio flags.write_flag_file(self.session_path.joinpath("transfer_me.flag")) (self.session_path / "raw_behavior_data" / "random.data1.ext").touch() transfer_rig_data.main(src_subjects_path, dst_subjects_path)
def test_transfer_training(self): src_subjects_path = self.root_data_folder / "src" dst_subjects_path = self.root_data_folder / "dst" transfer_rig_data.main(src_subjects_path, dst_subjects_path) gsrc = [x.name for x in list(src_subjects_path.rglob('*.*'))] gdst = [x.name for x in list(dst_subjects_path.rglob('*.*'))] self.assertTrue(set(gsrc).issubset(set(gdst))) dst_session_path = dst_subjects_path.joinpath( self.session_path.relative_to(src_subjects_path)) flag_files = [dst_session_path.joinpath('raw_session.flag')] for fl in flag_files: self.assertTrue(fl.exists()) # Test if folder exists not copy because no flag transfer_rig_data.main(src_subjects_path, dst_subjects_path) transfer_rig_data.main(src_subjects_path, dst_subjects_path) # Test if flag exists and folder exists in dst flags.write_flag_file(self.session_path.joinpath("transfer_me.flag")) (self.session_path / "raw_behavior_data" / '_iblrig_micData.raw.wav').touch() (self.session_path / "raw_video_data" / '_iblrig_leftCamera.raw.avi').touch() transfer_rig_data.main(src_subjects_path, dst_subjects_path, force=True) # Test transfer w/o video and audios flags.write_flag_file(self.session_path.joinpath("transfer_me.flag")) (self.session_path / "raw_behavior_data" / "random.data1.ext").touch() transfer_rig_data.main(src_subjects_path, dst_subjects_path, force=True)
def sync_merge_ephys(root_data_folder, dry=False): """ Post spike-sorting processing: - synchronization of probes - ks2 to ALF conversion for each probes in alf/probeXX folder - computes spike sorting QC - creates probes object in alf folder To start the job for a session, all electrophysiology ap files from session need to be associated with a `sync_merge_ephys.flag` file Outputs individual probes """ syncflags = list(Path(root_data_folder).rglob('sync_merge_ephys.flag')) session_paths = list(set([f.parents[2] for f in syncflags])) for session_path in session_paths: print(session_path) if dry: continue # first remove the flags [f.unlink() for f in syncflags if f.parents[2] == session_path] # first sync the probes sync_probes.sync(session_path) # then convert ks2 to ALF and resync spike sorting data spikes.sync_spike_sortings(session_path) # outputs the probes object in the ALF folder spikes.probes_description(session_path) # wrap up by removing flags and creating register_me flag flags.write_flag_file(session_path.joinpath('register_me.flag'))
def compress_ephys(root_data_folder, dry=False, max_sessions=5): """ Compress ephys files looking for `compress_ephys.flag` whithin the probes folder Original bin file will be removed The registration flag created contains targeted file names at the root of the session """ qcflags = Path(root_data_folder).rglob('compress_ephys.flag') c = 0 for qcflag in qcflags: probe_path = qcflag.parent c += 1 if c > max_sessions: return if dry: print(qcflag.parent) continue # no rglob: only the folder in which the flag is located gets searched ephys_files = spikeglx.glob_ephys_files(probe_path, recursive=False) out_files = [] for ef in ephys_files: for typ in ['ap', 'lf', 'nidq']: bin_file = ef.get(typ) if not bin_file: continue sr = spikeglx.Reader(bin_file) if not sr.is_mtscomp: out_files.append(sr.compress_file(keep_original=False)) qcflag.unlink() if out_files: session_path = probe_path.parents[1] file_list = [str(f.relative_to(session_path)) for f in out_files] flags.write_flag_file(probe_path.joinpath('register_me.flag'), file_list=file_list)
def bulk(subjects_folder, dry=False): ses_path = Path(subjects_folder).glob('**/extract_me.flag') for p in ses_path: # @alejandro no need for flags until personal project data starts going to server # the flag file may contains specific file names for a targeted extraction save = flags.read_flag_file(p) if dry: print(p) continue try: from_path(p.parent, force=True, save=save) except Exception as e: error_message = str( p.parent) + ' failed extraction' + '\n ' + str(e) error_message += traceback.format_exc() err_file = p.parent.joinpath('extract_me.error') p.replace(err_file) with open(err_file, 'w+') as f: f.write(error_message) logger_.error(error_message) continue p.unlink() flags.write_flag_file(p.parent.joinpath('register_me.flag'), file_list=save)
def check_create_raw_session_flag(session_folder: str) -> None: session_path = Path(session_folder) ephys = session_path.joinpath('ephys_data_transferred.flag') video = session_path.joinpath('video_data_transferred.flag') if ephys.exists() and video.exists(): flags.write_flag_file(session_path.joinpath('raw_session.flag')) ephys.unlink() video.unlink()
def main(local_folder: str, remote_folder: str, force: bool = False) -> None: local_folder = Path(local_folder) remote_folder = Path(remote_folder) src_session_paths = [ x.parent for x in local_folder.rglob("transfer_me.flag") ] if not src_session_paths: log.info("Nothing to transfer, exiting...") return # Create all dst paths dst_session_paths = [] for s in src_session_paths: mouse = s.parts[-3] date = s.parts[-2] sess = s.parts[-1] d = remote_folder / mouse / date / sess dst_session_paths.append(d) for src, dst in zip(src_session_paths, dst_session_paths): src_flag_file = src / "transfer_me.flag" flag = flags.read_flag_file(src_flag_file) if isinstance(flag, list): raise NotImplementedError else: if force: shutil.rmtree(dst, ignore_errors=True) log.info(f"Copying {src}...") shutil.copytree(src, dst, ignore=ig(str(src_flag_file.name))) # finally if folder was created delete the src flag_file and create compress_me.flag if dst.exists(): task_type = ibllib.io.extractors.base.get_session_extractor_type( Path(src)) if task_type not in ['ephys', 'ephys_sync', 'ephys_mock']: flags.write_flag_file(dst.joinpath('raw_session.flag')) settings = raw.load_settings(dst) if 'ephys' in settings[ 'PYBPOD_BOARD']: # Any traing task on an ephys rig dst.joinpath('raw_session.flag').unlink() log.info( f"Copied to {remote_folder}: Session {src_flag_file.parent}") src_flag_file.unlink() # Cleanup src_video_file = src / 'raw_video_data' / '_iblrig_leftCamera.raw.avi' dst_video_file = dst / 'raw_video_data' / '_iblrig_leftCamera.raw.avi' src_audio_file = src / 'raw_behavior_data' / '_iblrig_micData.raw.wav' dst_audio_file = dst / 'raw_behavior_data' / '_iblrig_micData.raw.wav' if src_audio_file.exists() and \ src_audio_file.stat().st_size == dst_audio_file.stat().st_size: src_audio_file.unlink() if src_video_file.exists() and \ src_video_file.stat().st_size == dst_video_file.stat().st_size: src_video_file.unlink()
def sync_merge(session_path, dry=False, force=False): """ Sync probes and merge spike sorting output. For single probe dataset, output ks2 as ALF dataset """ session_path = Path(session_path) sync(session_path, display=False) spikes.merge_probes(session_path) flags.write_flag_file(session_path.joinpath('register_me.flag'))
def setUp(self): self.tmp_dir = tempfile.TemporaryDirectory() self.root_data_folder = Path(self.tmp_dir.name) self.session_path = self.root_data_folder / "src" / 'algernon' / '2019-01-21' / '001' self.session_path.mkdir(parents=True, exist_ok=True) flags.write_flag_file(self.session_path.joinpath("transfer_me.flag")) (self.session_path / "raw_behavior_data").mkdir() (self.session_path / "raw_video_data").mkdir() (self.session_path / "raw_behavior_data" / "random.data1.ext").touch()
def bulk(subjects_folder, dry=False, glob_flag='**/extract_me.flag'): ses_path = Path(subjects_folder).glob(glob_flag) for p in ses_path: # the flag file may contains specific file names for a targeted extraction save = flags.read_flag_file(p) if dry: print(p) continue from_path(p.parent, force=True, save=save) p.unlink() flags.write_flag_file(p.parent.joinpath('register_me.flag'), file_list=save)
def main(local_folder: str, remote_folder: str, force: bool = True) -> None: local_folder = Path(local_folder) remote_folder = Path(remote_folder) src_session_paths = [ x.parent for x in local_folder.rglob("transfer_me.flag") ] if not src_session_paths: log.info("Nothing to transfer, exiting...") return # Create all dst paths dst_session_paths = [] for s in src_session_paths: mouse = s.parts[-3] date = s.parts[-2] sess = s.parts[-1] d = remote_folder / mouse / date / sess dst_session_paths.append(d) for src, dst in zip(src_session_paths, dst_session_paths): src_flag_file = src / "transfer_me.flag" flag = flags.read_flag_file(src_flag_file) if isinstance(flag, list): raise NotImplementedError else: if force: shutil.rmtree(dst, ignore_errors=True) log.info(f"Copying {src}...") shutil.copytree(src, dst, ignore=ig(str(src_flag_file.name))) # finally if folder was created delete the src flag_file and create compress_me.flag if dst.exists(): dst_flag_file = dst / 'extract_me.flag' flags.write_flag_file(dst_flag_file) flags.create_compress_flags(dst) log.info( f"Copied to {remote_folder}: Session {src_flag_file.parent}") src_flag_file.unlink() # Cleanup src_audio_file = src / 'raw_behavior_data' / '_iblrig_micData.raw.wav' src_video_file = src / 'raw_video_data' / '_iblrig_leftCamera.raw.avi' dst_audio_file = dst / 'raw_behavior_data' / '_iblrig_micData.raw.wav' dst_video_file = dst / 'raw_video_data' / '_iblrig_leftCamera.raw.avi' if src_audio_file.exists() and \ src_audio_file.stat().st_size == dst_audio_file.stat().st_size: src_audio_file.unlink() if src_video_file.exists() and \ src_video_file.stat().st_size == dst_video_file.stat().st_size: src_video_file.unlink()
def _compress(root_data_folder, command, flag_pattern, dry=False, max_sessions=None): # runs a command of the form command = "ls -1 {file_name}.avi" c = 0 for flag_file in Path(root_data_folder).rglob(flag_pattern): ses_path = flag_file.parent files2compress = flags.read_flag_file(flag_file) if isinstance(files2compress, bool): Path(flag_file).unlink() continue for f2c in files2compress: cfile = ses_path.joinpath(PureWindowsPath(f2c)) c += 1 if max_sessions and c > max_sessions: return print(cfile) if dry: continue if not cfile.exists(): _logger.error(f'NON-EXISTING RAW FILE: {cfile}. Skipping...') continue if flag_file.exists(): flag_file.unlink() # run the compression command redirecting output cfile.parent.joinpath(cfile.stem) # if the output file already exists, overwrite it outfile = cfile.parent / (cfile.stem + '.mp4') if outfile.exists(): outfile.unlink() command2run = command.format( file_name=cfile.parent.joinpath(cfile.stem)) process = subprocess.Popen(command2run, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) info, error = process.communicate() if process.returncode != 0: _logger.error('COMPRESSION FAILED FOR ' + str(cfile)) with open(cfile.parent.joinpath('extract.error'), 'w+') as fid: fid.write(command2run) fid.write(error.decode()) else: # if the command was successful delete the original file cfile.unlink() # and add the file to register_me.flag flags.write_flag_file(ses_path.joinpath('register_me.flag'), file_list=cfile.stem)
def qc_ephys(root_data_folder, dry=False, max_sessions=10, force=False): qcflags = Path(root_data_folder).rglob('qc_ephys.flag') c = 0 for qcflag in qcflags: session_path = qcflag.parent c += 1 if c >= max_sessions: return if dry: print(qcflag.parent) continue qc_files = ephysqc.qc_session(session_path, dry=dry, force=force) qcflag.unlink() flags.write_flag_file(session_path.joinpath('register_me.flag'), file_list=qc_files)
def testFlagFile(self): # empty file should return True self.assertEqual(flags.read_flag_file(self.tempfile.name), True) # test with 2 lines and a trailing with open(self.tempfile.name, 'w+') as fid: fid.write('file1\nfile2\n') self.assertEqual(flags.read_flag_file(self.tempfile.name), ['file1', 'file2']) # test with 2 lines and a trailing, Windows convention with open(self.tempfile.name, 'w+') as fid: fid.write('file1\r\nfile2\r\n') self.assertEqual(flags.read_flag_file(self.tempfile.name), ['file1', 'file2']) # test read after write file_list = [ '_ibl_extraRewards.times', '_ibl_lickPiezo.raw', '_ibl_lickPiezo.timestamps' ] flags.write_flag_file(self.tempfile.name, file_list) self.assertEqual(flags.read_flag_file(self.tempfile.name), file_list) # makes sure that read after write empty list returns True flags.write_flag_file(self.tempfile.name, []) self.assertEqual(flags.read_flag_file(self.tempfile.name), True) # makes sure that read after write None also returns True flags.write_flag_file(self.tempfile.name, None) self.assertEqual(flags.read_flag_file(self.tempfile.name), True) # make sure that read after write with a string workds flags.write_flag_file(self.tempfile.name, '_ibl_lickPiezo.raw') self.assertEqual(flags.read_flag_file(self.tempfile.name), ['_ibl_lickPiezo.raw'])
def create_ephys_flags(session_folder: str): session_path = Path(session_folder) flags.write_flag_file(session_path.joinpath('extract_ephys.flag')) flags.write_flag_file(session_path.joinpath('raw_ephys_qc.flag')) probe00_path = session_path / 'raw_ephys_data' / 'probe00' probe01_path = session_path / 'raw_ephys_data' / 'probe01' flags.write_flag_file(probe00_path.joinpath('spike_sorting.flag')) flags.write_flag_file(probe01_path.joinpath('spike_sorting.flag'))
def create_ephys_flags(session_folder: str): session_path = Path(session_folder) flags.write_flag_file(session_path.joinpath("extract_ephys.flag")) flags.write_flag_file(session_path.joinpath("raw_ephys_qc.flag")) probe00_path = session_path / "raw_ephys_data" / "probe00" probe01_path = session_path / "raw_ephys_data" / "probe01" flags.write_flag_file(probe00_path.joinpath("spike_sorting.flag")) flags.write_flag_file(probe01_path.joinpath("spike_sorting.flag"))
def test_transfer(self): src_subjects_path = self.root_data_folder / "src" dst_subjects_path = self.root_data_folder / "dst" transfer_rig_data.main(src_subjects_path, dst_subjects_path) gsrc = [x.name for x in list(src_subjects_path.rglob('*.*'))] gdst = [x.name for x in list(dst_subjects_path.rglob('*.*'))] self.assertTrue('extract_me.flag' in gdst) gdst = [x for x in gdst if x != 'extract_me.flag'] self.assertEqual(gsrc, gdst) # Test if folder exists not copy because no flag transfer_rig_data.main(src_subjects_path, dst_subjects_path) transfer_rig_data.main(src_subjects_path, dst_subjects_path) # Test if flag exists and folder exists in dst flags.write_flag_file(self.session_path.joinpath("transfer_me.flag")) transfer_rig_data.main(src_subjects_path, dst_subjects_path)
def check_create_raw_session_flag(session_folder: str) -> None: session_path = Path(session_folder) ephys = session_path.joinpath("ephys_data_transferred.flag") video = session_path.joinpath("video_data_transferred.flag") sett = raw.load_settings(session_path) if sett is None: log.error(f"No flag created for {session_path}") return is_biased = True if "biased" in sett["PYBPOD_PROTOCOL"] else False if video.exists() and is_biased: flags.write_flag_file(session_path.joinpath("raw_session.flag")) video.unlink() if video.exists() and ephys.exists(): flags.write_flag_file(session_path.joinpath("raw_session.flag")) ephys.unlink() video.unlink()
def raw_ephys_qc(root_data_folder, dry=False, max_sessions=10, force=False): """ Computes raw electrophysiology QC """ qcflags = Path(root_data_folder).rglob('raw_ephys_qc.flag') c = 0 for qcflag in qcflags: session_path = qcflag.parent c += 1 if c >= max_sessions: return if dry: print(qcflag.parent) continue qc_files = ephysqc.raw_qc_session(session_path, dry=dry, force=force) qcflag.unlink() flags.write_flag_file(session_path.joinpath('register_me.flag'), file_list=qc_files)
def extract(subjects_folder, dry=False): ses_path = Path(subjects_folder).glob('**/extract_me.flag') for p in ses_path: # @alejandro no need for flags until personal project data starts going to server # the flag file may contains specific file names for a targeted extraction if dry: print(p) continue try: extract_opto(p.parent, save=True) except: pass #p.unlink() commented we need the flag for the sync merge step save = flags.read_flag_file(p) flags.write_flag_file(p.parent.joinpath('opto_extracted.flag'), file_list=save)
def make_session(session_path, type='training'): flags.write_flag_file(session_path.joinpath("transfer_me.flag")) session_path.joinpath("raw_behavior_data").mkdir() session_path.joinpath("raw_video_data").mkdir() session_path.joinpath("raw_behavior_data", "_iblrig_micData.raw.wav").touch() session_path.joinpath("raw_video_data", '_iblrig_leftCamera.raw.avi').touch() sf = session_path.joinpath('raw_behavior_data', '_iblrig_taskSettings.raw.json') if type == 'training': pybpod_protcol = 'json_trainingChoiceWorld' elif type == 'ephys': pybpod_protcol = 'json_ephysChoiceWorld' session_path.joinpath("raw_video_data", '_iblrig_rightCamera.raw.avi').touch() session_path.joinpath("raw_video_data", '_iblrig_bodyCamera.raw.avi').touch() with open(sf, 'w+') as fid: fid.write(json.dumps({'PYBPOD_PROTOCOL': pybpod_protcol}))
def register_sync(self, root_data_folder, dry=False): """ Register sessions looking recursively for flag files :param root_data_folder: folder to look for register_me.flag :param dry: bool. Dry run if True :return: """ flag_files = Path(root_data_folder).glob('**/register_me.flag') for flag_file in flag_files: if dry: continue file_list = flags.read_flag_file(flag_file) _logger.info('registering ' + str(flag_file.parent)) self.register_session(flag_file.parent, file_list=file_list) flags.write_flag_file(flag_file.parent.joinpath('flatiron.flag'), file_list=file_list) flag_file.unlink() if flag_file.parent.joinpath('create_me.flag').exists(): flag_file.parent.joinpath('create_me.flag').unlink() _logger.info('registered' + '\n')
def _create_flags_for_task(dst, task_type): # create_flags_for_session() if task_type in ['habituation']: flags.write_flag_file(dst.joinpath('extract_me.flag')) if task_type in ['training', 'biased']: flags.write_flag_file(dst.joinpath('extract_me.flag')) flags.create_compress_video_flags(dst, flag_name='compress_video.flag') flags.create_audio_flags(dst, 'audio_training.flag') elif task_type in ['ephys', 'ephys_sync']: flags.write_flag_file(dst.joinpath('extract_ephys.flag')) flags.write_flag_file(dst.joinpath('raw_ephys_qc.flag')) flags.create_compress_video_flags( dst, flag_name='compress_video_ephys.flag') flags.create_audio_flags(dst, 'audio_ephys.flag')
def create_ephys_flags(session_folder: str): """ Create flags for processing an ephys session. Should be called after move_ephys_files :param session_folder: A path to an ephys session :return: """ session_path = Path(session_folder) flags.write_flag_file(session_path.joinpath("extract_ephys.flag")) flags.write_flag_file(session_path.joinpath("raw_ephys_qc.flag")) for probe_path in session_path.joinpath('raw_ephys_data').glob('probe*'): flags.write_flag_file(probe_path.joinpath("spike_sorting.flag"))
def testAppendFlagFile(self): # DO NOT CHANGE THE ORDER OF TESTS BELOW # prepare a file with 3 dataset types file_list = ['_ibl_extraRewards.times', '_ibl_lickPiezo.raw', '_ibl_lickPiezo.timestamps'] with open(self.tempfile.name, 'w+') as fid: fid.write('\n'.join(file_list)) self.assertEqual(set(flags.read_flag_file(self.tempfile.name)), set(file_list)) # with an existing file containing files, writing more files append to it file_list_2 = ['turltu'] # also makes sure that if a string is provided it works flags.write_flag_file(self.tempfile.name, file_list_2[0]) self.assertEqual(set(flags.read_flag_file(self.tempfile.name)), set(file_list + file_list_2)) # writing again keeps unique file values flags.write_flag_file(self.tempfile.name, file_list_2[0]) n = sum([1 for f in flags.read_flag_file(self.tempfile.name) if f == file_list_2[0]]) self.assertEqual(n, 1) # with an existing file containing files, writing empty filelist returns True for all files flags.write_flag_file(self.tempfile.name, None) self.assertEqual(flags.read_flag_file(self.tempfile.name), True) # with an existing empty file, writing filelist returns True for all files flags.write_flag_file(self.tempfile.name, ['file1', 'file2']) self.assertEqual(flags.read_flag_file(self.tempfile.name), True) # makes sure that read after write empty list also returns True flags.write_flag_file(self.tempfile.name, []) self.assertEqual(flags.read_flag_file(self.tempfile.name), True) # with an existing empty file, writing filelist returns the list if clobber flags.write_flag_file(self.tempfile.name, ['file1', 'file2', 'file3'], clobber=True) self.assertEqual(set(flags.read_flag_file(self.tempfile.name)), set(['file1', 'file2', 'file3'])) # test the removal of a file within the list flags.excise_flag_file(self.tempfile.name, removed_files='file1') self.assertEqual(sorted(flags.read_flag_file(self.tempfile.name)), ['file2', 'file3']) # if file-list is True it means all files and file_list should be empty after read flags.write_flag_file(self.tempfile.name, file_list=True) self.assertEqual(flags.read_flag_file(self.tempfile.name), True)
def create_video_transfer_done_flag(session_folder: str) -> None: session_path = Path(session_folder) flags.write_flag_file(session_path.joinpath('video_data_transferred.flag'))
def create_ephys_transfer_done_flag(session_folder: str) -> None: session_path = Path(session_folder) flags.write_flag_file(session_path.joinpath("ephys_data_transferred.flag"))