def testFlagFileRead(self):
# empty file should return True
self.assertEqual(flags.read_flag_file(self.tempfile.name), True)
# test with 2 lines and a trailing
with open(self.tempfile.name, 'w+') as fid:
fid.write('file1\nfile2\n')
self.assertEqual(set(flags.read_flag_file(self.tempfile.name)), set(['file1', 'file2']))
# test with 2 lines and a trailing, Windows convention
with open(self.tempfile.name, 'w+') as fid:
fid.write('file1\r\nfile2\r\n')
self.assertEqual(set(flags.read_flag_file(self.tempfile.name)), set(['file1', 'file2']))
def testAppendFlagFile(self):
# DO NOT CHANGE THE ORDER OF TESTS BELOW
# prepare a file with 3 dataset types
file_list = [
'_ibl_extraRewards.times', '_ibl_lickPiezo.raw',
'_ibl_lickPiezo.timestamps'
]
with open(self.tempfile.name, 'w+') as fid:
fid.write('\n'.join(file_list))
self.assertEqual(set(flags.read_flag_file(self.tempfile.name)),
set(file_list))
# with an existing file containing files, writing more files append to it
file_list_2 = ['turltu']
# also makes sure that if a string is provided it works
flags.write_flag_file(self.tempfile.name, file_list_2[0])
self.assertEqual(set(flags.read_flag_file(self.tempfile.name)),
set(file_list + file_list_2))
# writing again keeps unique file values
flags.write_flag_file(self.tempfile.name, file_list_2[0])
n = sum([
1 for f in flags.read_flag_file(self.tempfile.name)
if f == file_list_2[0]
])
self.assertEqual(n, 1)
# with an existing file containing files, writing empty filelist returns True for all files
flags.write_flag_file(self.tempfile.name, None)
self.assertEqual(flags.read_flag_file(self.tempfile.name), True)
# with an existing empty file, writing filelist returns True for all files
flags.write_flag_file(self.tempfile.name, ['file1', 'file2'])
self.assertEqual(flags.read_flag_file(self.tempfile.name), True)
# makes sure that read after write empty list also returns True
flags.write_flag_file(self.tempfile.name, [])
self.assertEqual(flags.read_flag_file(self.tempfile.name), True)
# with an existing empty file, writing filelist returns the list if clobber
flags.write_flag_file(self.tempfile.name, ['file1', 'file2', 'file3'],
clobber=True)
self.assertEqual(set(flags.read_flag_file(self.tempfile.name)),
set(['file1', 'file2', 'file3']))
# test the removal of a file within the list
flags.excise_flag_file(self.tempfile.name, removed_files='file1')
self.assertEqual(sorted(flags.read_flag_file(self.tempfile.name)),
['file2', 'file3'])
# if file-list is True it means all files and file_list should be empty after read
flags.write_flag_file(self.tempfile.name, file_list=True)
self.assertEqual(flags.read_flag_file(self.tempfile.name), True)
def transfer_session(src: Path, dst: Path, force: bool = False):
print(src, dst)
# log.info(f"Attempting to copy from {src} to {dst}...")
src = Path(folders.session_path(src))
dst_sess = Path(folders.session_path(dst))
if src is None:
return
if dst_sess is None:
dst = dst / Path(*src.parts[-3:])
src_flag_file = src / "transfer_me.flag"
if not src_flag_file.exists():
return
flag = flags.read_flag_file(src_flag_file)
if isinstance(flag, list):
raise (NotImplementedError)
else:
if force:
shutil.rmtree(dst, ignore_errors=True)
log.info(f"Copying all files from {src} to {dst} ...")
shutil.copytree(src, dst, ignore=ig(str(src_flag_file.name)))
# If folder was created delete the src_flag_file
if (dst / 'raw_video_data').exists():
log.info(
f"{Path(*src.parts[-3:]) / 'raw_video_data'} copied to {dst.parent.parent.parent}"
)
src_flag_file.unlink()
def register_sync(self, root_data_folder, dry=False):
flag_files = Path(root_data_folder).glob('**/register_me.flag')
for flag_file in flag_files:
if dry:
print(flag_file)
continue
file_list = flags.read_flag_file(flag_file)
logger_.info('registering ' + str(flag_file.parent))
status_str = self.register_session(flag_file.parent,
file_list=file_list)
if status_str:
error_message = str(flag_file.parent) + ' failed registration'
error_message += '\n' + ' ' * 8 + status_str
error_message += traceback.format_exc()
logger_.error(error_message)
err_file = flag_file.parent.joinpath('register_me.error')
flag_file.replace(err_file)
with open(err_file, 'w+') as f:
f.write(error_message)
continue
flags.write_flag_file(flag_file.parent.joinpath('flatiron.flag'),
file_list=file_list)
flag_file.unlink()
if flag_file.parent.joinpath('create_me.flag').exists():
flag_file.parent.joinpath('create_me.flag').unlink()
logger_.info('registered' + '\n')
def bulk(subjects_folder, dry=False):
ses_path = Path(subjects_folder).glob('**/extract_me.flag')
for p in ses_path:
# @alejandro no need for flags until personal project data starts going to server
# the flag file may contains specific file names for a targeted extraction
save = flags.read_flag_file(p)
if dry:
print(p)
continue
try:
from_path(p.parent, force=True, save=save)
except Exception as e:
error_message = str(
p.parent) + ' failed extraction' + '\n ' + str(e)
error_message += traceback.format_exc()
err_file = p.parent.joinpath('extract_me.error')
p.replace(err_file)
with open(err_file, 'w+') as f:
f.write(error_message)
logger_.error(error_message)
continue
p.unlink()
flags.write_flag_file(p.parent.joinpath('register_me.flag'),
file_list=save)
def main(local_folder: str, remote_folder: str, force: bool = False) -> None:
local_folder = Path(local_folder)
remote_folder = Path(remote_folder)
src_session_paths = [
x.parent for x in local_folder.rglob("transfer_me.flag")
]
if not src_session_paths:
log.info("Nothing to transfer, exiting...")
return
# Create all dst paths
dst_session_paths = []
for s in src_session_paths:
mouse = s.parts[-3]
date = s.parts[-2]
sess = s.parts[-1]
d = remote_folder / mouse / date / sess
dst_session_paths.append(d)
for src, dst in zip(src_session_paths, dst_session_paths):
src_flag_file = src / "transfer_me.flag"
flag = flags.read_flag_file(src_flag_file)
if isinstance(flag, list):
raise NotImplementedError
else:
if force:
shutil.rmtree(dst, ignore_errors=True)
log.info(f"Copying {src}...")
shutil.copytree(src, dst, ignore=ig(str(src_flag_file.name)))
# finally if folder was created delete the src flag_file and create compress_me.flag
if dst.exists():
task_type = ibllib.io.extractors.base.get_session_extractor_type(
Path(src))
if task_type not in ['ephys', 'ephys_sync', 'ephys_mock']:
flags.write_flag_file(dst.joinpath('raw_session.flag'))
settings = raw.load_settings(dst)
if 'ephys' in settings[
'PYBPOD_BOARD']: # Any traing task on an ephys rig
dst.joinpath('raw_session.flag').unlink()
log.info(
f"Copied to {remote_folder}: Session {src_flag_file.parent}")
src_flag_file.unlink()
# Cleanup
src_video_file = src / 'raw_video_data' / '_iblrig_leftCamera.raw.avi'
dst_video_file = dst / 'raw_video_data' / '_iblrig_leftCamera.raw.avi'
src_audio_file = src / 'raw_behavior_data' / '_iblrig_micData.raw.wav'
dst_audio_file = dst / 'raw_behavior_data' / '_iblrig_micData.raw.wav'
if src_audio_file.exists() and \
src_audio_file.stat().st_size == dst_audio_file.stat().st_size:
src_audio_file.unlink()
if src_video_file.exists() and \
src_video_file.stat().st_size == dst_video_file.stat().st_size:
src_video_file.unlink()
def testFlagFile(self):
# empty file should return True
self.assertEqual(flags.read_flag_file(self.tempfile.name), True)
# test with 2 lines and a trailing
with open(self.tempfile.name, 'w+') as fid:
fid.write('file1\nfile2\n')
self.assertEqual(flags.read_flag_file(self.tempfile.name),
['file1', 'file2'])
# test with 2 lines and a trailing, Windows convention
with open(self.tempfile.name, 'w+') as fid:
fid.write('file1\r\nfile2\r\n')
self.assertEqual(flags.read_flag_file(self.tempfile.name),
['file1', 'file2'])
# test read after write
file_list = [
'_ibl_extraRewards.times', '_ibl_lickPiezo.raw',
'_ibl_lickPiezo.timestamps'
]
flags.write_flag_file(self.tempfile.name, file_list)
self.assertEqual(flags.read_flag_file(self.tempfile.name), file_list)
# makes sure that read after write empty list returns True
flags.write_flag_file(self.tempfile.name, [])
self.assertEqual(flags.read_flag_file(self.tempfile.name), True)
# makes sure that read after write None also returns True
flags.write_flag_file(self.tempfile.name, None)
self.assertEqual(flags.read_flag_file(self.tempfile.name), True)
# make sure that read after write with a string workds
flags.write_flag_file(self.tempfile.name, '_ibl_lickPiezo.raw')
self.assertEqual(flags.read_flag_file(self.tempfile.name),
['_ibl_lickPiezo.raw'])
def bulk(subjects_folder, dry=False, glob_flag='**/extract_me.flag'):
ses_path = Path(subjects_folder).glob(glob_flag)
for p in ses_path:
# the flag file may contains specific file names for a targeted extraction
save = flags.read_flag_file(p)
if dry:
print(p)
continue
from_path(p.parent, force=True, save=save)
p.unlink()
flags.write_flag_file(p.parent.joinpath('register_me.flag'), file_list=save)
def main(local_folder: str, remote_folder: str, force: bool = True) -> None:
local_folder = Path(local_folder)
remote_folder = Path(remote_folder)
src_session_paths = [
x.parent for x in local_folder.rglob("transfer_me.flag")
]
if not src_session_paths:
log.info("Nothing to transfer, exiting...")
return
# Create all dst paths
dst_session_paths = []
for s in src_session_paths:
mouse = s.parts[-3]
date = s.parts[-2]
sess = s.parts[-1]
d = remote_folder / mouse / date / sess
dst_session_paths.append(d)
for src, dst in zip(src_session_paths, dst_session_paths):
src_flag_file = src / "transfer_me.flag"
flag = flags.read_flag_file(src_flag_file)
if isinstance(flag, list):
raise NotImplementedError
else:
if force:
shutil.rmtree(dst, ignore_errors=True)
log.info(f"Copying {src}...")
shutil.copytree(src, dst, ignore=ig(str(src_flag_file.name)))
# finally if folder was created delete the src flag_file and create compress_me.flag
if dst.exists():
dst_flag_file = dst / 'extract_me.flag'
flags.write_flag_file(dst_flag_file)
flags.create_compress_flags(dst)
log.info(
f"Copied to {remote_folder}: Session {src_flag_file.parent}")
src_flag_file.unlink()
# Cleanup
src_audio_file = src / 'raw_behavior_data' / '_iblrig_micData.raw.wav'
src_video_file = src / 'raw_video_data' / '_iblrig_leftCamera.raw.avi'
dst_audio_file = dst / 'raw_behavior_data' / '_iblrig_micData.raw.wav'
dst_video_file = dst / 'raw_video_data' / '_iblrig_leftCamera.raw.avi'
if src_audio_file.exists() and \
src_audio_file.stat().st_size == dst_audio_file.stat().st_size:
src_audio_file.unlink()
if src_video_file.exists() and \
src_video_file.stat().st_size == dst_video_file.stat().st_size:
src_video_file.unlink()
def _compress(root_data_folder,
command,
flag_pattern,
dry=False,
max_sessions=None):
# runs a command of the form command = "ls -1 {file_name}.avi"
c = 0
for flag_file in Path(root_data_folder).rglob(flag_pattern):
ses_path = flag_file.parent
files2compress = flags.read_flag_file(flag_file)
if isinstance(files2compress, bool):
Path(flag_file).unlink()
continue
for f2c in files2compress:
cfile = ses_path.joinpath(PureWindowsPath(f2c))
c += 1
if max_sessions and c > max_sessions:
return
print(cfile)
if dry:
continue
if not cfile.exists():
_logger.error(f'NON-EXISTING RAW FILE: {cfile}. Skipping...')
continue
if flag_file.exists():
flag_file.unlink()
# run the compression command redirecting output
cfile.parent.joinpath(cfile.stem)
# if the output file already exists, overwrite it
outfile = cfile.parent / (cfile.stem + '.mp4')
if outfile.exists():
outfile.unlink()
command2run = command.format(
file_name=cfile.parent.joinpath(cfile.stem))
process = subprocess.Popen(command2run,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
info, error = process.communicate()
if process.returncode != 0:
_logger.error('COMPRESSION FAILED FOR ' + str(cfile))
with open(cfile.parent.joinpath('extract.error'), 'w+') as fid:
fid.write(command2run)
fid.write(error.decode())
else:
# if the command was successful delete the original file
cfile.unlink()
# and add the file to register_me.flag
flags.write_flag_file(ses_path.joinpath('register_me.flag'),
file_list=cfile.stem)
def extract(subjects_folder, dry=False):
ses_path = Path(subjects_folder).glob('**/extract_me.flag')
for p in ses_path:
# @alejandro no need for flags until personal project data starts going to server
# the flag file may contains specific file names for a targeted extraction
if dry:
print(p)
continue
try:
extract_opto(p.parent, save=True)
except:
pass
#p.unlink() commented we need the flag for the sync merge step
save = flags.read_flag_file(p)
flags.write_flag_file(p.parent.joinpath('opto_extracted.flag'), file_list=save)
def register_sync(self, root_data_folder, dry=False):
"""
Register sessions looking recursively for flag files
:param root_data_folder: folder to look for register_me.flag
:param dry: bool. Dry run if True
:return:
"""
flag_files = Path(root_data_folder).glob('**/register_me.flag')
for flag_file in flag_files:
if dry:
continue
file_list = flags.read_flag_file(flag_file)
_logger.info('registering ' + str(flag_file.parent))
self.register_session(flag_file.parent, file_list=file_list)
flags.write_flag_file(flag_file.parent.joinpath('flatiron.flag'),
file_list=file_list)
flag_file.unlink()
if flag_file.parent.joinpath('create_me.flag').exists():
flag_file.parent.joinpath('create_me.flag').unlink()
_logger.info('registered' + '\n')
def bulk(subjects_folder, dry=False):
ses_path = Path(subjects_folder).glob('**/extract_me.flag')
for p in ses_path:
# the flag file may contains specific file names for a targeted extraction
save = flags.read_flag_file(p)
if dry:
print(p)
continue
try:
from_path(p.parent, force=True, save=save)
except Exception as e:
error_message = str(
p.parent) + ' failed extraction' + '\n ' + str(e)
error_message += traceback.format_exc()
logger_.error(error_message)
err_file = p.parent.joinpath('extract_me.error')
p.rename(err_file)
with open(err_file, 'w+') as f:
f.write(error_message)
continue
p.rename(p.parent.joinpath('register_me.flag'))