def testAppendFlagFile(self):
# DO NOT CHANGE THE ORDER OF TESTS BELOW
# prepare a file with 3 dataset types
file_list = [
'_ibl_extraRewards.times', '_ibl_lickPiezo.raw',
'_ibl_lickPiezo.timestamps'
]
with open(self.tempfile.name, 'w+') as fid:
fid.write('\n'.join(file_list))
self.assertEqual(set(flags.read_flag_file(self.tempfile.name)),
set(file_list))
# with an existing file containing files, writing more files append to it
file_list_2 = ['turltu']
# also makes sure that if a string is provided it works
flags.write_flag_file(self.tempfile.name, file_list_2[0])
self.assertEqual(set(flags.read_flag_file(self.tempfile.name)),
set(file_list + file_list_2))
# writing again keeps unique file values
flags.write_flag_file(self.tempfile.name, file_list_2[0])
n = sum([
1 for f in flags.read_flag_file(self.tempfile.name)
if f == file_list_2[0]
])
self.assertEqual(n, 1)
# with an existing file containing files, writing empty filelist returns True for all files
flags.write_flag_file(self.tempfile.name, None)
self.assertEqual(flags.read_flag_file(self.tempfile.name), True)
# with an existing empty file, writing filelist returns True for all files
flags.write_flag_file(self.tempfile.name, ['file1', 'file2'])
self.assertEqual(flags.read_flag_file(self.tempfile.name), True)
# makes sure that read after write empty list also returns True
flags.write_flag_file(self.tempfile.name, [])
self.assertEqual(flags.read_flag_file(self.tempfile.name), True)
# with an existing empty file, writing filelist returns the list if clobber
flags.write_flag_file(self.tempfile.name, ['file1', 'file2', 'file3'],
clobber=True)
self.assertEqual(set(flags.read_flag_file(self.tempfile.name)),
set(['file1', 'file2', 'file3']))
# test the removal of a file within the list
flags.excise_flag_file(self.tempfile.name, removed_files='file1')
self.assertEqual(sorted(flags.read_flag_file(self.tempfile.name)),
['file2', 'file3'])
# if file-list is True it means all files and file_list should be empty after read
flags.write_flag_file(self.tempfile.name, file_list=True)
self.assertEqual(flags.read_flag_file(self.tempfile.name), True)
def _compress(root_data_folder,
command,
flag_pattern,
dry=False,
max_sessions=None):
# runs a command of the form command = "ls -1 {file_name}.avi"
c = 0
for flag_file in Path(root_data_folder).rglob(flag_pattern):
ses_path = flag_file.parent
files2compress = flags.read_flag_file(flag_file)
for f2c in files2compress:
cfile = ses_path.joinpath(PureWindowsPath(f2c))
c += 1
if max_sessions and c > max_sessions:
return
print(cfile)
if dry:
continue
if not cfile.exists():
logger.error('NON-EXISTING RAW FILE: ' + str(cfile))
# run the compression command redirecting output
cfile.parent.joinpath(cfile.stem)
# if the output file already exists, overwrite it
outfile = cfile.parent / (cfile.stem + '.mp4')
if outfile.exists():
outfile.unlink()
command2run = command.format(
file_name=cfile.parent.joinpath(cfile.stem))
process = subprocess.Popen(command2run,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
info, error = process.communicate()
if process.returncode != 0:
logger.error('COMPRESSION FAILED FOR ' + str(cfile))
flags.excise_flag_file(flag_file, removed_files=f2c)
with open(cfile.parent.joinpath('extract.error'), 'w+') as fid:
fid.write(command2run)
fid.write(error.decode())
continue
# if the command was successful delete the original file
cfile.unlink()
# then remove the file from the compress flag file
flags.excise_flag_file(flag_file, removed_files=f2c)
# and add the file to register_me.flag
flags.write_flag_file(ses_path.joinpath('register_me.flag'),
file_list=cfile.stem)
