def job_creator(root_path,
one=None,
dry=False,
rerun=False,
max_md5_size=None):
"""
Server function that will look for creation flags and for each:
1) create the sessions on Alyx
2) register the corresponding raw data files on Alyx
3) create the tasks to be run on Alyx
:param root_path: main path containing sessions or session path
:param one
:param dry
:param rerun
:param max_md5_size
:return:
"""
if not one:
one = ONE()
rc = registration.RegistrationClient(one=one)
flag_files = list(Path(root_path).glob('**/extract_me.flag'))
flag_files += list(Path(root_path).glob('**/raw_session.flag'))
all_datasets = []
for flag_file in flag_files:
session_path = flag_file.parent
_logger.info(f'creating session for {session_path}')
if dry:
continue
flag_file.unlink()
# if the subject doesn't exist in the database, skip
try:
rc.create_session(session_path)
except ibllib.exceptions.AlyxSubjectNotFound:
continue
files, dsets = registration.register_session_raw_data(
session_path, one=one, max_md5_size=max_md5_size)
if dsets is not None:
all_datasets.extend(dsets)
session_type = rawio.get_session_extractor_type(session_path)
if session_type in ['biased', 'habituation', 'training']:
pipe = training_preprocessing.TrainingExtractionPipeline(
session_path, one=one)
# only start extracting ephys on a raw_session.flag
elif session_type in ['ephys'
] and flag_file.name == 'raw_session.flag':
pipe = ephys_preprocessing.EphysExtractionPipeline(session_path,
one=one)
else:
_logger.info(
f"Session type {session_type} as no matching extractor {session_path}"
)
return
if rerun:
rerun__status__in = '__all__'
else:
rerun__status__in = ['Waiting']
pipe.create_alyx_tasks(rerun__status__in=rerun__status__in)
return all_datasets
def _run(self, overwrite=False):
out_files, _ = register_session_raw_data(self.session_path,
one=self.one,
dry=True)
return out_files