def get_project_tasks(self, xnat, project_id, sessions_local,
is_valid_assessor):
"""
Get list of tasks for a specific project where each task agrees
the is_valid_assessor conditions
:param xnat: pyxnat.Interface object
:param project_id: project ID on XNAT
:param sessions_local: list of sessions to update tasks associated
to the project locally
:param is_valid_assessor: method to validate the assessor
:return: list of tasks
"""
task_list = list()
# Get lists of processors for this project
pp_dict = self.project_process_dict[project_id]
sess_procs, scan_procs = processors.processors_by_type(pp_dict)
# Get lists of assessors for this project
assr_list = self.get_assessors_list(xnat, project_id, sessions_local)
# Match each assessor to a processor, get a task, and add to list
for assr_info in assr_list:
if is_valid_assessor(assr_info):
cur_task = self.generate_task(xnat, assr_info, sess_procs,
scan_procs)
if cur_task:
task_list.append(cur_task)
return task_list
def get_project_tasks(self, xnat, project_id, sessions_local, is_valid_assessor):
"""
Get list of tasks for a specific project where each task agrees
the is_valid_assessor conditions
:param xnat: pyxnat.Interface object
:param project_id: project ID on XNAT
:param sessions_local: list of sessions to update tasks associated
to the project locally
:param is_valid_assessor: method to validate the assessor
:return: list of tasks
"""
task_list = list()
# Get lists of processors for this project
pp_dict = self.project_process_dict[project_id]
sess_procs, scan_procs = processors.processors_by_type(pp_dict)
# Get lists of assessors for this project
assr_list = self.get_assessors_list(xnat, project_id, sessions_local)
# Match each assessor to a processor, get a task, and add to list
for assr_info in assr_list:
if is_valid_assessor(assr_info):
cur_task = self.generate_task(xnat, assr_info, sess_procs, scan_procs)
if cur_task:
task_list.append(cur_task)
return task_list
def update_project(self, xnat, project_id, lockfile_prefix):
exp_mod_list, scan_mod_list = [],[]
exp_proc_list, scan_proc_list = [],[]
#Modules prerun
print(' *Modules Prerun')
self.module_prerun(project_id, lockfile_prefix)
# Get lists of modules/processors per scan/exp for this project
exp_mod_list, scan_mod_list = modules.modules_by_type(self.project_modules_dict[project_id])
exp_proc_list, scan_proc_list = processors.processors_by_type(self.project_process_dict[project_id])
# Update each session
for sess_info in XnatUtils.list_sessions(xnat, project_id):
last_mod = datetime.strptime(sess_info['last_modified'][0:19], '%Y-%m-%d %H:%M:%S')
last_up = self.get_lastupdated(sess_info)
if (last_up != None and last_mod < last_up):
print(' +Session:'+sess_info['label']+': skipping, last_mod='+str(last_mod)+',last_up='+str(last_up))
else:
print(' +Session:'+sess_info['label']+': updating...')
# NOTE: we set update time here, so if the sess is changed below it will be checked again
self.set_session_lastupdated(xnat, sess_info)
self.update_session(xnat, sess_info, exp_proc_list, scan_proc_list, exp_mod_list, scan_mod_list)
# Modules after run
print(' *Modules Afterrun')
self.module_afterrun(xnat,project_id)
def get_project_open_tasks(self, xnat, project_id):
task_list = []
# Get lists of processors for this project
sess_proc_list, scan_proc_list = processors.processors_by_type(self.project_process_dict[project_id])
# Get lists of assessors for this project
assr_list = XnatUtils.list_project_assessors(xnat, project_id)
# Match each assessor to a processor, get a task, and add to list
for assr_info in assr_list:
if assr_info['procstatus'] not in task.OPEN_STATUS_LIST and assr_info['qcstatus'] not in task.OPEN_QC_LIST:
continue
task_proc = self.match_proc(xnat, assr_info, sess_proc_list, scan_proc_list)
if task_proc == None:
print('WARN:no matching processor found:'+assr_info['assessor_label'])
continue
# Get a new task with the matched processor
assr = XnatUtils.get_full_object(xnat, assr_info)
cur_task = Task(task_proc,assr,self.upload_dir)
task_list.append(cur_task)
return task_list
def build_project(self, xnat, project_id, lockfile_prefix, sessions_local):
"""
Build the project
:param xnat: pyxnat.Interface object
:param project_id: project ID on XNAT
:param lockfile_prefix: prefix for flag file to lock the launcher
:param sessions_local: list of sessions to launch tasks
:return: None
"""
#Modules prerun
LOGGER.info(' *Modules Prerun')
if sessions_local:
self.module_prerun(project_id, 'manual_update')
else:
self.module_prerun(project_id, lockfile_prefix)
# Get lists of modules/processors per scan/exp for this project
exp_mods, scan_mods = modules.modules_by_type(self.project_modules_dict[project_id])
exp_procs, scan_procs = processors.processors_by_type(self.project_process_dict[project_id])
# Check for new processors
has_new = self.has_new_processors(xnat, project_id, exp_procs, scan_procs)
# Get the list of sessions:
sessions = self.get_sessions_list(xnat, project_id, sessions_local)
# Update each session from the list:
for sess_info in sessions:
last_mod = datetime.strptime(sess_info['last_modified'][0:19], UPDATE_FORMAT)
now_date = datetime.today()
last_up = self.get_lastupdated(sess_info)
#If sessions_local is set, skip checking the date
if not has_new and last_up != None and \
last_mod < last_up and not sessions_local and \
now_date < last_mod+timedelta(days=int(self.max_age)):
mess = """ +Session:{sess}: skipping, last_mod={mod},last_up={up}"""
mess_str = mess.format(sess=sess_info['label'], mod=str(last_mod), up=str(last_up))
LOGGER.info(mess_str)
else:
update_run_count = 0
got_updated = False
while update_run_count < 3 and not got_updated:
mess = """ +Session:{sess}: updating (count:{count})..."""
LOGGER.info(mess.format(sess=sess_info['label'], count=update_run_count))
# NOTE: we keep the starting time of the update
# and will check if something change during the update
update_start_time = datetime.now()
self.build_session(xnat, sess_info, exp_procs, scan_procs, exp_mods, scan_mods)
got_updated = self.set_session_lastupdated(xnat, sess_info, update_start_time)
update_run_count = update_run_count+1
LOGGER.debug('\n')
if not sessions_local or sessions_local.lower() == 'all':
# Modules after run
LOGGER.debug('*Modules Afterrun')
self.module_afterrun(xnat, project_id)
def build_project(self,
xnat,
project_id,
lockfile_prefix,
sessions_local,
mod_delta=None):
"""
Build the project
:param xnat: pyxnat.Interface object
:param project_id: project ID on XNAT
:param lockfile_prefix: prefix for flag file to lock the launcher
:param sessions_local: list of sessions to launch tasks
:return: None
"""
#Modules prerun
LOGGER.info(' *Modules Prerun')
if sessions_local:
self.module_prerun(project_id, 'manual_update')
else:
self.module_prerun(project_id, lockfile_prefix)
# TODO: make a project settings to store skip_lastupdate, processors, modules, etc
# Get lists of modules/processors per scan/exp for this project
exp_mods, scan_mods = modules.modules_by_type(
self.project_modules_dict[project_id])
exp_procs, scan_procs = processors.processors_by_type(
self.project_process_dict[project_id])
if mod_delta:
lastmod_delta = str_to_timedelta(mod_delta)
else:
lastmod_delta = None
# Check for new processors
has_new = self.has_new_processors(xnat, project_id, exp_procs,
scan_procs)
# Get the list of sessions:
sessions = self.get_sessions_list(xnat, project_id, sessions_local)
# Update each session from the list:
for sess_info in sessions:
if not self.skip_lastupdate and not has_new and not sessions_local:
last_mod = datetime.strptime(sess_info['last_modified'][0:19],
UPDATE_FORMAT)
now_date = datetime.today()
last_up = self.get_lastupdated(sess_info)
if last_up != None and \
last_mod < last_up and \
now_date < last_mod + timedelta(days=int(self.max_age)):
mess = """ +Session:{sess}: skipping, last_mod={mod},last_up={up}"""
mess_str = mess.format(sess=sess_info['label'],
mod=str(last_mod),
up=str(last_up))
LOGGER.info(mess_str)
continue
elif lastmod_delta:
last_mod = datetime.strptime(sess_info['last_modified'][0:19],
UPDATE_FORMAT)
now_date = datetime.today()
if now_date > last_mod + lastmod_delta:
mess = """+Session:{sess}:skipping not modified within delta, last_mod={mod}"""
mess_str = mess.format(sess=sess_info['label'],
mod=str(last_mod))
LOGGER.info(mess_str)
continue
else:
print('lastmod=' + str(last_mod))
mess = """ +Session:{sess}: building..."""
LOGGER.info(mess.format(sess=sess_info['label']))
if not self.skip_lastupdate:
update_start_time = datetime.now()
try:
self.build_session(xnat, sess_info, exp_procs, scan_procs,
exp_mods, scan_mods)
except Exception as E:
LOGGER.critical('Caught exception building sessions %s' %
sess_info['session_label'])
LOGGER.critical('Exception class %s caught with message %s' %
(E.__class__, E.message))
try:
if not self.skip_lastupdate:
self.set_session_lastupdated(xnat, sess_info,
update_start_time)
except Exception as E:
LOGGER.critical(
'Caught exception setting session timestamp %s' %
sess_info['session_label'])
LOGGER.critical('Exception class %s caught with message %s' %
(E.__class__, E.message))
if not sessions_local or sessions_local.lower() == 'all':
# Modules after run
LOGGER.debug('*Modules Afterrun')
try:
self.module_afterrun(xnat, project_id)
except Exception as E:
LOGGER.critical('Caught exception after running modules %s')
LOGGER.critical('Exception class %s caught with message %s' %
(E.__class__, E.message))
