def get_project_open_tasks(self, xnat, project_id):
task_list = []
# Get lists of processors for this project
sess_proc_list, scan_proc_list = processors.processors_by_type(self.project_process_dict[project_id])
# Get lists of assessors for this project
assr_list = XnatUtils.list_project_assessors(xnat, project_id)
# Match each assessor to a processor, get a task, and add to list
for assr_info in assr_list:
if assr_info['procstatus'] not in task.OPEN_STATUS_LIST and assr_info['qcstatus'] not in task.OPEN_QC_LIST:
continue
task_proc = self.match_proc(xnat, assr_info, sess_proc_list, scan_proc_list)
if task_proc == None:
print('WARN:no matching processor found:'+assr_info['assessor_label'])
continue
# Get a new task with the matched processor
assr = XnatUtils.get_full_object(xnat, assr_info)
cur_task = Task(task_proc,assr,self.upload_dir)
task_list.append(cur_task)
return task_list
def update_session(self, xnat, sess_info, sess_proc_list, scan_proc_list, sess_mod_list, scan_mod_list):
# Scans
if scan_proc_list or scan_mod_list:
scan_list = XnatUtils.list_scans(xnat, sess_info['project'], sess_info['subject_ID'], sess_info['ID'])
for scan_info in scan_list:
print' +SCAN: '+scan_info['scan_id']
self.update_scan(xnat, scan_info, scan_proc_list, scan_mod_list)
# Modules
for sess_mod in sess_mod_list:
print' * Module: '+sess_mod.getname()
sess_obj = None
if (sess_mod.needs_run(sess_info, xnat)):
if sess_obj == None:
sess_obj = XnatUtils.get_full_object(xnat, sess_info)
sess_mod.run(sess_info, sess_obj)
# Processors
for sess_proc in sess_proc_list:
if sess_proc.should_run(sess_info, xnat):
sess_task = sess_proc.get_task(xnat, sess_info, self.upload_dir)
print(' *Processor:'+sess_proc.name+':updating status:'+sess_task.assessor_label)
sess_task.update_status()
def set_session_lastupdated(self, xnat, sess_info):
# We set update to one minute into the future since setting update field will change last modified time
now = (datetime.now() + timedelta(minutes=1)).strftime(UPDATE_FORMAT)
print('DEBUG:setting last_updated for:'+sess_info['label']+' to '+now)
sess_obj = XnatUtils.get_full_object(xnat, sess_info)
xsi_type = sess_info['xsiType']
sess_obj.attrs.set(xsi_type+'/original', UPDATE_PREFIX+now)
def build_scan(self, xnat, cscan, scan_proc_list, scan_mod_list):
"""
Build the scan
:param xnat: pyxnat.Interface object
:param cscan: CachedImageScan from XnatUtils
:param scan_proc_list: list of processors running on a scan
:param scan_mod_list: list of modules running on a scan
:return: None
"""
scan_info = cscan.info()
scan_obj = None
# Modules
for scan_mod in scan_mod_list:
LOGGER.debug('* Module: '+scan_mod.getname())
if scan_mod.needs_run(cscan, xnat):
if scan_obj == None:
scan_obj = XnatUtils.get_full_object(xnat, scan_info)
scan_mod.run(scan_info, scan_obj)
# Processors
for scan_proc in scan_proc_list:
if scan_proc.should_run(scan_info):
assr_name = scan_proc.get_assessor_name(cscan)
# Look for existing assessor
proc_assr = None
for assr in cscan.parent().assessors():
if assr.info()['label'] == assr_name:
proc_assr = assr
# Create it if it doesn't exist
if proc_assr == None or proc_assr.info()['procstatus'] == task.NEED_INPUTS:
scan_task = scan_proc.get_task(xnat, cscan, RESULTS_DIR)
self.log_updating_status(scan_proc.name, scan_task.assessor_label)
has_inputs, qcstatus = scan_proc.has_inputs(cscan)
if has_inputs == 1:
scan_task.set_status(task.NEED_TO_RUN)
scan_task.set_qcstatus(task.JOB_PENDING)
elif has_inputs == -1:
scan_task.set_status(task.NO_DATA)
scan_task.set_qcstatus(qcstatus)
else:
scan_task.set_qcstatus(qcstatus)
else:
# Other statuses handled by dax_update_open_tasks
pass
def get_task(self, intf, csess, upload_dir):
"""
Return the Task object
:param intf: XNAT interface see pyxnat.Interface
:param csess: CachedImageSession from XnatUtils
:param upload_dir: directory to put the data after run on the node
:return: Task object of the assessor
"""
sess_info = csess.info()
assessor_name = self.get_assessor_name(csess)
session = XnatUtils.get_full_object(intf, sess_info)
assessor = session.assessor(assessor_name)
return task.Task(self, assessor, upload_dir)
def get_task(self, intf, csess, upload_dir):
"""
Return the Task object
:param intf: XNAT interface see pyxnat.Interface
:param csess: CachedImageSession from XnatUtils
:param upload_dir: directory to put the data after run on the node
:return: Task object of the assessor
"""
sess_info = csess.info()
assessor_name = self.get_assessor_name(csess)
session = XnatUtils.get_full_object(intf, sess_info)
assessor = session.assessor(assessor_name)
return task.Task(self, assessor, upload_dir)
def get_task(self, intf, cscan, upload_dir):
"""
Get the Task object
:param intf: XNAT interface (pyxnat.Interface class)
:param cscan: CachedImageScan object from XnatUtils
:param upload_dir: the directory to put the processed data when the
process is done
:return: Task object
"""
scan_dict = cscan.info()
assessor_name = self.get_assessor_name(cscan)
scan = XnatUtils.get_full_object(intf, scan_dict)
assessor = scan.parent().assessor(assessor_name)
return task.Task(self, assessor, upload_dir)
def get_task(self, intf, cscan, upload_dir):
"""
Get the Task object
:param intf: XNAT interface (pyxnat.Interface class)
:param cscan: CachedImageScan object from XnatUtils
:param upload_dir: the directory to put the processed data when the
process is done
:return: Task object
"""
scan_dict = cscan.info()
assessor_name = self.get_assessor_name(cscan)
scan = XnatUtils.get_full_object(intf, scan_dict)
assessor = scan.parent().assessor(assessor_name)
return task.Task(self, assessor, upload_dir)
def update_scan(self, xnat, scan_info, scan_proc_list, scan_mod_list):
# Modules
scan_obj = None
for scan_mod in scan_mod_list:
print' * Module: '+scan_mod.getname()
if (scan_mod.needs_run(scan_info, xnat)):
if scan_obj == None:
scan_obj = XnatUtils.get_full_object(xnat, scan_info)
scan_mod.run(scan_info, scan_obj)
# Processors
for scan_proc in scan_proc_list:
if scan_proc.should_run(scan_info):
scan_task = scan_proc.get_task(xnat, scan_info, self.upload_dir)
print(' *Processor: '+scan_proc.name+':updating status:'+scan_task.assessor_label)
scan_task.update_status()
def generate_task(self, xnat, assr_info, sess_proc_list, scan_proc_list):
"""
Generate a task for the assessor in the info
:param xnat: pyxnat.Interface object
:param assr_info: dictionary containing the assessor info (See XnatUtils.list_assessors)
:param sess_proc_list: list of processors running on a session
:param scan_proc_list: list of processors running on a scan
:return: task if processor and assessor match, None otherwise
"""
task_proc = self.match_proc(assr_info, sess_proc_list, scan_proc_list)
if task_proc == None:
LOGGER.warn('no matching processor found:'+assr_info['assessor_label'])
return None
else:
# Get a new task with the matched processor
assr = XnatUtils.get_full_object(xnat, assr_info)
cur_task = Task(task_proc, assr, RESULTS_DIR)
return cur_task
def build_scan_modules(self, xnat, cscan, scan_mod_list):
scan_info = cscan.info()
scan_obj = None
# Modules
for scan_mod in scan_mod_list:
LOGGER.debug('* Module: ' + scan_mod.getname())
if scan_mod.needs_run(cscan, xnat):
if scan_obj == None:
scan_obj = XnatUtils.get_full_object(xnat, scan_info)
try:
scan_mod.run(scan_info, scan_obj)
except Exception as E:
LOGGER.critical(
'Caught exception building session scan module in session %s'
% scan_info['session_label'])
LOGGER.critical(
'Exception class %s caught with message %s' %
(E.__class__, E.message))
def generate_task(self, xnat, assr_info, sess_proc_list, scan_proc_list):
"""
Generate a task for the assessor in the info
:param xnat: pyxnat.Interface object
:param assr_info: dictionary containing the assessor info (See XnatUtils.list_assessors)
:param sess_proc_list: list of processors running on a session
:param scan_proc_list: list of processors running on a scan
:return: task if processor and assessor match, None otherwise
"""
task_proc = self.match_proc(assr_info, sess_proc_list, scan_proc_list)
if task_proc == None:
LOGGER.warn('no matching processor found:' +
assr_info['assessor_label'])
return None
else:
# Get a new task with the matched processor
assr = XnatUtils.get_full_object(xnat, assr_info)
cur_task = Task(task_proc, assr, DAX_SETTINGS.get_results_dir())
return cur_task
def set_session_lastupdated(xnat, sess_info, update_start_time):
"""
Set the last session update on XNAT
:param xnat: pyxnat.Interface object
:param sess_info: dictionary of session information
:param update_start_time: date when the update started
:return: False if the session change and don't set the last update date, True otherwise
"""
xsi_type = sess_info['xsiType']
sess_obj = XnatUtils.get_full_object(xnat, sess_info)
last_modified_xnat = sess_obj.attrs.get(xsi_type+'/meta/last_modified')
last_mod = datetime.strptime(last_modified_xnat[0:19], '%Y-%m-%d %H:%M:%S')
if last_mod > update_start_time:
return False
else:
#format:
update_str = (datetime.now()+timedelta(minutes=1)).strftime(UPDATE_FORMAT)
# We set update to one minute into the future
# since setting update field will change last modified time
LOGGER.debug('setting last_updated for:'+sess_info['label']+' to '+update_str)
sess_obj.attrs.set(xsi_type+'/original', UPDATE_PREFIX+update_str)
return True
def set_session_lastupdated(xnat, sess_info, update_start_time):
"""
Set the last session update on XNAT
:param xnat: pyxnat.Interface object
:param sess_info: dictionary of session information
:param update_start_time: date when the update started
:return: False if the session change and don't set the last update date, True otherwise
"""
xsi_type = sess_info['xsiType']
sess_obj = XnatUtils.get_full_object(xnat, sess_info)
last_modified_xnat = sess_obj.attrs.get(xsi_type +
'/meta/last_modified')
last_mod = datetime.strptime(last_modified_xnat[0:19],
'%Y-%m-%d %H:%M:%S')
if last_mod > update_start_time:
return False
# format:
update_str = (datetime.now() +
timedelta(minutes=1)).strftime(UPDATE_FORMAT)
# We set update to one minute into the future
# since setting update field will change last modified time
LOGGER.debug('setting last_updated for:' + sess_info['label'] +
' to ' + update_str)
try:
sess_obj.attrs.set(xsi_type + '/original',
UPDATE_PREFIX + update_str)
except Exception as E:
LOGGER.critical(
'Caught exception setting update timestamp for session %s' %
sess_info['session_label'])
LOGGER.critical('Exception class %s caught with message %s' %
(E.__class__, E.message))
return True
def get_task(self, intf, scan_dict, upload_dir):
scan = XnatUtils.get_full_object(intf,scan_dict)
assessor_name = self.get_assessor_name(scan_dict)
assessor = scan.parent().assessor(assessor_name)
return task.Task(self,assessor,upload_dir)
def build_scan_processors(self, xnat, cscan, scan_proc_list):
"""
Build the scan
:param xnat: pyxnat.Interface object
:param cscan: CachedImageScan from XnatUtils
:param scan_proc_list: list of processors running on a scan
:param scan_mod_list: list of modules running on a scan
:return: None
"""
scan_info = cscan.info()
# Processors
for scan_proc in scan_proc_list:
if not scan_proc.should_run(scan_info):
continue
assr_name = scan_proc.get_assessor_name(cscan)
# Look for existing assessor
proc_assr = None
for assr in cscan.parent().assessors():
if assr.info()['label'] == assr_name:
proc_assr = assr
if self.launcher_type in ['diskq-xnat', 'diskq-combined']:
if proc_assr == None or proc_assr.info()['procstatus'] in [
task.NEED_INPUTS, task.NEED_TO_RUN
] or proc_assr.info()['qcstatus'] in [task.RERUN, task.REPROC]:
# TODO: get session object directly
scan = XnatUtils.get_full_object(xnat, scan_info)
assessor = scan.parent().assessor(assr_name)
xtask = XnatTask(
scan_proc, assessor, DAX_SETTINGS.get_results_dir(),
os.path.join(DAX_SETTINGS.get_results_dir(), 'DISKQ'))
if proc_assr != None and proc_assr.info()['qcstatus'] in [
task.RERUN, task.REPROC
]:
xtask.update_status()
LOGGER.debug('building task:' + assr_name)
(proc_status,
qc_status) = xtask.build_task(cscan, self.root_job_dir,
self.job_email,
self.job_email_options)
LOGGER.debug('proc_status=' + proc_status +
', qc_status=' + qc_status)
else:
# TODO: check that it actually exists in QUEUE
LOGGER.debug('skipping, already built:' + assr_name)
else:
if proc_assr == None or proc_assr.info(
)['procstatus'] == task.NEED_INPUTS:
scan_task = scan_proc.get_task(
xnat, cscan, DAX_SETTINGS.get_results_dir())
log_updating_status(scan_proc.name,
scan_task.assessor_label)
has_inputs, qcstatus = scan_proc.has_inputs(cscan)
try:
if has_inputs == 1:
scan_task.set_status(task.NEED_TO_RUN)
scan_task.set_qcstatus(task.JOB_PENDING)
elif has_inputs == -1:
scan_task.set_status(task.NO_DATA)
scan_task.set_qcstatus(qcstatus)
else:
scan_task.set_qcstatus(qcstatus)
except Exception as E:
LOGGER.critical(
'Caught exception building sessions %s' %
scan_info['session_label'])
LOGGER.critical(
'Exception class %s caught with message %s' %
(E.__class__, E.message))
else:
# Other statuses handled by dax_update_open_tasks
pass
def get_sess_lastmod(xnat, sess_info):
xsi_type = sess_info['xsiType']
sess_obj = XnatUtils.get_full_object(xnat, sess_info)
last_modified_xnat = sess_obj.attrs.get(xsi_type + '/meta/last_modified')
last_mod = datetime.strptime(last_modified_xnat[0:19], '%Y-%m-%d %H:%M:%S')
return last_mod
def get_task(self, intf, session_dict, upload_dir):
session = XnatUtils.get_full_object(intf,session_dict)
assessor_name = self.get_assessor_name(session_dict)
assessor = session.assessor(assessor_name)
return task.Task(self,assessor,upload_dir)
def build_session(self, xnat, sess_info, sess_proc_list,
scan_proc_list, sess_mod_list, scan_mod_list):
"""
Build a session
:param xnat: pyxnat.Interface object
:param sess_info: python ditionary from XnatUtils.list_sessions method
:param sess_proc_list: list of processors running on a session
:param scan_proc_list: list of processors running on a scan
:param sess_mod_list: list of modules running on a session
:param scan_mod_list: list of modules running on a scan
:return: None
"""
csess = XnatUtils.CachedImageSession(xnat,
sess_info['project_label'],
sess_info['subject_label'],
sess_info['session_label'])
session_info = csess.info()
sess_obj = None
# Modules on session
LOGGER.debug('== Build modules for session ==')
for sess_mod in sess_mod_list:
LOGGER.debug('* Module: '+sess_mod.getname())
if sess_mod.needs_run(csess, xnat):
if sess_obj == None:
sess_obj = XnatUtils.get_full_object(xnat, session_info)
sess_mod.run(session_info, sess_obj)
# Scans
LOGGER.debug('== Build modules/processors for scans in session ==')
if scan_proc_list or scan_mod_list:
for cscan in csess.scans():
LOGGER.debug('+SCAN: '+cscan.info()['scan_id'])
self.build_scan(xnat, cscan, scan_proc_list, scan_mod_list)
# Processors
LOGGER.debug('== Build processors for session ==')
for sess_proc in sess_proc_list:
if sess_proc.should_run(session_info):
assr_name = sess_proc.get_assessor_name(csess)
# Look for existing assessor
proc_assr = None
for assr in csess.assessors():
if assr.info()['label'] == assr_name:
proc_assr = assr
if proc_assr == None:
# Create it if it doesn't exist
sess_task = sess_proc.get_task(xnat, csess, RESULTS_DIR)
self.log_updating_status(sess_proc.name, sess_task.assessor_label)
has_inputs, qcstatus = sess_proc.has_inputs(csess)
if has_inputs == 1:
sess_task.set_status(task.NEED_TO_RUN)
sess_task.set_qcstatus(task.JOB_PENDING)
elif has_inputs == -1:
sess_task.set_status(task.NO_DATA)
sess_task.set_qcstatus(qcstatus)
elif proc_assr.info()['procstatus'] == task.NEED_INPUTS:
has_inputs, qcstatus = sess_proc.has_inputs(csess)
if has_inputs == 1:
sess_task = sess_proc.get_task(xnat, csess, RESULTS_DIR)
self.log_updating_status(sess_proc.name, sess_task.assessor_label)
sess_task.set_status(task.NEED_TO_RUN)
sess_task.set_qcstatus(task.JOB_PENDING)
elif has_inputs == -1:
sess_task = sess_proc.get_task(xnat, csess, RESULTS_DIR)
self.log_updating_status(sess_proc.name, sess_task.assessor_label)
sess_task.set_status(task.NO_DATA)
sess_task.set_qcstatus(qcstatus)
else:
# Leave as NEED_INPUTS
pass
else:
# Other statuses handled by dax_update_tasks
pass
