def get_project_open_tasks(self, xnat, project_id): task_list = [] # Get lists of processors for this project sess_proc_list, scan_proc_list = processors.processors_by_type(self.project_process_dict[project_id]) # Get lists of assessors for this project assr_list = XnatUtils.list_project_assessors(xnat, project_id) # Match each assessor to a processor, get a task, and add to list for assr_info in assr_list: if assr_info['procstatus'] not in task.OPEN_STATUS_LIST and assr_info['qcstatus'] not in task.OPEN_QC_LIST: continue task_proc = self.match_proc(xnat, assr_info, sess_proc_list, scan_proc_list) if task_proc == None: print('WARN:no matching processor found:'+assr_info['assessor_label']) continue # Get a new task with the matched processor assr = XnatUtils.get_full_object(xnat, assr_info) cur_task = Task(task_proc,assr,self.upload_dir) task_list.append(cur_task) return task_list
def update_session(self, xnat, sess_info, sess_proc_list, scan_proc_list, sess_mod_list, scan_mod_list): # Scans if scan_proc_list or scan_mod_list: scan_list = XnatUtils.list_scans(xnat, sess_info['project'], sess_info['subject_ID'], sess_info['ID']) for scan_info in scan_list: print' +SCAN: '+scan_info['scan_id'] self.update_scan(xnat, scan_info, scan_proc_list, scan_mod_list) # Modules for sess_mod in sess_mod_list: print' * Module: '+sess_mod.getname() sess_obj = None if (sess_mod.needs_run(sess_info, xnat)): if sess_obj == None: sess_obj = XnatUtils.get_full_object(xnat, sess_info) sess_mod.run(sess_info, sess_obj) # Processors for sess_proc in sess_proc_list: if sess_proc.should_run(sess_info, xnat): sess_task = sess_proc.get_task(xnat, sess_info, self.upload_dir) print(' *Processor:'+sess_proc.name+':updating status:'+sess_task.assessor_label) sess_task.update_status()
def set_session_lastupdated(self, xnat, sess_info): # We set update to one minute into the future since setting update field will change last modified time now = (datetime.now() + timedelta(minutes=1)).strftime(UPDATE_FORMAT) print('DEBUG:setting last_updated for:'+sess_info['label']+' to '+now) sess_obj = XnatUtils.get_full_object(xnat, sess_info) xsi_type = sess_info['xsiType'] sess_obj.attrs.set(xsi_type+'/original', UPDATE_PREFIX+now)
def build_scan(self, xnat, cscan, scan_proc_list, scan_mod_list): """ Build the scan :param xnat: pyxnat.Interface object :param cscan: CachedImageScan from XnatUtils :param scan_proc_list: list of processors running on a scan :param scan_mod_list: list of modules running on a scan :return: None """ scan_info = cscan.info() scan_obj = None # Modules for scan_mod in scan_mod_list: LOGGER.debug('* Module: '+scan_mod.getname()) if scan_mod.needs_run(cscan, xnat): if scan_obj == None: scan_obj = XnatUtils.get_full_object(xnat, scan_info) scan_mod.run(scan_info, scan_obj) # Processors for scan_proc in scan_proc_list: if scan_proc.should_run(scan_info): assr_name = scan_proc.get_assessor_name(cscan) # Look for existing assessor proc_assr = None for assr in cscan.parent().assessors(): if assr.info()['label'] == assr_name: proc_assr = assr # Create it if it doesn't exist if proc_assr == None or proc_assr.info()['procstatus'] == task.NEED_INPUTS: scan_task = scan_proc.get_task(xnat, cscan, RESULTS_DIR) self.log_updating_status(scan_proc.name, scan_task.assessor_label) has_inputs, qcstatus = scan_proc.has_inputs(cscan) if has_inputs == 1: scan_task.set_status(task.NEED_TO_RUN) scan_task.set_qcstatus(task.JOB_PENDING) elif has_inputs == -1: scan_task.set_status(task.NO_DATA) scan_task.set_qcstatus(qcstatus) else: scan_task.set_qcstatus(qcstatus) else: # Other statuses handled by dax_update_open_tasks pass
def get_task(self, intf, csess, upload_dir): """ Return the Task object :param intf: XNAT interface see pyxnat.Interface :param csess: CachedImageSession from XnatUtils :param upload_dir: directory to put the data after run on the node :return: Task object of the assessor """ sess_info = csess.info() assessor_name = self.get_assessor_name(csess) session = XnatUtils.get_full_object(intf, sess_info) assessor = session.assessor(assessor_name) return task.Task(self, assessor, upload_dir)
def get_task(self, intf, csess, upload_dir): """ Return the Task object :param intf: XNAT interface see pyxnat.Interface :param csess: CachedImageSession from XnatUtils :param upload_dir: directory to put the data after run on the node :return: Task object of the assessor """ sess_info = csess.info() assessor_name = self.get_assessor_name(csess) session = XnatUtils.get_full_object(intf, sess_info) assessor = session.assessor(assessor_name) return task.Task(self, assessor, upload_dir)
def get_task(self, intf, cscan, upload_dir): """ Get the Task object :param intf: XNAT interface (pyxnat.Interface class) :param cscan: CachedImageScan object from XnatUtils :param upload_dir: the directory to put the processed data when the process is done :return: Task object """ scan_dict = cscan.info() assessor_name = self.get_assessor_name(cscan) scan = XnatUtils.get_full_object(intf, scan_dict) assessor = scan.parent().assessor(assessor_name) return task.Task(self, assessor, upload_dir)
def get_task(self, intf, cscan, upload_dir): """ Get the Task object :param intf: XNAT interface (pyxnat.Interface class) :param cscan: CachedImageScan object from XnatUtils :param upload_dir: the directory to put the processed data when the process is done :return: Task object """ scan_dict = cscan.info() assessor_name = self.get_assessor_name(cscan) scan = XnatUtils.get_full_object(intf, scan_dict) assessor = scan.parent().assessor(assessor_name) return task.Task(self, assessor, upload_dir)
def update_scan(self, xnat, scan_info, scan_proc_list, scan_mod_list): # Modules scan_obj = None for scan_mod in scan_mod_list: print' * Module: '+scan_mod.getname() if (scan_mod.needs_run(scan_info, xnat)): if scan_obj == None: scan_obj = XnatUtils.get_full_object(xnat, scan_info) scan_mod.run(scan_info, scan_obj) # Processors for scan_proc in scan_proc_list: if scan_proc.should_run(scan_info): scan_task = scan_proc.get_task(xnat, scan_info, self.upload_dir) print(' *Processor: '+scan_proc.name+':updating status:'+scan_task.assessor_label) scan_task.update_status()
def generate_task(self, xnat, assr_info, sess_proc_list, scan_proc_list): """ Generate a task for the assessor in the info :param xnat: pyxnat.Interface object :param assr_info: dictionary containing the assessor info (See XnatUtils.list_assessors) :param sess_proc_list: list of processors running on a session :param scan_proc_list: list of processors running on a scan :return: task if processor and assessor match, None otherwise """ task_proc = self.match_proc(assr_info, sess_proc_list, scan_proc_list) if task_proc == None: LOGGER.warn('no matching processor found:'+assr_info['assessor_label']) return None else: # Get a new task with the matched processor assr = XnatUtils.get_full_object(xnat, assr_info) cur_task = Task(task_proc, assr, RESULTS_DIR) return cur_task
def build_scan_modules(self, xnat, cscan, scan_mod_list): scan_info = cscan.info() scan_obj = None # Modules for scan_mod in scan_mod_list: LOGGER.debug('* Module: ' + scan_mod.getname()) if scan_mod.needs_run(cscan, xnat): if scan_obj == None: scan_obj = XnatUtils.get_full_object(xnat, scan_info) try: scan_mod.run(scan_info, scan_obj) except Exception as E: LOGGER.critical( 'Caught exception building session scan module in session %s' % scan_info['session_label']) LOGGER.critical( 'Exception class %s caught with message %s' % (E.__class__, E.message))
def generate_task(self, xnat, assr_info, sess_proc_list, scan_proc_list): """ Generate a task for the assessor in the info :param xnat: pyxnat.Interface object :param assr_info: dictionary containing the assessor info (See XnatUtils.list_assessors) :param sess_proc_list: list of processors running on a session :param scan_proc_list: list of processors running on a scan :return: task if processor and assessor match, None otherwise """ task_proc = self.match_proc(assr_info, sess_proc_list, scan_proc_list) if task_proc == None: LOGGER.warn('no matching processor found:' + assr_info['assessor_label']) return None else: # Get a new task with the matched processor assr = XnatUtils.get_full_object(xnat, assr_info) cur_task = Task(task_proc, assr, DAX_SETTINGS.get_results_dir()) return cur_task
def set_session_lastupdated(xnat, sess_info, update_start_time): """ Set the last session update on XNAT :param xnat: pyxnat.Interface object :param sess_info: dictionary of session information :param update_start_time: date when the update started :return: False if the session change and don't set the last update date, True otherwise """ xsi_type = sess_info['xsiType'] sess_obj = XnatUtils.get_full_object(xnat, sess_info) last_modified_xnat = sess_obj.attrs.get(xsi_type+'/meta/last_modified') last_mod = datetime.strptime(last_modified_xnat[0:19], '%Y-%m-%d %H:%M:%S') if last_mod > update_start_time: return False else: #format: update_str = (datetime.now()+timedelta(minutes=1)).strftime(UPDATE_FORMAT) # We set update to one minute into the future # since setting update field will change last modified time LOGGER.debug('setting last_updated for:'+sess_info['label']+' to '+update_str) sess_obj.attrs.set(xsi_type+'/original', UPDATE_PREFIX+update_str) return True
def set_session_lastupdated(xnat, sess_info, update_start_time): """ Set the last session update on XNAT :param xnat: pyxnat.Interface object :param sess_info: dictionary of session information :param update_start_time: date when the update started :return: False if the session change and don't set the last update date, True otherwise """ xsi_type = sess_info['xsiType'] sess_obj = XnatUtils.get_full_object(xnat, sess_info) last_modified_xnat = sess_obj.attrs.get(xsi_type + '/meta/last_modified') last_mod = datetime.strptime(last_modified_xnat[0:19], '%Y-%m-%d %H:%M:%S') if last_mod > update_start_time: return False # format: update_str = (datetime.now() + timedelta(minutes=1)).strftime(UPDATE_FORMAT) # We set update to one minute into the future # since setting update field will change last modified time LOGGER.debug('setting last_updated for:' + sess_info['label'] + ' to ' + update_str) try: sess_obj.attrs.set(xsi_type + '/original', UPDATE_PREFIX + update_str) except Exception as E: LOGGER.critical( 'Caught exception setting update timestamp for session %s' % sess_info['session_label']) LOGGER.critical('Exception class %s caught with message %s' % (E.__class__, E.message)) return True
def get_task(self, intf, scan_dict, upload_dir): scan = XnatUtils.get_full_object(intf,scan_dict) assessor_name = self.get_assessor_name(scan_dict) assessor = scan.parent().assessor(assessor_name) return task.Task(self,assessor,upload_dir)
def build_scan_processors(self, xnat, cscan, scan_proc_list): """ Build the scan :param xnat: pyxnat.Interface object :param cscan: CachedImageScan from XnatUtils :param scan_proc_list: list of processors running on a scan :param scan_mod_list: list of modules running on a scan :return: None """ scan_info = cscan.info() # Processors for scan_proc in scan_proc_list: if not scan_proc.should_run(scan_info): continue assr_name = scan_proc.get_assessor_name(cscan) # Look for existing assessor proc_assr = None for assr in cscan.parent().assessors(): if assr.info()['label'] == assr_name: proc_assr = assr if self.launcher_type in ['diskq-xnat', 'diskq-combined']: if proc_assr == None or proc_assr.info()['procstatus'] in [ task.NEED_INPUTS, task.NEED_TO_RUN ] or proc_assr.info()['qcstatus'] in [task.RERUN, task.REPROC]: # TODO: get session object directly scan = XnatUtils.get_full_object(xnat, scan_info) assessor = scan.parent().assessor(assr_name) xtask = XnatTask( scan_proc, assessor, DAX_SETTINGS.get_results_dir(), os.path.join(DAX_SETTINGS.get_results_dir(), 'DISKQ')) if proc_assr != None and proc_assr.info()['qcstatus'] in [ task.RERUN, task.REPROC ]: xtask.update_status() LOGGER.debug('building task:' + assr_name) (proc_status, qc_status) = xtask.build_task(cscan, self.root_job_dir, self.job_email, self.job_email_options) LOGGER.debug('proc_status=' + proc_status + ', qc_status=' + qc_status) else: # TODO: check that it actually exists in QUEUE LOGGER.debug('skipping, already built:' + assr_name) else: if proc_assr == None or proc_assr.info( )['procstatus'] == task.NEED_INPUTS: scan_task = scan_proc.get_task( xnat, cscan, DAX_SETTINGS.get_results_dir()) log_updating_status(scan_proc.name, scan_task.assessor_label) has_inputs, qcstatus = scan_proc.has_inputs(cscan) try: if has_inputs == 1: scan_task.set_status(task.NEED_TO_RUN) scan_task.set_qcstatus(task.JOB_PENDING) elif has_inputs == -1: scan_task.set_status(task.NO_DATA) scan_task.set_qcstatus(qcstatus) else: scan_task.set_qcstatus(qcstatus) except Exception as E: LOGGER.critical( 'Caught exception building sessions %s' % scan_info['session_label']) LOGGER.critical( 'Exception class %s caught with message %s' % (E.__class__, E.message)) else: # Other statuses handled by dax_update_open_tasks pass
def get_sess_lastmod(xnat, sess_info): xsi_type = sess_info['xsiType'] sess_obj = XnatUtils.get_full_object(xnat, sess_info) last_modified_xnat = sess_obj.attrs.get(xsi_type + '/meta/last_modified') last_mod = datetime.strptime(last_modified_xnat[0:19], '%Y-%m-%d %H:%M:%S') return last_mod
def get_task(self, intf, session_dict, upload_dir): session = XnatUtils.get_full_object(intf,session_dict) assessor_name = self.get_assessor_name(session_dict) assessor = session.assessor(assessor_name) return task.Task(self,assessor,upload_dir)
def build_session(self, xnat, sess_info, sess_proc_list, scan_proc_list, sess_mod_list, scan_mod_list): """ Build a session :param xnat: pyxnat.Interface object :param sess_info: python ditionary from XnatUtils.list_sessions method :param sess_proc_list: list of processors running on a session :param scan_proc_list: list of processors running on a scan :param sess_mod_list: list of modules running on a session :param scan_mod_list: list of modules running on a scan :return: None """ csess = XnatUtils.CachedImageSession(xnat, sess_info['project_label'], sess_info['subject_label'], sess_info['session_label']) session_info = csess.info() sess_obj = None # Modules on session LOGGER.debug('== Build modules for session ==') for sess_mod in sess_mod_list: LOGGER.debug('* Module: '+sess_mod.getname()) if sess_mod.needs_run(csess, xnat): if sess_obj == None: sess_obj = XnatUtils.get_full_object(xnat, session_info) sess_mod.run(session_info, sess_obj) # Scans LOGGER.debug('== Build modules/processors for scans in session ==') if scan_proc_list or scan_mod_list: for cscan in csess.scans(): LOGGER.debug('+SCAN: '+cscan.info()['scan_id']) self.build_scan(xnat, cscan, scan_proc_list, scan_mod_list) # Processors LOGGER.debug('== Build processors for session ==') for sess_proc in sess_proc_list: if sess_proc.should_run(session_info): assr_name = sess_proc.get_assessor_name(csess) # Look for existing assessor proc_assr = None for assr in csess.assessors(): if assr.info()['label'] == assr_name: proc_assr = assr if proc_assr == None: # Create it if it doesn't exist sess_task = sess_proc.get_task(xnat, csess, RESULTS_DIR) self.log_updating_status(sess_proc.name, sess_task.assessor_label) has_inputs, qcstatus = sess_proc.has_inputs(csess) if has_inputs == 1: sess_task.set_status(task.NEED_TO_RUN) sess_task.set_qcstatus(task.JOB_PENDING) elif has_inputs == -1: sess_task.set_status(task.NO_DATA) sess_task.set_qcstatus(qcstatus) elif proc_assr.info()['procstatus'] == task.NEED_INPUTS: has_inputs, qcstatus = sess_proc.has_inputs(csess) if has_inputs == 1: sess_task = sess_proc.get_task(xnat, csess, RESULTS_DIR) self.log_updating_status(sess_proc.name, sess_task.assessor_label) sess_task.set_status(task.NEED_TO_RUN) sess_task.set_qcstatus(task.JOB_PENDING) elif has_inputs == -1: sess_task = sess_proc.get_task(xnat, csess, RESULTS_DIR) self.log_updating_status(sess_proc.name, sess_task.assessor_label) sess_task.set_status(task.NO_DATA) sess_task.set_qcstatus(qcstatus) else: # Leave as NEED_INPUTS pass else: # Other statuses handled by dax_update_tasks pass