def update_project(self, xnat, project_id, lockfile_prefix): exp_mod_list, scan_mod_list = [],[] exp_proc_list, scan_proc_list = [],[] #Modules prerun print(' *Modules Prerun') self.module_prerun(project_id, lockfile_prefix) # Get lists of modules/processors per scan/exp for this project exp_mod_list, scan_mod_list = modules.modules_by_type(self.project_modules_dict[project_id]) exp_proc_list, scan_proc_list = processors.processors_by_type(self.project_process_dict[project_id]) # Update each session for sess_info in XnatUtils.list_sessions(xnat, project_id): last_mod = datetime.strptime(sess_info['last_modified'][0:19], '%Y-%m-%d %H:%M:%S') last_up = self.get_lastupdated(sess_info) if (last_up != None and last_mod < last_up): print(' +Session:'+sess_info['label']+': skipping, last_mod='+str(last_mod)+',last_up='+str(last_up)) else: print(' +Session:'+sess_info['label']+': updating...') # NOTE: we set update time here, so if the sess is changed below it will be checked again self.set_session_lastupdated(xnat, sess_info) self.update_session(xnat, sess_info, exp_proc_list, scan_proc_list, exp_mod_list, scan_mod_list) # Modules after run print(' *Modules Afterrun') self.module_afterrun(xnat,project_id)
def build_project(self, xnat, project_id, lockfile_prefix, sessions_local): """ Build the project :param xnat: pyxnat.Interface object :param project_id: project ID on XNAT :param lockfile_prefix: prefix for flag file to lock the launcher :param sessions_local: list of sessions to launch tasks :return: None """ #Modules prerun LOGGER.info(' *Modules Prerun') if sessions_local: self.module_prerun(project_id, 'manual_update') else: self.module_prerun(project_id, lockfile_prefix) # Get lists of modules/processors per scan/exp for this project exp_mods, scan_mods = modules.modules_by_type(self.project_modules_dict[project_id]) exp_procs, scan_procs = processors.processors_by_type(self.project_process_dict[project_id]) # Check for new processors has_new = self.has_new_processors(xnat, project_id, exp_procs, scan_procs) # Get the list of sessions: sessions = self.get_sessions_list(xnat, project_id, sessions_local) # Update each session from the list: for sess_info in sessions: last_mod = datetime.strptime(sess_info['last_modified'][0:19], UPDATE_FORMAT) now_date = datetime.today() last_up = self.get_lastupdated(sess_info) #If sessions_local is set, skip checking the date if not has_new and last_up != None and \ last_mod < last_up and not sessions_local and \ now_date < last_mod+timedelta(days=int(self.max_age)): mess = """ +Session:{sess}: skipping, last_mod={mod},last_up={up}""" mess_str = mess.format(sess=sess_info['label'], mod=str(last_mod), up=str(last_up)) LOGGER.info(mess_str) else: update_run_count = 0 got_updated = False while update_run_count < 3 and not got_updated: mess = """ +Session:{sess}: updating (count:{count})...""" LOGGER.info(mess.format(sess=sess_info['label'], count=update_run_count)) # NOTE: we keep the starting time of the update # and will check if something change during the update update_start_time = datetime.now() self.build_session(xnat, sess_info, exp_procs, scan_procs, exp_mods, scan_mods) got_updated = self.set_session_lastupdated(xnat, sess_info, update_start_time) update_run_count = update_run_count+1 LOGGER.debug('\n') if not sessions_local or sessions_local.lower() == 'all': # Modules after run LOGGER.debug('*Modules Afterrun') self.module_afterrun(xnat, project_id)
def build_project(self, xnat, project_id, lockfile_prefix, sessions_local, mod_delta=None): """ Build the project :param xnat: pyxnat.Interface object :param project_id: project ID on XNAT :param lockfile_prefix: prefix for flag file to lock the launcher :param sessions_local: list of sessions to launch tasks :return: None """ #Modules prerun LOGGER.info(' *Modules Prerun') if sessions_local: self.module_prerun(project_id, 'manual_update') else: self.module_prerun(project_id, lockfile_prefix) # TODO: make a project settings to store skip_lastupdate, processors, modules, etc # Get lists of modules/processors per scan/exp for this project exp_mods, scan_mods = modules.modules_by_type( self.project_modules_dict[project_id]) exp_procs, scan_procs = processors.processors_by_type( self.project_process_dict[project_id]) if mod_delta: lastmod_delta = str_to_timedelta(mod_delta) else: lastmod_delta = None # Check for new processors has_new = self.has_new_processors(xnat, project_id, exp_procs, scan_procs) # Get the list of sessions: sessions = self.get_sessions_list(xnat, project_id, sessions_local) # Update each session from the list: for sess_info in sessions: if not self.skip_lastupdate and not has_new and not sessions_local: last_mod = datetime.strptime(sess_info['last_modified'][0:19], UPDATE_FORMAT) now_date = datetime.today() last_up = self.get_lastupdated(sess_info) if last_up != None and \ last_mod < last_up and \ now_date < last_mod + timedelta(days=int(self.max_age)): mess = """ +Session:{sess}: skipping, last_mod={mod},last_up={up}""" mess_str = mess.format(sess=sess_info['label'], mod=str(last_mod), up=str(last_up)) LOGGER.info(mess_str) continue elif lastmod_delta: last_mod = datetime.strptime(sess_info['last_modified'][0:19], UPDATE_FORMAT) now_date = datetime.today() if now_date > last_mod + lastmod_delta: mess = """+Session:{sess}:skipping not modified within delta, last_mod={mod}""" mess_str = mess.format(sess=sess_info['label'], mod=str(last_mod)) LOGGER.info(mess_str) continue else: print('lastmod=' + str(last_mod)) mess = """ +Session:{sess}: building...""" LOGGER.info(mess.format(sess=sess_info['label'])) if not self.skip_lastupdate: update_start_time = datetime.now() try: self.build_session(xnat, sess_info, exp_procs, scan_procs, exp_mods, scan_mods) except Exception as E: LOGGER.critical('Caught exception building sessions %s' % sess_info['session_label']) LOGGER.critical('Exception class %s caught with message %s' % (E.__class__, E.message)) try: if not self.skip_lastupdate: self.set_session_lastupdated(xnat, sess_info, update_start_time) except Exception as E: LOGGER.critical( 'Caught exception setting session timestamp %s' % sess_info['session_label']) LOGGER.critical('Exception class %s caught with message %s' % (E.__class__, E.message)) if not sessions_local or sessions_local.lower() == 'all': # Modules after run LOGGER.debug('*Modules Afterrun') try: self.module_afterrun(xnat, project_id) except Exception as E: LOGGER.critical('Caught exception after running modules %s') LOGGER.critical('Exception class %s caught with message %s' % (E.__class__, E.message))