def build(self, lockfile_prefix, project_local, sessions_local):
"""
Main method to build the tasks and the sessions
:param lockfile_prefix: prefix for flag file to lock the launcher
:param project_local: project to run locally
:param sessions_local: list of sessions to launch tasks
associated to the project locally
:return: None
"""
LOGGER.info('-------------- Build --------------\n')
flagfile = os.path.join(RESULTS_DIR, 'FlagFiles', lockfile_prefix+'_'+BUILD_SUFFIX)
project_list = self.init_script(flagfile, project_local, type_update=1, start_end=1)
try:
LOGGER.info('Connecting to XNAT at '+self.xnat_host)
xnat = XnatUtils.get_interface(self.xnat_host, self.xnat_user, self.xnat_pass)
if not XnatUtils.has_dax_datatypes(xnat):
raise Exception('error: dax datatypes are not installed on your xnat <%s>' % (self.xnat_host))
#Priority if set:
if self.priority_project and not project_local:
unique_list = set(self.project_process_dict.keys()+self.project_modules_dict.keys())
project_list = self.get_project_list(list(unique_list))
# Build projects
for project_id in project_list:
LOGGER.info('===== PROJECT:'+project_id+' =====')
self.build_project(xnat, project_id, lockfile_prefix, sessions_local)
finally:
self.finish_script(xnat, flagfile, project_list, 1, 2, project_local)
def pi_from_project(project):
"""
Get the last name of PI who owns the project on XNAT
:param project: String of the ID of project on XNAT.
:return: String of the PIs last name
"""
pi_name = ''
try:
xnat = XnatUtils.get_interface()
proj = xnat.select.project(project)
pi_name = proj.attrs.get('xnat:projectdata/pi/lastname')
except:
pass
finally:
xnat.disconnect()
return pi_name
def pi_from_project(project):
"""
Get the last name of PI who owns the project on XNAT
:param project: String of the ID of project on XNAT.
:return: String of the PIs last name
"""
pi_name = ''
try:
xnat = XnatUtils.get_interface()
proj = xnat.select.project(project)
pi_name = proj.attrs.get('xnat:projectdata/pi/lastname')
except:
pass
finally:
xnat.disconnect()
return pi_name
def update_tasks(self, lockfile_prefix, project_local, sessions_local):
"""
Main method to Update the tasks
:param lockfile_prefix: prefix for flag file to lock the launcher
:param project_local: project to run locally
:param sessions_local: list of sessions to update tasks associated
to the project locally
:return: None
"""
LOGGER.info('-------------- Update Tasks --------------\n')
flagfile = os.path.join(RESULTS_DIR, 'FlagFiles', lockfile_prefix+'_'+UPDATE_SUFFIX)
project_list = self.init_script(flagfile, project_local, type_update=2, start_end=1)
try:
LOGGER.info('Connecting to XNAT at '+self.xnat_host)
xnat = XnatUtils.get_interface(self.xnat_host, self.xnat_user, self.xnat_pass)
if not XnatUtils.has_dax_datatypes(xnat):
raise Exception('error: dax datatypes are not installed on your xnat <%s>' % (self.xnat_host))
LOGGER.info('Getting task list...')
task_list = self.get_tasks(xnat,
self.is_updatable_tasks,
project_list,
sessions_local)
LOGGER.info(str(len(task_list))+' open tasks found')
LOGGER.info('Updating tasks...')
for cur_task in task_list:
LOGGER.info(' Updating task:'+cur_task.assessor_label)
cur_task.update_status()
finally:
self.finish_script(xnat, flagfile, project_list, 2, 2, project_local)
def launch_jobs(self, lockfile_prefix, project_local, sessions_local, writeonly=False, pbsdir=None):
"""
Main Method to launch the tasks
:param lockfile_prefix: prefix for flag file to lock the launcher
:param project_local: project to run locally
:param sessions_local: list of sessions to launch tasks
associated to the project locally
:param writeonly: write the job files without submitting them
:param pbsdir: folder to store the pbs file
:return: None
"""
LOGGER.info('-------------- Launch Tasks --------------\n')
flagfile = os.path.join(RESULTS_DIR, 'FlagFiles', lockfile_prefix+'_'+LAUNCH_SUFFIX)
project_list = self.init_script(flagfile, project_local, type_update=3, start_end=1)
try:
LOGGER.info('Connecting to XNAT at '+self.xnat_host)
xnat = XnatUtils.get_interface(self.xnat_host, self.xnat_user, self.xnat_pass)
if not XnatUtils.has_dax_datatypes(xnat):
raise Exception('error: dax datatypes are not installed on your xnat <%s>' % (self.xnat_host))
LOGGER.info('Getting launchable tasks list...')
task_list = self.get_tasks(xnat,
self.is_launchable_tasks,
project_list,
sessions_local)
LOGGER.info(str(len(task_list))+' tasks that need to be launched found')
#Launch the task that need to be launch
self.launch_tasks(task_list, writeonly, pbsdir)
finally:
self.finish_script(xnat, flagfile, project_list, 3, 2, project_local)
def build(self,
lockfile_prefix,
project_local,
sessions_local,
mod_delta=None):
"""
Main method to build the tasks and the sessions
:param lockfile_prefix: prefix for flag file to lock the launcher
:param project_local: project to run locally
:param sessions_local: list of sessions to launch tasks
associated to the project locally
:return: None
"""
if self.launcher_type == 'diskq-cluster':
LOGGER.error('cannot build jobs with this launcher type:' +
self.launcher_type)
return
LOGGER.info('-------------- Build --------------\n')
LOGGER.info('launcher_type = ' + self.launcher_type)
LOGGER.info('mod delta=' + str(mod_delta))
flagfile = os.path.join(
os.path.join(DAX_SETTINGS.get_results_dir(), 'FlagFiles'),
lockfile_prefix + '_' + BUILD_SUFFIX)
project_list = self.init_script(flagfile,
project_local,
type_update=1,
start_end=1)
try:
LOGGER.info('Connecting to XNAT at ' + self.xnat_host)
xnat = XnatUtils.get_interface(self.xnat_host, self.xnat_user,
self.xnat_pass)
if not XnatUtils.has_dax_datatypes(xnat):
raise Exception(
'error: dax datatypes are not installed on your xnat <%s>'
% (self.xnat_host))
#Priority if set:
if self.priority_project and not project_local:
unique_list = set(self.project_process_dict.keys() +
self.project_modules_dict.keys())
project_list = self.get_project_list(list(unique_list))
# Build projects
for project_id in project_list:
LOGGER.info('===== PROJECT:' + project_id + ' =====')
try:
self.build_project(xnat,
project_id,
lockfile_prefix,
sessions_local,
mod_delta=mod_delta)
except Exception as E:
LOGGER.critical('Caught exception building project %s' %
project_id)
LOGGER.critical(
'Exception class %s caught with message %s' %
(E.__class__, E.message))
finally:
self.finish_script(xnat, flagfile, project_list, 1, 2,
project_local)
def update_tasks(self, lockfile_prefix, project_local, sessions_local):
"""
Main method to Update the tasks
:param lockfile_prefix: prefix for flag file to lock the launcher
:param project_local: project to run locally
:param sessions_local: list of sessions to update tasks associated
to the project locally
:return: None
"""
if self.launcher_type == 'diskq-xnat':
LOGGER.error('cannot update jobs with this launcher type:' +
self.launcher_type)
return
LOGGER.info('-------------- Update Tasks --------------\n')
LOGGER.info('launcher_type = ' + self.launcher_type)
xnat = None
flagfile = os.path.join(
os.path.join(DAX_SETTINGS.get_results_dir(), 'FlagFiles'),
lockfile_prefix + '_' + UPDATE_SUFFIX)
project_list = self.init_script(flagfile,
project_local,
type_update=2,
start_end=1)
try:
if self.launcher_type in ['diskq-cluster', 'diskq-combined']:
LOGGER.info(
'Loading task queue from:' +
os.path.join(DAX_SETTINGS.get_results_dir(), 'DISKQ'))
task_list = load_task_queue()
LOGGER.info(str(len(task_list)) + ' tasks found.')
LOGGER.info('Updating tasks...')
for cur_task in task_list:
LOGGER.info('Updating task:' + cur_task.assessor_label)
cur_task.update_status()
else:
LOGGER.info('Connecting to XNAT at ' + self.xnat_host)
xnat = XnatUtils.get_interface(self.xnat_host, self.xnat_user,
self.xnat_pass)
if not XnatUtils.has_dax_datatypes(xnat):
raise Exception(
'error: dax datatypes are not installed on your xnat <%s>'
% (self.xnat_host))
LOGGER.info('Getting task list...')
task_list = self.get_tasks(xnat, self.is_updatable_tasks,
project_list, sessions_local)
LOGGER.info(str(len(task_list)) + ' open tasks found')
LOGGER.info('Updating tasks...')
for cur_task in task_list:
LOGGER.info(' Updating task:' +
cur_task.assessor_label)
cur_task.update_status()
finally:
self.finish_script(xnat, flagfile, project_list, 2, 2,
project_local)
def launch_jobs(self,
lockfile_prefix,
project_local,
sessions_local,
writeonly=False,
pbsdir=None):
"""
Main Method to launch the tasks
:param lockfile_prefix: prefix for flag file to lock the launcher
:param project_local: project to run locally
:param sessions_local: list of sessions to launch tasks
associated to the project locally
:param writeonly: write the job files without submitting them
:param pbsdir: folder to store the pbs file
:return: None
"""
if self.launcher_type == 'diskq-xnat':
LOGGER.error('cannot launch jobs with this launcher type:' +
self.launcher_type)
return
LOGGER.info('-------------- Launch Tasks --------------\n')
LOGGER.info('launcher_type = ' + self.launcher_type)
xnat = None
flagfile = os.path.join(
os.path.join(DAX_SETTINGS.get_results_dir(), 'FlagFiles'),
lockfile_prefix + '_' + LAUNCH_SUFFIX)
project_list = self.init_script(flagfile,
project_local,
type_update=3,
start_end=1)
try:
if self.launcher_type in ['diskq-cluster', 'diskq-combined']:
LOGGER.info(
'Loading task queue from:' +
os.path.join(DAX_SETTINGS.get_results_dir(), 'DISKQ'))
task_list = load_task_queue(status=task.NEED_TO_RUN)
LOGGER.info(
str(len(task_list)) +
' tasks that need to be launched found')
self.launch_tasks(task_list)
else:
LOGGER.info('Connecting to XNAT at ' + self.xnat_host)
xnat = XnatUtils.get_interface(self.xnat_host, self.xnat_user,
self.xnat_pass)
if not XnatUtils.has_dax_datatypes(xnat):
raise Exception(
'error: dax datatypes are not installed on your xnat <%s>'
% (self.xnat_host))
LOGGER.info('Getting launchable tasks list...')
task_list = self.get_tasks(xnat, self.is_launchable_tasks,
project_list, sessions_local)
LOGGER.info(
str(len(task_list)) +
' tasks that need to be launched found')
# Launch the task that need to be launch
self.launch_tasks(task_list, writeonly, pbsdir)
finally:
self.finish_script(xnat, flagfile, project_list, 3, 2,
project_local)
