def get_available_new_run_ids(conf):
"""Get the list of the available runs.
Arguments:
conf: configuration dictionary
"""
result = set()
hiseq_run_ids_do_not_process = hiseq_run.load_deny_run_ids(conf)
for hiseq_data_path in hiseq_run.get_hiseq_data_paths(conf):
files = os.listdir(hiseq_data_path)
for f in files:
# Do not process denied runs
if f in hiseq_run_ids_do_not_process:
continue
if not (os.path.isdir(hiseq_data_path + '/' + f) and hiseq_run.check_run_id(f, conf)):
# No valid entry
continue
# NextSeq sequencer create this file after clusterisation step
if not os.path.exists(hiseq_data_path + '/' + f + '/RunInfo.xml'):
continue
if common.get_rta_major_version(f, conf) == 1 and \
not os.path.exists(hiseq_data_path + '/' + common.FIRST_BASE_REPORT_FILE):
continue
if not hiseq_run.check_end_run(f, conf):
# TODO
# os.path.exists(hiseq_data_path + '/' + f + '/First_Base_Report.htm'):
result.add(f)
return result
def launch_steps(conf):
"""Launch steps.
Arguments:
conf: configuration object
Returns:
a boolean. True if launch_steps() is a success.
"""
# Load run do not process
hiseq_run_ids_do_not_process = hiseq_run.load_deny_run_ids(conf)
# Discover new runs
try:
detection_new_run.discover_new_runs(hiseq_run_ids_do_not_process, conf)
except:
exception_error(None , 'Failed to discover new runs', conf)
# Discover finished runs
try:
hiseq_run_ids_done = detection_end_run.discover_finished_runs(hiseq_run_ids_do_not_process, conf)
except:
exception_error(None , 'Failed to discover finished runs', conf)
#
# Sync hiseq and storage
#
sync_run_ids_done = sync_run.load_processed_run_ids(conf)
# Load run with higher priority
prioritized_run_ids = common.load_prioritized_run_ids(conf)
# Get the list of run available on HiSeq output
if sync_run.is_sync_step_enable(conf):
try:
sync_denied_run_ids = sync_run.load_denied_run_ids(conf)
for run_id in run_id_sorted_by_priority(
hiseq_run_ids_done - sync_run_ids_done - hiseq_run_ids_do_not_process - sync_denied_run_ids,
prioritized_run_ids):
# print 'DEBUG sync launch on '+ str(run_id)
if lock_sync_step(conf, run_id):
welcome(conf)
common.log('INFO', 'Synchronizing ' + run_id, conf)
if sync_run.sync(run_id, conf):
sync_run.add_run_id_to_processed_run_ids(run_id, conf)
sync_run_ids_done.add(run_id)
unlock_sync_step(conf, run_id)
else:
unlock_sync_step(conf, run_id)
return False
else:
common.log('INFO', 'Synchronizing ' + run_id + ' is locked.', conf)
except:
exception_error('sync' , 'Failed synchronization for run ' + run_id, conf)
#
# Demultiplexing
#
if common.is_conf_value_equals_true(DEMUX_USE_HISEQ_OUTPUT_KEY, conf):
sync_run_ids_done = hiseq_run_ids_done
demux_run_ids_done = demux_run.load_processed_run_ids(conf)
# print 'DEBUG launchStep demux done '+ str(demux_run_ids_done)
# print 'DEBUG runs to demux '+ str(sync_run_ids_done - demux_run_ids_done)
if common.is_conf_value_equals_true(DEMUX_STEP_KEY, conf):
try:
demux_denied_run_ids = demux_run.load_denied_run_ids(conf)
for run_id in run_id_sorted_by_priority(sync_run_ids_done - demux_run_ids_done - demux_denied_run_ids,
prioritized_run_ids):
# print 'DEBUG demux launch on ' + str(run_id)
if lock_demux_step(conf, run_id):
welcome(conf)
common.log('INFO', 'Demultiplexing ' + run_id, conf)
if demux_run.demux(run_id, conf):
demux_run.add_run_id_to_processed_run_ids(run_id, conf)
demux_run_ids_done.add(run_id)
unlock_demux_step(conf, run_id)
else:
unlock_demux_step(conf, run_id)
return False
else:
common.log('INFO', 'Demultiplexing ' + run_id + ' is locked.', conf)
except:
exception_error('demux', 'Failed demultiplexing for run ' + run_id, conf)
#
# Recompression
#
recompress_run_ids_done = recompress_run.load_processed_run_ids(conf)
if common.is_conf_value_equals_true(RECOMPRESS_STEP_KEY, conf):
try:
recompress_denied_run_ids = recompress_run.load_denied_run_ids(conf)
for run_id in run_id_sorted_by_priority(demux_run_ids_done - recompress_run_ids_done - recompress_denied_run_ids,
prioritized_run_ids):
if lock_recompress_step(conf, run_id):
welcome(conf)
common.log('INFO', 'Recompressing ' + run_id, conf)
if recompress_run.recompress(run_id, conf):
recompress_run.add_run_id_to_processed_run_ids(run_id, conf)
recompress_run_ids_done.add(run_id)
unlock_recompress_step(conf, run_id)
else:
unlock_recompress_step(conf, run_id)
return False
else:
common.log('INFO', 'Recompressing ' + run_id + ' is locked.', conf)
except:
exception_msg = str(sys.exc_info()[0]) + ' (' + str(sys.exc_info()[1]) + ')'
traceback_msg = traceback.format_exc(sys.exc_info()[2])
recompress_run.error("Failed recompression for run " + run_id + ", catch exception " + exception_msg,
"Failed recompression for run " + run_id + ", catch exception " + exception_msg + "\n Stacktrace : \n" + traceback_msg,
conf)
#
# Quality control
#
if not common.is_conf_value_equals_true(RECOMPRESS_STEP_KEY, conf):
recompress_run_ids_done = demux_run_ids_done
qc_run_ids_done = qc_run.load_processed_run_ids(conf)
if common.is_conf_value_equals_true(QC_STEP_KEY, conf):
try:
qc_denied_run_ids = qc_run.load_denied_run_ids(conf)
for run_id in run_id_sorted_by_priority(recompress_run_ids_done - qc_run_ids_done - qc_denied_run_ids,
prioritized_run_ids):
if lock_qc_step(conf, run_id):
welcome(conf)
common.log('INFO', 'Quality control ' + run_id, conf)
if qc_run.qc(run_id, conf):
qc_run.add_run_id_to_processed_run_ids(run_id, conf)
qc_run_ids_done.add(run_id)
unlock_qc_step(conf, run_id)
else:
unlock_qc_step(conf, run_id)
return False
else:
common.log('INFO', 'Quality control ' + run_id + ' is locked.', conf)
except:
exception_error('qc', 'Failed quality control for run ' + run_id, conf)
#
# Partial synchronization
#
working_run_ids = hiseq_run.get_working_run_ids(conf)
if common.is_conf_value_equals_true(SYNC_CONTINUOUS_SYNC_KEY, conf):
for run_id in (working_run_ids - sync_run_ids_done - hiseq_run_ids_do_not_process):
if lock_partial_sync_step(conf, run_id):
welcome(conf)
common.log('INFO', 'Partial synchronizing ' + run_id, conf)
if not sync_run.partial_sync(run_id, False, conf):
unlock_partial_sync_step(conf, run_id)
return False
unlock_partial_sync_step(conf, run_id)
else:
common.log('INFO', 'Partial synchronizing ' + run_id + ' is locked.', conf)
# Close Docker connections
DockerConnection.getInstance(conf[DOCKER_URI_KEY]).closeConnections()
# Everything is OK
return True
