def get_available_new_run_ids(conf): """Get the list of the available runs. Arguments: conf: configuration dictionary """ result = set() hiseq_run_ids_do_not_process = hiseq_run.load_deny_run_ids(conf) for hiseq_data_path in hiseq_run.get_hiseq_data_paths(conf): files = os.listdir(hiseq_data_path) for f in files: # Do not process denied runs if f in hiseq_run_ids_do_not_process: continue if not (os.path.isdir(hiseq_data_path + '/' + f) and hiseq_run.check_run_id(f, conf)): # No valid entry continue # NextSeq sequencer create this file after clusterisation step if not os.path.exists(hiseq_data_path + '/' + f + '/RunInfo.xml'): continue if common.get_rta_major_version(f, conf) == 1 and \ not os.path.exists(hiseq_data_path + '/' + common.FIRST_BASE_REPORT_FILE): continue if not hiseq_run.check_end_run(f, conf): # TODO # os.path.exists(hiseq_data_path + '/' + f + '/First_Base_Report.htm'): result.add(f) return result
def launch_steps(conf): """Launch steps. Arguments: conf: configuration object Returns: a boolean. True if launch_steps() is a success. """ # Load run do not process hiseq_run_ids_do_not_process = hiseq_run.load_deny_run_ids(conf) # Discover new runs try: detection_new_run.discover_new_runs(hiseq_run_ids_do_not_process, conf) except: exception_error(None , 'Failed to discover new runs', conf) # Discover finished runs try: hiseq_run_ids_done = detection_end_run.discover_finished_runs(hiseq_run_ids_do_not_process, conf) except: exception_error(None , 'Failed to discover finished runs', conf) # # Sync hiseq and storage # sync_run_ids_done = sync_run.load_processed_run_ids(conf) # Load run with higher priority prioritized_run_ids = common.load_prioritized_run_ids(conf) # Get the list of run available on HiSeq output if sync_run.is_sync_step_enable(conf): try: sync_denied_run_ids = sync_run.load_denied_run_ids(conf) for run_id in run_id_sorted_by_priority( hiseq_run_ids_done - sync_run_ids_done - hiseq_run_ids_do_not_process - sync_denied_run_ids, prioritized_run_ids): # print 'DEBUG sync launch on '+ str(run_id) if lock_sync_step(conf, run_id): welcome(conf) common.log('INFO', 'Synchronizing ' + run_id, conf) if sync_run.sync(run_id, conf): sync_run.add_run_id_to_processed_run_ids(run_id, conf) sync_run_ids_done.add(run_id) unlock_sync_step(conf, run_id) else: unlock_sync_step(conf, run_id) return False else: common.log('INFO', 'Synchronizing ' + run_id + ' is locked.', conf) except: exception_error('sync' , 'Failed synchronization for run ' + run_id, conf) # # Demultiplexing # if common.is_conf_value_equals_true(DEMUX_USE_HISEQ_OUTPUT_KEY, conf): sync_run_ids_done = hiseq_run_ids_done demux_run_ids_done = demux_run.load_processed_run_ids(conf) # print 'DEBUG launchStep demux done '+ str(demux_run_ids_done) # print 'DEBUG runs to demux '+ str(sync_run_ids_done - demux_run_ids_done) if common.is_conf_value_equals_true(DEMUX_STEP_KEY, conf): try: demux_denied_run_ids = demux_run.load_denied_run_ids(conf) for run_id in run_id_sorted_by_priority(sync_run_ids_done - demux_run_ids_done - demux_denied_run_ids, prioritized_run_ids): # print 'DEBUG demux launch on ' + str(run_id) if lock_demux_step(conf, run_id): welcome(conf) common.log('INFO', 'Demultiplexing ' + run_id, conf) if demux_run.demux(run_id, conf): demux_run.add_run_id_to_processed_run_ids(run_id, conf) demux_run_ids_done.add(run_id) unlock_demux_step(conf, run_id) else: unlock_demux_step(conf, run_id) return False else: common.log('INFO', 'Demultiplexing ' + run_id + ' is locked.', conf) except: exception_error('demux', 'Failed demultiplexing for run ' + run_id, conf) # # Recompression # recompress_run_ids_done = recompress_run.load_processed_run_ids(conf) if common.is_conf_value_equals_true(RECOMPRESS_STEP_KEY, conf): try: recompress_denied_run_ids = recompress_run.load_denied_run_ids(conf) for run_id in run_id_sorted_by_priority(demux_run_ids_done - recompress_run_ids_done - recompress_denied_run_ids, prioritized_run_ids): if lock_recompress_step(conf, run_id): welcome(conf) common.log('INFO', 'Recompressing ' + run_id, conf) if recompress_run.recompress(run_id, conf): recompress_run.add_run_id_to_processed_run_ids(run_id, conf) recompress_run_ids_done.add(run_id) unlock_recompress_step(conf, run_id) else: unlock_recompress_step(conf, run_id) return False else: common.log('INFO', 'Recompressing ' + run_id + ' is locked.', conf) except: exception_msg = str(sys.exc_info()[0]) + ' (' + str(sys.exc_info()[1]) + ')' traceback_msg = traceback.format_exc(sys.exc_info()[2]) recompress_run.error("Failed recompression for run " + run_id + ", catch exception " + exception_msg, "Failed recompression for run " + run_id + ", catch exception " + exception_msg + "\n Stacktrace : \n" + traceback_msg, conf) # # Quality control # if not common.is_conf_value_equals_true(RECOMPRESS_STEP_KEY, conf): recompress_run_ids_done = demux_run_ids_done qc_run_ids_done = qc_run.load_processed_run_ids(conf) if common.is_conf_value_equals_true(QC_STEP_KEY, conf): try: qc_denied_run_ids = qc_run.load_denied_run_ids(conf) for run_id in run_id_sorted_by_priority(recompress_run_ids_done - qc_run_ids_done - qc_denied_run_ids, prioritized_run_ids): if lock_qc_step(conf, run_id): welcome(conf) common.log('INFO', 'Quality control ' + run_id, conf) if qc_run.qc(run_id, conf): qc_run.add_run_id_to_processed_run_ids(run_id, conf) qc_run_ids_done.add(run_id) unlock_qc_step(conf, run_id) else: unlock_qc_step(conf, run_id) return False else: common.log('INFO', 'Quality control ' + run_id + ' is locked.', conf) except: exception_error('qc', 'Failed quality control for run ' + run_id, conf) # # Partial synchronization # working_run_ids = hiseq_run.get_working_run_ids(conf) if common.is_conf_value_equals_true(SYNC_CONTINUOUS_SYNC_KEY, conf): for run_id in (working_run_ids - sync_run_ids_done - hiseq_run_ids_do_not_process): if lock_partial_sync_step(conf, run_id): welcome(conf) common.log('INFO', 'Partial synchronizing ' + run_id, conf) if not sync_run.partial_sync(run_id, False, conf): unlock_partial_sync_step(conf, run_id) return False unlock_partial_sync_step(conf, run_id) else: common.log('INFO', 'Partial synchronizing ' + run_id + ' is locked.', conf) # Close Docker connections DockerConnection.getInstance(conf[DOCKER_URI_KEY]).closeConnections() # Everything is OK return True