def discover_finished_runs(denied_runs, conf): """Discover new ended runs Arguments: conf: configuration object """ run_ids_done = load_processed_run_ids(conf) if common.is_conf_value_equals_true(HISEQ_STEP_KEY, conf): for run_id in (get_available_finished_run_ids(conf) - run_ids_done - denied_runs): if run_id is None or len(run_id) == 0: # No run id found return [] aozan.welcome(conf) common.log('INFO', 'Ending run detection ' + str(run_id) + ' on ' + common.get_instrument_name(run_id, conf), conf) if hiseq_run.get_read_count(run_id, conf) == 0: send_failed_run_message(run_id, common.MAX_DELAY_TO_SEND_TERMINATED_RUN_EMAIL, conf) add_run_id_to_denied_run_ids(run_id, conf) create_run_summary_reports(run_id, conf) else: if create_run_summary_reports(run_id, conf): send_mail_if_recent_run(run_id, common.MAX_DELAY_TO_SEND_TERMINATED_RUN_EMAIL, conf) add_run_id_to_processed_run_ids(run_id, conf) run_ids_done.add(run_id) return run_ids_done
def discover_new_runs(denied_runs, conf): """Discover new runs. Arguments: conf: configuration object """ # # Discover new run # run_already_discovered = load_processed_run_ids(conf) if common.is_conf_value_equals_true(FIRST_BASE_REPORT_STEP_KEY, conf): for run_id in (get_available_new_run_ids(conf) - run_already_discovered - denied_runs): aozan.welcome(conf) common.log('INFO', 'First base report ' + run_id + ' on sequencer ' + common.get_instrument_name(run_id, conf), conf) if send_report(run_id, conf): add_run_id_to_processed_run_ids(run_id, conf) run_already_discovered.add(run_id) # Verify space needed during the first base report estimate_space_needed.estimate(run_id, conf)
def create_bcl2fastq_command_line(run_id, command_path, input_run_data_path, fastq_output_dir, samplesheet_csv_path, tmp_path, nb_mismatch, conf): nb_threads = str(get_cpu_count(conf)) if command_path is None: final_command_path = 'bcl2fastq' else: final_command_path = command_path # List arg args = [] args.append(final_command_path) args.extend(['--loading-threads', nb_threads]) args.extend(['--demultiplexing-threads', nb_threads]) args.extend(['--processing-threads', nb_threads]) args.extend(['--writing-threads', nb_threads]) args.extend(['--sample-sheet', samplesheet_csv_path]) args.extend(['--barcode-mismatches', nb_mismatch]) # Common parameter, setting per default args.extend(['--input-dir', quote(input_run_data_path + '/Data/Intensities/BaseCalls')]) args.extend(['--output-dir', quote(fastq_output_dir)]) if common.is_conf_value_equals_true(BCL2FASTQ_WITH_FAILED_READS_KEY, conf): args.append('--with-failed-reads') # Specific parameter args.extend(['--runfolder-dir', quote(input_run_data_path)]) args.extend(['--interop-dir', quote(fastq_output_dir + '/InterOp')]) args.extend(['--min-log-level', 'TRACE']) # args.extend(['--stats-dir', fastq_output_dir + '/Stats']) # args.extend(['--reports-dir', fastq_output_dir + '/Reports']) # Set the compression level if common.is_conf_key_exists(BCL2FASTQ_COMPRESSION_LEVEL_KEY, conf): level_str = conf[BCL2FASTQ_COMPRESSION_LEVEL_KEY].strip() try: level_int = int(level_str) if level_int > 0 and level_int < 10: args.extend(['--fastq-compression-level', str(level_int)]) except ValueError: pass if common.is_conf_key_exists(BCL2FASTQ_ADDITIONNAL_ARGUMENTS_KEY, conf): additional_args = conf[BCL2FASTQ_ADDITIONNAL_ARGUMENTS_KEY] additional_args = re.sub('\\s+', ' ', additional_args).strip() args.extends(additional_args.split(' ')) # Retrieve output in file args.extend(['>', quote(tmp_path + '/bcl2fastq_output_' + run_id + '.out')]) args.extend(['2>', quote(tmp_path + '/bcl2fastq_output_' + run_id + '.err')]) return " ".join(args)
def get_exclude_files_list(run_id, conf): """ Build list of excluded extension file from sequencer type. Arguments: run_id: the run id conf: configuration dictionary Return: excluded extension files list """ # Check if excluded file is required if not common.is_conf_value_equals_true(SYNC_EXCLUDE_CIF_KEY, conf): return [] # TODO exclude *bcl.bgzf.bci with RTA 2 ? return ['*.cif', '*_pos.txt', '*.errorMap', '*.FWHMMap']
def is_sync_step_enable(conf): """Check if all parameters useful for synchronization step are defined Arguments: conf: configuration dictionary """ # Synchronization step: True if common.is_conf_value_equals_true(SYNC_STEP_KEY, conf): # Check bcl path must be defined if common.is_conf_key_exists(BCL_DATA_PATH_KEY, conf): bcl_path = conf[BCL_DATA_PATH_KEY] # Check bcl not same hiseq output path for path in hiseq_run.get_hiseq_data_paths(conf): if path == bcl_path: error('Configuration error.', 'Basecalling path and hiseq output data path must be different: ' + bcl_path, conf) return False return True return False
def demux(run_id, conf): """Add a processed run id to the list of the run ids. Arguments: run_id: The run id conf: configuration dictionary """ start_time = time.time() common.log('INFO', 'Demux step: Starting', conf) reports_data_base_path = conf[REPORTS_DATA_PATH_KEY] reports_data_path = common.get_report_run_data_path(run_id, conf) samplesheet_filename = build_samplesheet_filename(run_id, conf) bcl2fastq_samplesheet_path = conf[TMP_PATH_KEY] + '/' + samplesheet_filename + '.csv' input_run_data_path = common.get_input_run_data_path(run_id, conf) if input_run_data_path is None: return False fastq_output_dir = conf[FASTQ_DATA_PATH_KEY] + '/' + run_id basecall_stats_prefix = 'basecall_stats_' basecall_stats_file = basecall_stats_prefix + run_id + '.tar.bz2' # Check if root input bcl data directory exists if not os.path.exists(input_run_data_path): error("Basecalling data directory does not exist", "Basecalling data directory does not exist: " + str(input_run_data_path), conf) # return False # Check if root input fastq data directory exists if not common.is_dir_exists(FASTQ_DATA_PATH_KEY, conf): error("FASTQ data directory does not exist", "FASTQ data directory does not exist: " + conf[FASTQ_DATA_PATH_KEY], conf) return False # Check if bcl2fastq samplesheets path exists if not common.is_dir_exists(BCL2FASTQ_SAMPLESHEETS_PATH_KEY, conf): error("Bcl2fastq samplesheet directory does not exist", "Bcl2fastq samplesheet directory does not exist: " + conf[BCL2FASTQ_SAMPLESHEETS_PATH_KEY], conf) return False # Check if bcl2fastq basedir path exists if not common.is_conf_value_equals_true(BCL2FASTQ_USE_DOCKER_KEY, conf): if not common.is_dir_exists(BCL2FASTQ_PATH_KEY, conf): error("Bcl2fastq directory does not exist", "Bcl2fastq directory does not exist: " + conf[BCL2FASTQ_PATH_KEY], conf) return False # Check if temporary directory exists if not common.is_dir_exists(TMP_PATH_KEY, conf): error("Temporary directory does not exist", "Temporary directory does not exist: " + conf[TMP_PATH_KEY], conf) return False # Check if reports_data_path exists if not os.path.exists(reports_data_base_path): error("Report directory does not exist", "Report directory does not exist: " + reports_data_base_path, conf) return False # Create if not exist report directory for the run if not os.path.exists(reports_data_path): os.mkdir(reports_data_path) # Check if basecall stats archive exists if os.path.exists(reports_data_path + '/' + basecall_stats_file): error('Basecall stats archive already exists for run ' + run_id, 'Basecall stats archive already exists for run ' + run_id + ': ' + basecall_stats_file, conf) return False # Check if the output directory already exists if os.path.exists(fastq_output_dir): error("FASTQ output directory already exists for run " + run_id, 'FASTQ output directory already exists for run ' + run_id + ': ' + fastq_output_dir, conf) return False # Compute disk usage and disk free to check if enough disk space is available input_path_du = common.du(input_run_data_path) output_df = common.df(conf[FASTQ_DATA_PATH_KEY]) du_factor = float(conf[DEMUX_SPACE_FACTOR_KEY]) space_needed = input_path_du * du_factor common.log("WARNING", "Demux step: input disk usage: " + str(input_path_du), conf) common.log("WARNING", "Demux step: output disk free: " + str(output_df), conf) common.log("WARNING", "Demux step: space needed: " + str(space_needed), conf) common.log("CONFIG", "Bcl2fastq Docker mode: " + str( common.is_conf_value_equals_true(Settings.BCL2FASTQ_USE_DOCKER_KEY, conf)), conf) # Check if free space is available if output_df < space_needed: error("Not enough disk space to perform demultiplexing for run " + run_id, "Not enough disk space to perform demultiplexing for run " + run_id + '.\n%.2f Gb' % (space_needed / 1024 / 1024 / 1024) + ' is needed (factor x' + str( du_factor) + ') on ' + fastq_output_dir + '.', conf) return False # Load RunInfo object run_info = RunInfo.parse(input_run_data_path + '/RunInfo.xml') # Load samplesheet samplesheet, original_samplesheet_path = load_samplesheet(run_id, input_run_data_path, samplesheet_filename, conf) if samplesheet is None: return False # Update samplesheet if not update_samplesheet(samplesheet, run_id, run_info.getFlowCellLaneCount(), conf): return False # Check samplesheet check_result, samplesheet_warnings = check_samplesheet(samplesheet, run_id, run_info.getFlowCell(), conf) if not check_result: return False # Get the number of mismatches nb_mismatch = get_bcl2fastq_mismatches(samplesheet, conf[BCL2FASTQ_MISMATCHES_KEY]) # Write final samplesheet if not write_bcl2fastq_samplesheet(samplesheet, bcl2fastq_samplesheet_path, conf): return False # Run demultiplexing if common.is_conf_value_equals_true(Settings.BCL2FASTQ_USE_DOCKER_KEY, conf): # With image docker if not demux_run_with_docker(run_id, input_run_data_path, fastq_output_dir, bcl2fastq_samplesheet_path, nb_mismatch, conf): return False else: if not demux_run_standalone(run_id, input_run_data_path, fastq_output_dir, bcl2fastq_samplesheet_path, nb_mismatch, conf): return False # Check if the output directory has been created if not os.path.exists(fastq_output_dir): error("Error while demultiplexing run " + run_id + ' on ' + common.get_instrument_name(run_id, conf), 'Error while demultiplexing run ' + run_id + '.\n' + 'The output directory of bcl2fastq has been created: ' + fastq_output_dir, conf) return False # Check if the output directory has been created if os.path.isfile(fastq_output_dir): error("Error while demultiplexing run " + run_id + ' on ' + common.get_instrument_name(run_id, conf), 'Error while demultiplexing run ' + run_id + '.\n' + 'The output directory of bcl2fastq is a file instead of a directory: ' + fastq_output_dir, conf) return False # Copy bcl2fastq log to output directory cmd = 'cp ' + quote(conf[TMP_PATH_KEY]) + '/bcl2fastq_output_' + run_id + '.* ' + quote(fastq_output_dir) common.log("INFO", "exec: " + cmd, conf) if os.system(cmd) != 0: error("Error while copying bcl2fastq log to the output FASTQ directory" + run_id_msg, 'Error while copying bcl2fastq log to the output FASTQ directory.\nCommand line:\n' + cmd, conf) return False # The output directory must be read only if not common.chmod_files_in_dir(fastq_output_dir, ".fastq", conf): error("Error while setting the output FASTQ directory to read only" + run_id_msg, 'Error while setting the output FASTQ directory to read only.\nCommand line:\n' + cmd, conf) return False if not check_if_output_fastq_files_exists(fastq_output_dir): error("Error with bcl2fastq execution for run " + run_id, "Error with bcl2fastq execution for run " + run_id + " no FASTQ file found in " + fastq_output_dir, conf) return False # Copy samplesheet to output directory cmd = 'cp -p ' + quote(bcl2fastq_samplesheet_path) + ' ' + quote(fastq_output_dir + '/SampleSheet.csv') common.log("INFO", "exec: " + cmd, conf) if os.system(cmd) != 0: error("Error while copying samplesheet file to FASTQ directory for run " + run_id, 'Error while copying samplesheet file to FASTQ directory.\nCommand line:\n' + cmd, conf) return False # Create archives on demultiplexing statistics if not archive_demux_stat(run_id, fastq_output_dir, reports_data_path, basecall_stats_file, basecall_stats_prefix, bcl2fastq_samplesheet_path, conf): return False # Archive samplesheet if not archive_samplesheet(run_id, original_samplesheet_path, bcl2fastq_samplesheet_path, conf): return False # Remove temporary samplesheet files if os.path.exists(bcl2fastq_samplesheet_path): os.remove(bcl2fastq_samplesheet_path) # Create index.hml file common.create_html_index_file(conf, run_id, [Settings.HISEQ_STEP_KEY, Settings.DEMUX_STEP_KEY]) df_in_bytes = common.df(fastq_output_dir) du_in_bytes = common.du(fastq_output_dir) df = df_in_bytes / (1024 * 1024 * 1024) du = du_in_bytes / (1024 * 1024 * 1024) common.log("WARNING", "Demux step: output disk free after demux: " + str(df_in_bytes), conf) common.log("WARNING", "Demux step: space used by demux: " + str(du_in_bytes), conf) duration = time.time() - start_time msg = 'Ending demultiplexing with ' + nb_mismatch + ' mismatch(es) for run ' + run_id + '.' + \ '\nJob finished at ' + common.time_to_human_readable(time.time()) + \ ' without error in ' + common.duration_to_human_readable(duration) + '.\n\n' + \ 'FASTQ files for this run ' + \ 'can be found in the following directory:\n ' + fastq_output_dir if samplesheet_warnings.size() > 0: msg += '\n\nSamplesheet warnings:' for warn in samplesheet_warnings: msg += "\n - " + warn # Add path to report if reports.url exists if common.is_conf_key_exists(REPORTS_URL_KEY, conf): msg += '\n\nRun reports can be found at following location:\n ' + conf[REPORTS_URL_KEY] + '/' + run_id msg += '\n\nFor this task %.2f GB has been used and %.2f GB still free.' % (du, df) common.send_msg('[Aozan] Ending demultiplexing for run ' + run_id + ' on ' + common.get_instrument_name(run_id, conf), msg, False, conf) common.log('INFO', 'Demux step: successful in ' + common.duration_to_human_readable(duration), conf) return True
def launch_steps(conf): """Launch steps. Arguments: conf: configuration object Returns: a boolean. True if launch_steps() is a success. """ # Load run do not process hiseq_run_ids_do_not_process = hiseq_run.load_deny_run_ids(conf) # Discover new runs try: detection_new_run.discover_new_runs(hiseq_run_ids_do_not_process, conf) except: exception_error(None , 'Failed to discover new runs', conf) # Discover finished runs try: hiseq_run_ids_done = detection_end_run.discover_finished_runs(hiseq_run_ids_do_not_process, conf) except: exception_error(None , 'Failed to discover finished runs', conf) # # Sync hiseq and storage # sync_run_ids_done = sync_run.load_processed_run_ids(conf) # Load run with higher priority prioritized_run_ids = common.load_prioritized_run_ids(conf) # Get the list of run available on HiSeq output if sync_run.is_sync_step_enable(conf): try: sync_denied_run_ids = sync_run.load_denied_run_ids(conf) for run_id in run_id_sorted_by_priority( hiseq_run_ids_done - sync_run_ids_done - hiseq_run_ids_do_not_process - sync_denied_run_ids, prioritized_run_ids): # print 'DEBUG sync launch on '+ str(run_id) if lock_sync_step(conf, run_id): welcome(conf) common.log('INFO', 'Synchronizing ' + run_id, conf) if sync_run.sync(run_id, conf): sync_run.add_run_id_to_processed_run_ids(run_id, conf) sync_run_ids_done.add(run_id) unlock_sync_step(conf, run_id) else: unlock_sync_step(conf, run_id) return False else: common.log('INFO', 'Synchronizing ' + run_id + ' is locked.', conf) except: exception_error('sync' , 'Failed synchronization for run ' + run_id, conf) # # Demultiplexing # if common.is_conf_value_equals_true(DEMUX_USE_HISEQ_OUTPUT_KEY, conf): sync_run_ids_done = hiseq_run_ids_done demux_run_ids_done = demux_run.load_processed_run_ids(conf) # print 'DEBUG launchStep demux done '+ str(demux_run_ids_done) # print 'DEBUG runs to demux '+ str(sync_run_ids_done - demux_run_ids_done) if common.is_conf_value_equals_true(DEMUX_STEP_KEY, conf): try: demux_denied_run_ids = demux_run.load_denied_run_ids(conf) for run_id in run_id_sorted_by_priority(sync_run_ids_done - demux_run_ids_done - demux_denied_run_ids, prioritized_run_ids): # print 'DEBUG demux launch on ' + str(run_id) if lock_demux_step(conf, run_id): welcome(conf) common.log('INFO', 'Demultiplexing ' + run_id, conf) if demux_run.demux(run_id, conf): demux_run.add_run_id_to_processed_run_ids(run_id, conf) demux_run_ids_done.add(run_id) unlock_demux_step(conf, run_id) else: unlock_demux_step(conf, run_id) return False else: common.log('INFO', 'Demultiplexing ' + run_id + ' is locked.', conf) except: exception_error('demux', 'Failed demultiplexing for run ' + run_id, conf) # # Recompression # recompress_run_ids_done = recompress_run.load_processed_run_ids(conf) if common.is_conf_value_equals_true(RECOMPRESS_STEP_KEY, conf): try: recompress_denied_run_ids = recompress_run.load_denied_run_ids(conf) for run_id in run_id_sorted_by_priority(demux_run_ids_done - recompress_run_ids_done - recompress_denied_run_ids, prioritized_run_ids): if lock_recompress_step(conf, run_id): welcome(conf) common.log('INFO', 'Recompressing ' + run_id, conf) if recompress_run.recompress(run_id, conf): recompress_run.add_run_id_to_processed_run_ids(run_id, conf) recompress_run_ids_done.add(run_id) unlock_recompress_step(conf, run_id) else: unlock_recompress_step(conf, run_id) return False else: common.log('INFO', 'Recompressing ' + run_id + ' is locked.', conf) except: exception_msg = str(sys.exc_info()[0]) + ' (' + str(sys.exc_info()[1]) + ')' traceback_msg = traceback.format_exc(sys.exc_info()[2]) recompress_run.error("Failed recompression for run " + run_id + ", catch exception " + exception_msg, "Failed recompression for run " + run_id + ", catch exception " + exception_msg + "\n Stacktrace : \n" + traceback_msg, conf) # # Quality control # if not common.is_conf_value_equals_true(RECOMPRESS_STEP_KEY, conf): recompress_run_ids_done = demux_run_ids_done qc_run_ids_done = qc_run.load_processed_run_ids(conf) if common.is_conf_value_equals_true(QC_STEP_KEY, conf): try: qc_denied_run_ids = qc_run.load_denied_run_ids(conf) for run_id in run_id_sorted_by_priority(recompress_run_ids_done - qc_run_ids_done - qc_denied_run_ids, prioritized_run_ids): if lock_qc_step(conf, run_id): welcome(conf) common.log('INFO', 'Quality control ' + run_id, conf) if qc_run.qc(run_id, conf): qc_run.add_run_id_to_processed_run_ids(run_id, conf) qc_run_ids_done.add(run_id) unlock_qc_step(conf, run_id) else: unlock_qc_step(conf, run_id) return False else: common.log('INFO', 'Quality control ' + run_id + ' is locked.', conf) except: exception_error('qc', 'Failed quality control for run ' + run_id, conf) # # Partial synchronization # working_run_ids = hiseq_run.get_working_run_ids(conf) if common.is_conf_value_equals_true(SYNC_CONTINUOUS_SYNC_KEY, conf): for run_id in (working_run_ids - sync_run_ids_done - hiseq_run_ids_do_not_process): if lock_partial_sync_step(conf, run_id): welcome(conf) common.log('INFO', 'Partial synchronizing ' + run_id, conf) if not sync_run.partial_sync(run_id, False, conf): unlock_partial_sync_step(conf, run_id) return False unlock_partial_sync_step(conf, run_id) else: common.log('INFO', 'Partial synchronizing ' + run_id + ' is locked.', conf) # Close Docker connections DockerConnection.getInstance(conf[DOCKER_URI_KEY]).closeConnections() # Everything is OK return True
return False except Throwable, exp: error("Error while computing QC report for run " + run_id + ".", common.exception_msg(exp, conf), conf) return False # Remove QC data if not demand if common.is_conf_value_defined(QC_REPORT_SAVE_RAW_DATA_KEY, 'false', conf): try: os.remove(qc_output_dir + '/data-' + run_id + '.txt') # qc.writeRawData(report, qc_output_dir + '/data-' + run_id + '.txt') except AozanException, exp: error("Error while removing QC raw data for run " + run_id + ".", exp.getMessage(), conf) return False # Write the XML report if common.is_conf_value_equals_true(QC_REPORT_SAVE_REPORT_DATA_KEY, conf): try: qc.writeXMLReport(report, qc_output_dir + '/' + run_id + '.xml') except AozanException, exp: error("Error while computing QC XML report for run " + run_id + ".", common.exception_msg(exp, conf), conf) return False except Throwable, exp: error("Error while computing QC XML report for run " + run_id + ".", common.exception_msg(exp, conf), conf) return False # Remove tmp extension of temporary QC directory os.rename(qc_output_dir, qc_output_dir[:-len(tmp_extension)]) qc_output_dir = qc_output_dir[:-len(tmp_extension)] # Write the HTML report html_report_file = qc_output_dir + '/' + run_id + '.html'
def recompress(run_id, conf): """Proceed to recompression of a run. Arguments: run_id: The run id conf: configuration dictionary """ common.log('INFO', 'Recompress step: Starting', conf) # Check if input root fastq root data exists if not common.is_dir_exists(FASTQ_DATA_PATH_KEY, conf): error("FASTQ data directory does not exist", "FASTQ data directory does not exist: " + conf[FASTQ_DATA_PATH_KEY], conf) return False start_time = time.time() fastq_input_dir = conf[FASTQ_DATA_PATH_KEY] + '/' + run_id # initial du for comparing with ending disk usage previous_du_in_bytes = common.du(fastq_input_dir) # get information about compression type compression_type = conf[RECOMPRESS_COMPRESSION_KEY] compression_level = conf[RECOMPRESS_COMPRESSION_LEVEL_KEY] compression_info_tuple = get_info_from_file_type(compression_type, compression_level) if compression_info_tuple is None: error("Unknown compression type", "Unknown compression type: " + compression_type, conf) return False (compression_type_result, output_file_extension, output_compression_command, output_decompression_command, compression_level_argument) = compression_info_tuple # The following list contains the processed type of files to recompress types_to_recompress = ["fastq.gz", "fastq"] # list of program to check if exists in path before execution program_set = {"bash", "tee", "touch", "chmod", "md5sum", output_compression_command, output_decompression_command} # get list of file to process input_files = [] for extension in types_to_recompress: input_files.extend(list_files(fastq_input_dir, extension)) simple_extension = os.path.splitext(extension)[-1][1:] extension_info_tuple = get_info_from_file_type(simple_extension) if extension_info_tuple is None: error("Unknown extension type", "Unknown extension type: " + extension, conf) return False program_set.add(extension_info_tuple[3]) # actual program list check for program in program_set: if not common.exists_in_path(program): error("Can't find all needed commands in PATH env var", "Can't find all needed commands in PATH env var. Unable to find: " + program + " command.", conf) return False # Create executor and for parallelization of processus executor = Executors.newFixedThreadPool(int(conf[RECOMPRESS_THREADS_KEY])) workers = [] # process each fastq and fastq.gz recursively in each fastq directory for input_file in input_files: simple_extension = os.path.splitext(input_file)[-1][1:] # get info about the type of input file extension_info_tuple = get_info_from_file_type(simple_extension) if extension_info_tuple is None: error("Unknown extension type", "Unknown extension type: " + simple_extension, conf) return False input_decompression_command = extension_info_tuple[3] # get file base name and create output_file name, if file is already .fastq its ready to be base_input_file base_input_file = input_file[0: input_file.index(".fastq") + 6] output_file = base_input_file + "." + output_file_extension # Skip if the output_file already exists if not os.path.exists(output_file): # Create worker then execute in thread worker = Worker(input_file, output_file, input_decompression_command, output_compression_command, output_decompression_command, compression_level_argument, common.is_conf_value_equals_true(RECOMPRESS_DELETE_ORIGINAL_FASTQ_KEY, conf)) workers.append(worker) executor.execute(worker) else: common.log("WARNING", "Recompress step: Omitting processing file " + input_file + ". The associated output file " + output_file + " already exists.", conf) # Wait for all thread to finish executor.shutdown() while not executor.isTerminated(): time.sleep(1) # Check if any worker is in error for worker in workers: if not worker.is_successful(): error(worker.get_error_message(), worker.get_long_error_message(), conf) return False # check new disk usage df_in_bytes = common.df(fastq_input_dir) du_in_bytes = common.du(fastq_input_dir) previous_du = previous_du_in_bytes / (1024 * 1024) df = df_in_bytes / (1024 * 1024 * 1024) du = du_in_bytes / (1024 * 1024) common.log("WARNING", "Recompress step: output disk free after step: " + str(df_in_bytes), conf) common.log("WARNING", "Recompress step: space previously used: " + str(previous_du_in_bytes), conf) common.log("WARNING", "Recompress step: space now used by step: " + str(du_in_bytes), conf) duration = time.time() - start_time msg = 'Ending recompression for run ' + run_id + '.' + \ '\nJob finished at ' + common.time_to_human_readable(time.time()) + \ ' without error in ' + common.duration_to_human_readable(duration) + '. ' msg += '\n\nAfter recompress step FASTQ folder is now %.2f MB (previously %.2f MB) and %.2f GB still free.' % ( du, previous_du, df) common.send_msg('[Aozan] Ending recompress for run ' + run_id + ' on ' + common.get_instrument_name(run_id, conf), msg, False, conf) common.log('INFO', 'Recompress step: successful in ' + common.duration_to_human_readable(duration), conf) return True