def demux_run_standalone(run_id, input_run_data_path, fastq_output_dir, samplesheet_csv_path, nb_mismatch, conf): """ Demultiplexing the run with bcl2fastq on version parameter. Arguments: run_id: The run id input_run_data_path: input run data path to demultiplexing fastq_output_dir: fastq directory to save result on demultiplexing samplesheet_csv_path: samplesheet path in csv format, version used by bcl2fastq conf: configuration dictionary """ bcl2fastq_executable_path = conf[BCL2FASTQ_PATH_KEY] tmp = conf[TMP_PATH_KEY] run_id_msg = " for run " + run_id + ' on ' + common.get_instrument_name(run_id, conf) bcl2fastq_log_file = tmp + "/bcl2fastq_output_" + run_id + ".err" # Check if the bcl2fastq path is OK if os.path.isdir(bcl2fastq_executable_path): bcl2fastq_executable_path += '/bcl2fastq' elif not os.path.isfile(bcl2fastq_executable_path): error("Error while setting executable command file bcl2fastq" + run_id_msg + ", invalid bcl2fastq path: " + bcl2fastq_executable_path, "Error while setting executable command file bcl2fastq" + run_id_msg + ", invalid bcl2fastq path: " + bcl2fastq_executable_path, conf) return False cmd = create_bcl2fastq_command_line(run_id, bcl2fastq_executable_path, input_run_data_path, fastq_output_dir, samplesheet_csv_path, tmp, nb_mismatch, conf) common.log('INFO', 'Demultiplexing in standalone mode using the following command line: ' + str(cmd), conf) exit_code = os.system(cmd) if exit_code != 0: error("Error while executing bcl2fastq " + run_id_msg, 'Error while executing bcl2fastq (exit code: ' + str( exit_code) + ').\nCommand line:\n' + cmd, conf) msg = 'Error while executing bcl2fastq ' + run_id_msg + ' (exit code: ' + str( exit_code) + ').\nCommand line:\n' + cmd # Check if the log file has been generated if not os.path.exists(bcl2fastq_log_file): error("Error with bcl2fastq log for run " + run_id + ".", "No bcl2fastq log available", conf) common.send_msg('[Aozan] Failed demultiplexing ' + run_id_msg, msg, True, conf) else: msg += "\n\nPlease check the attached bcl2fastq output error file." common.send_msg_with_attachment('[Aozan] Failed demultiplexing ' + run_id_msg, msg, bcl2fastq_log_file, True, conf) return False return True
def send_report(run_id, conf): """Send a mail with the first base report. Arguments: run_id: the run id conf: configuration dictionary """ # # Retrieve features the current run in RunInfos.xml file # run_info = hiseq_run.get_run_info(run_id, conf) if run_info is None: return False # TODO ?? add check sample-sheet if demux step enable # add warning in report if useful reads = run_info.getReads() error_cycles_per_read_not_indexes_count = 0 reads_indexed_count = 0 reads_not_indexed_count = 0 cycles_count = 0 cycles_per_read_not_indexed = 0 for read in reads: cycles_count += read.getNumberCycles() if read.isIndexedRead(): reads_indexed_count += 1 else: reads_not_indexed_count += 1 if cycles_per_read_not_indexed == 0: cycles_per_read_not_indexed = read.getNumberCycles() # Check same cycles count for each reads not indexed error_cycles_per_read_not_indexes_count = cycles_per_read_not_indexed != read.getNumberCycles() # Identification type run according to data in RunInfos.xml : SR or PE if reads_not_indexed_count == 1: type_run_estimated = "SR-" + str(cycles_per_read_not_indexed) + " with " + str( reads_indexed_count) + " index" if reads_indexed_count > 1: type_run_estimated += "es" elif reads_not_indexed_count == 2: type_run_estimated = "PE-" + str(cycles_per_read_not_indexed) + " with " + str( reads_indexed_count) + " index" if reads_indexed_count > 1: type_run_estimated += "es" else: type_run_estimated = "Undetermined run type (" + str(reads_not_indexed_count) + " reads with " + str( reads_indexed_count) + " index)" if reads_indexed_count > 1: type_run_estimated += "es" type_run_estimated += ")" description_run = "Informations about this run:\n" description_run += "\t- Sequencer: " + common.get_instrument_name(run_id, conf) + ".\n" description_run += "\t- " + str(run_info.getFlowCellLaneCount()) + " lanes with " + str( run_info.alignToPhix.size()) + " aligned to Phix.\n" description_run += "\t- " + str(reads_not_indexed_count) + " read" if reads_not_indexed_count > 1: description_run += "s" description_run += " and " + str(reads_indexed_count) + " index" if reads_indexed_count > 1: description_run += "es" description_run += ".\n" if error_cycles_per_read_not_indexes_count or cycles_per_read_not_indexed == 0: description_run += "\t- ERROR : cycles count per read different between reads (" + str( cycles_count) + " total cycles).\n" else: description_run += "\t- " + str(cycles_per_read_not_indexed) + " cycles per read (" + str( cycles_count) + " total cycles).\n" description_run += "\t- Estimated run type: " + type_run_estimated + ".\n" attachment_file = str(hiseq_run.find_hiseq_run_path(run_id, conf)) + '/' + run_id + '/' + common.FIRST_BASE_REPORT_FILE # If the First base report file exists, send it by email if common.is_file_readable(attachment_file): message = 'You will find attached to this message the first base report for the run ' + \ run_id + '.\n\n' + description_run common.send_msg_with_attachment('[Aozan] First base report for the run ' + type_run_estimated + ' ' + run_id + ' on ' + common.get_instrument_name(run_id, conf), message, attachment_file, False, conf) else: # With other no attachment file message = 'You will find below the parameters of the run ' + run_id + '.\n\n' + description_run common.send_msg('[Aozan] New run ' + type_run_estimated + ' ' + run_id + ' on ' + common.get_instrument_name(run_id, conf), message, False, conf) return True
def demux_run_with_docker(run_id, input_run_data_path, fastq_output_dir, samplesheet_csv_path, nb_mismatch, conf): """ Demultiplexing the run with bcl2fastq on version parameter with image Docker. Arguments: run_id: The run id input_run_data_path: input run data path to demultiplexing fastq_output_dir: fastq directory to save result on demultiplexing samplesheet_csv_path: samplesheet path in csv format, version used by bcl2fastq conf: configuration dictionary """ # In docker mount with input_run_data_path input_docker = '/data/input' input_run_data_path_in_docker = input_docker run_id_msg = " for run " + run_id + ' on ' + common.get_instrument_name(run_id, conf) # In docker mount with fastq_output_dir output_docker = '/data/output' fastq_data_path_in_docker = output_docker + '/' + os.path.basename(fastq_output_dir) tmp = conf[TMP_PATH_KEY] tmp_docker = '/tmp' bcl2fastq_log_file = tmp + "/bcl2fastq_output_" + run_id + ".err" samplesheet_csv_docker = tmp_docker + '/' + os.path.basename(samplesheet_csv_path) cmd = create_bcl2fastq_command_line(run_id, None, input_run_data_path_in_docker, fastq_data_path_in_docker, samplesheet_csv_docker, tmp_docker, nb_mismatch, conf) try: # Set working in docker on parent demultiplexing run directory. # Demultiplexing run directory will create by bcl2fastq docker = DockerCommand(conf[Settings.DOCKER_URI_KEY], ['/bin/bash', '-c', cmd], 'bcl2fastq2', common.BCL2FASTQ2_VERSION) common.log("CONFIG", "Demultiplexing using docker image from " + docker.getImageDockerName() + " with command line " + cmd, conf) common.log("CONFIG", "Bcl2fastq docker mount: " + str(os.path.dirname(fastq_output_dir)) + ":" + str(output_docker) + "; " + input_run_data_path + ":" + input_docker + "; " + tmp + ":" + tmp_docker, conf) # Mount input directory docker.addMountDirectory(input_run_data_path, input_docker) docker.addMountDirectory(os.path.dirname(fastq_output_dir), output_docker) docker.addMountDirectory(tmp, tmp_docker) docker.run() exit_code = docker.getExitValue() if exit_code != 0: error("Error while demultiplexing run " + run_id, 'Error while demultiplexing run (exit code: ' + str(exit_code) + ').\nCommand line:\n' + cmd, conf) msg = 'Error while executing bcl2fastq ' + run_id_msg + ' (exit code: ' + str( exit_code) + ').\nCommand line:\n' + cmd # Check if the log file has been generated if not os.path.exists(bcl2fastq_log_file): error("Error with bcl2fastq log for run " + run_id + ".", "No bcl2fastq log available " + bcl2fastq_log_file, conf) common.send_msg('[Aozan] Failed demultiplexing ' + run_id_msg, msg, True, conf) else: msg += "\n\nPlease check the attached bcl2fastq output error file." common.send_msg_with_attachment('[Aozan] Failed demultiplexing ' + run_id_msg, msg, bcl2fastq_log_file, True, conf) return False except Throwable, exp: error("Error while running Docker image", common.exception_msg(exp, conf), conf) return False
sessions = [Settings.HISEQ_STEP_KEY, Settings.DEMUX_STEP_KEY, Settings.QC_STEP_KEY] common.create_html_index_file(conf, run_id, sessions) df_in_bytes = common.df(qc_output_dir) du_in_bytes = common.du(qc_output_dir) df = df_in_bytes / (1024 * 1024 * 1024) du = du_in_bytes / (1024 * 1024) common.log("WARNING", "QC step: output disk free after QC: " + str(df_in_bytes), conf) common.log("WARNING", "QC step: space used by QC: " + str(du_in_bytes), conf) duration = time.time() - start_time msg = 'Ending quality control for run ' + run_id + '.' + \ '\nJob finished at ' + common.time_to_human_readable(time.time()) + \ ' without error in ' + common.duration_to_human_readable(duration) + '. ' + \ 'You will find attached to this message the quality control report.\n\n' + \ 'QC files for this run ' + \ 'can be found in the following directory:\n ' + qc_output_dir # Add path to report if reports.url exists if common.is_conf_key_exists(REPORTS_URL_KEY, conf): msg += '\n\nRun reports can be found at following location:\n ' + conf[REPORTS_URL_KEY] + '/' + run_id msg += '\n\nFor this task %.2f MB has been used and %.2f GB still free.' % (du, df) common.send_msg_with_attachment('[Aozan] Ending quality control for run ' + run_id + ' on ' + common.get_instrument_name(run_id, conf), msg, html_report_file, False, conf) common.log('INFO', 'QC step: successful in ' + common.duration_to_human_readable(duration), conf) return True