def demux_run_with_docker(run_id, input_run_data_path, fastq_output_dir, samplesheet_csv_path, nb_mismatch, conf): """ Demultiplexing the run with bcl2fastq on version parameter with image Docker. Arguments: run_id: The run id input_run_data_path: input run data path to demultiplexing fastq_output_dir: fastq directory to save result on demultiplexing samplesheet_csv_path: samplesheet path in csv format, version used by bcl2fastq conf: configuration dictionary """ # In docker mount with input_run_data_path input_docker = '/data/input' input_run_data_path_in_docker = input_docker run_id_msg = " for run " + run_id + ' on ' + common.get_instrument_name(run_id, conf) # In docker mount with fastq_output_dir output_docker = '/data/output' fastq_data_path_in_docker = output_docker + '/' + os.path.basename(fastq_output_dir) tmp = conf[TMP_PATH_KEY] tmp_docker = '/tmp' bcl2fastq_log_file = tmp + "/bcl2fastq_output_" + run_id + ".err" samplesheet_csv_docker = tmp_docker + '/' + os.path.basename(samplesheet_csv_path) cmd = create_bcl2fastq_command_line(run_id, None, input_run_data_path_in_docker, fastq_data_path_in_docker, samplesheet_csv_docker, tmp_docker, nb_mismatch, conf) try: # Set working in docker on parent demultiplexing run directory. # Demultiplexing run directory will create by bcl2fastq docker = DockerCommand(conf[Settings.DOCKER_URI_KEY], ['/bin/bash', '-c', cmd], 'bcl2fastq2', common.BCL2FASTQ2_VERSION) common.log("CONFIG", "Demultiplexing using docker image from " + docker.getImageDockerName() + " with command line " + cmd, conf) common.log("CONFIG", "Bcl2fastq docker mount: " + str(os.path.dirname(fastq_output_dir)) + ":" + str(output_docker) + "; " + input_run_data_path + ":" + input_docker + "; " + tmp + ":" + tmp_docker, conf) # Mount input directory docker.addMountDirectory(input_run_data_path, input_docker) docker.addMountDirectory(os.path.dirname(fastq_output_dir), output_docker) docker.addMountDirectory(tmp, tmp_docker) docker.run() exit_code = docker.getExitValue() if exit_code != 0: error("Error while demultiplexing run " + run_id, 'Error while demultiplexing run (exit code: ' + str(exit_code) + ').\nCommand line:\n' + cmd, conf) msg = 'Error while executing bcl2fastq ' + run_id_msg + ' (exit code: ' + str( exit_code) + ').\nCommand line:\n' + cmd # Check if the log file has been generated if not os.path.exists(bcl2fastq_log_file): error("Error with bcl2fastq log for run " + run_id + ".", "No bcl2fastq log available " + bcl2fastq_log_file, conf) common.send_msg('[Aozan] Failed demultiplexing ' + run_id_msg, msg, True, conf) else: msg += "\n\nPlease check the attached bcl2fastq output error file." common.send_msg_with_attachment('[Aozan] Failed demultiplexing ' + run_id_msg, msg, bcl2fastq_log_file, True, conf) return False except Throwable, exp: error("Error while running Docker image", common.exception_msg(exp, conf), conf) return False
def qc(run_id, conf): """Proceed to quality control of a run. Arguments: run_id: The run id conf: configuration dictionary """ start_time = time.time() input_run_data_path = common.get_input_run_data_path(run_id, conf) if input_run_data_path is None: return False fastq_input_dir = conf[FASTQ_DATA_PATH_KEY] + '/' + run_id reports_data_base_path = conf[REPORTS_DATA_PATH_KEY] reports_data_path = reports_data_base_path + '/' + run_id qc_output_dir = reports_data_path + '/qc_' + run_id tmp_extension = '.tmp' common.log('INFO', 'QC step: Starting', conf) # Check if input run data data exists if input_run_data_path is None: error("Basecalling data directory does not exist", "Basecalling data directory does not exist.", conf) return False # Check if input root fastq root data exists if not common.is_dir_exists(FASTQ_DATA_PATH_KEY, conf): error("FASTQ data directory does not exist", "FASTQ data directory does not exist: " + conf[FASTQ_DATA_PATH_KEY], conf) return False # Create if not exist report directory for the run if not os.path.exists(reports_data_path): os.mkdir(reports_data_path) # Check if temporary directory exists if not common.is_dir_exists(TMP_PATH_KEY, conf): error("Temporary directory does not exist", "Temporary directory does not exist: " + conf[TMP_PATH_KEY], conf) return False # Check if the output directory already exists if os.path.exists(qc_output_dir): error("The quality control report directory already exists for run " + run_id, 'The quality control report directory already exists for run ' + run_id + ': ' + qc_output_dir, conf) return False # Check if the output directory already exists if os.path.exists(reports_data_path + '/qc_' + run_id + '.tar.bz2'): error("The quality control report archive already exists for run " + run_id, 'The quality control report archive already exists for run ' + run_id + ': ' + reports_data_path + '/qc_' + run_id + '.tar.bz2', conf) return False # Check if enough free space is available if common.df(conf[REPORTS_DATA_PATH_KEY]) < 1 * 1024 * 1024 * 1024: error("Not enough disk space to store aozan quality control for run " + run_id, "Not enough disk space to store aozan reports for run " + run_id + '.\nNeed more than 10 Gb on ' + conf[REPORTS_DATA_PATH_KEY] + '.', conf) return False # Create temporary temporary directory qc_output_dir += tmp_extension if not os.path.exists(qc_output_dir): os.mkdir(qc_output_dir) try: # Initialize the QC object qc = QC(Settings(conf), input_run_data_path, fastq_input_dir, qc_output_dir, conf[TMP_PATH_KEY], run_id) # Compute the report report = qc.computeReport() except AozanException, exp: error("Error while computing QC report for run " + run_id + ".", common.exception_msg(exp, conf), conf) return False
def aozan_main(): """Aozan main method. """ # Define command line parser parser = OptionParser(usage='usage: ' + Globals.APP_NAME_LOWER_CASE + '.sh [options] conf_file') parser.add_option('-q', '--quiet', action='store_true', dest='quiet', default=False, help='quiet') parser.add_option('-v', '--version', action='store_true', dest='version', help='Aozan version') parser.add_option('-e', '--exit-code', action='store_true', dest='exit_code', help='Returns non zero exit code if a step fails') parser.add_option('-c', '--conf', action='store_true', dest='conf', help='Default Aozan configuration, loads before configuration file.') # Parse command line arguments (options, args) = parser.parse_args() # Print Aozan current version if options.version: print Globals.WELCOME_MSG sys.exit(0) # Print default configuration option if options.conf: print common.print_default_configuration() sys.exit(0) # If no argument print usage if len(args) < 1: parser.print_help() sys.exit(1) # Create configuration object conf = LinkedHashMap() # Set the default value in the configuration object common.set_default_conf(conf) # Use default (US) locale Locale.setDefault(Globals.DEFAULT_LOCALE) # Check if OS is Linux if not SystemUtils.isLinux(): sys.stderr.write('ERROR: Aozan can not be executed. Operating system is not Linux\n') sys.exit(1) # Check if configuration file exists conf_file = args[0] if not os.path.isfile(conf_file): sys.stderr.write('ERROR: Aozan can not be executed. Configuration file is missing: ' + \ conf_file + '\n') sys.exit(1) # Load Aozan conf file common.load_conf(conf, conf_file) # End of Aozan if aozan is not enable if common.is_conf_value_defined(AOZAN_ENABLE_KEY, 'false', conf): sys.exit(0) # Init logger try: Common.initLogger(conf[AOZAN_LOG_PATH_KEY], conf[AOZAN_LOG_LEVEL_KEY]) except AozanException, exp: common.exception_msg(exp, conf)
qc_output_dir += tmp_extension if not os.path.exists(qc_output_dir): os.mkdir(qc_output_dir) try: # Initialize the QC object qc = QC(Settings(conf), input_run_data_path, fastq_input_dir, qc_output_dir, conf[TMP_PATH_KEY], run_id) # Compute the report report = qc.computeReport() except AozanException, exp: error("Error while computing QC report for run " + run_id + ".", common.exception_msg(exp, conf), conf) return False except Throwable, exp: error("Error while computing QC report for run " + run_id + ".", common.exception_msg(exp, conf), conf) return False # Remove QC data if not demand if common.is_conf_value_defined(QC_REPORT_SAVE_RAW_DATA_KEY, 'false', conf): try: os.remove(qc_output_dir + '/data-' + run_id + '.txt') # qc.writeRawData(report, qc_output_dir + '/data-' + run_id + '.txt') except AozanException, exp: error("Error while removing QC raw data for run " + run_id + ".", exp.getMessage(), conf) return False # Write the XML report if common.is_conf_value_equals_true(QC_REPORT_SAVE_REPORT_DATA_KEY, conf): try: qc.writeXMLReport(report, qc_output_dir + '/' + run_id + '.xml')