def run_bowtie2(bowtie2_path, pair1, pair2, orphans, database, sam, threads, options, reorder): """ Run bowtie2 with the options provided """ command = [bowtie2_path] # if pairs are to be run as single end, then provide them all as orphans if orphans: orphans = ",".join([pair1, pair2, orphans]) else: orphans = ",".join([pair1, pair2]) command += [ "--threads", str(threads), "-x", database, "-S", sam, "--no-head" ] if orphans: command += ["-U", orphans] if options: command += utilities.format_options_to_list([options]) if reorder: command += ["--reorder"] try: return_code = subprocess.check_call(command) except (EnvironmentError, subprocess.CalledProcessError) as e: message = "Unable to run bowtie2: " + " ".join(command) if hasattr(e, 'output') and e.output: message += "\nError message returned from bowtie2:\n" + e.output sys.exit(message)
def bowtie2_unpaired_command(args, input_fastq_path, output_sam_path, maintain_input_ordering): # Bowtie2 from input as unpaired reads to output, using the provided index command=[args.bowtie2, "-U", input_fastq_path, "-S", output_sam_path, "-x", args.index] # Runtime options command+=["--threads",str(args.threads)] if args.bowtie2_options: command+=utilities.format_options_to_list([args.bowtie2_options]) if not args.verbose: command+=["--quiet"] # Maintain ordering of the fastq, if required if maintain_input_ordering: command+=["--reorder"] return command
def main(): # Parse the arguments from the user args = parse_arguments(sys.argv) # Update the configuration args = update_configuration(args) # set the prefix for the output files full_path_output_prefix = os.path.join(args.output_dir, args.output_prefix) # Start logging setup_logging(args) temp_output_files = [] # Check for compressed files, bam files, or sam files for index in range(len(args.input)): args.input[index] = utilities.get_decompressed_file( args.input[index], args.output_dir, temp_output_files, args.input) args.input[index] = utilities.get_sam_from_bam_file( args.input[index], args.output_dir, temp_output_files, args.input) args.input[index] = utilities.get_fastq_from_sam_file( args.input[index], args.output_dir, temp_output_files, args.input) # Get the format of the first input file file_format = utilities.get_file_format(args.input[0]) if file_format != "fastq": message = "Your input file is of type: " + file_format + ". Please provide an input file of fastq format." logger.critical(message) sys.exit(message) # if this is the new illumina identifier format, create temp files after reformatting the headers for index in range(len(args.input)): args.input[index] = utilities.get_reformatted_identifiers( args.input[index], index, args.output_dir, temp_output_files, args.input) # check for reads that are not ordered and order if needed (if trimmomatic is run) if not args.bypass_trim and len(args.input) == 2: args.input = utilities.check_and_reorder_reads(args.input, args.output_dir, temp_output_files) # remove any temp files from decompress/reformat that are no longer needed utilities.update_temp_output_files(temp_output_files, [], args.input) # set trimmomatic options # this is done after the decompression and conversions from sam/bam # as the default requires the read length from the input sequences if args.trimmomatic_options: # parse the options from the user into an array of options args.trimmomatic_options = utilities.format_options_to_list( args.trimmomatic_options) else: # if trimmomatic options not set by user, then set to default options # use read length of input file for minlen args.trimmomatic_options = utilities.get_default_trimmomatic_options( utilities.get_read_length_fastq(args.input[0]), path=config.trimmomatic_adapter_folder, type="PE" if len(args.input) == 2 else "SE", sequencer_source=args.sequencer_source) # Get the number of reads initially utilities.log_read_count_for_files(args.input, "raw", "Initial number of reads", args.verbose) # Run fastqc if set to run at start of workflow if args.fastqc_start or args.run_trim_repetitive: run.fastqc(args.fastqc_path, args.output_dir, original_input_files, args.threads, args.verbose) #Setting fastqc output zip and txt file path output_txt_files = [] for input_file_name in original_input_files: temp_file = os.path.splitext(input_file_name)[0] if (temp_file.count('fastq') > 0 or temp_file.count('fq') > 0): temp_file = os.path.splitext(temp_file)[0] output_txt_files.append(args.output_dir + "/fastqc/" + temp_file.split('/')[-1] + "_fastqc/fastqc_data.txt") if not args.bypass_trim: if args.run_trim_repetitive: # Get the Min Overrepresented Seq Length args.trimmomatic_options = utilities.get_updated_trimmomatic_parameters( output_txt_files, args.output_dir, args.trimmomatic_options) trimmomatic_output_files = run.trim( args.input, full_path_output_prefix, args.trimmomatic_path, args.trimmomatic_quality_scores, args.max_memory, args.trimmomatic_options, args.threads, args.verbose) else: message = "Bypass trimming" logger.info(message) print(message) trimmomatic_output_files = [args.input] # Get the number of reads after trimming utilities.log_read_count_for_files(trimmomatic_output_files, "trimmed", "Total reads after trimming", args.verbose) # run TRF, if set if not args.bypass_trf: # run trf on all output files trf_output_files = run.tandem( trimmomatic_output_files, full_path_output_prefix, args.match, args.mismatch, args.delta, args.pm, args.pi, args.minscore, args.maxperiod, args.trf_path, args.processes, args.verbose, args.remove_temp_output, args.threads) # remove the aligment files, if intermediate output files should be removed if args.reference_db and args.remove_intermediate_output: temp_output_files += utilities.resolve_sublists( trimmomatic_output_files) else: trf_output_files = trimmomatic_output_files # If a reference database is not provided, then bypass decontamination step if not args.reference_db: message = "Bypass decontamination" logger.info(message) print(message) # resolve sub-lists if present final_output_files = trf_output_files else: final_output_files = run.decontaminate(args, full_path_output_prefix, trf_output_files) # remove trimmed output files, if set to remove intermediate outputx if not args.bypass_trim and args.remove_intermediate_output: temp_output_files += utilities.resolve_sublists(trf_output_files) # If set, concat the final output files if there is more than one final_output_files = utilities.resolve_sublists(final_output_files) if args.cat_final_output and len(final_output_files) > 1: cat_output_file = full_path_output_prefix + config.fastq_file_extension utilities.cat_files(final_output_files, cat_output_file) # if removing intermediate output, then remove the files that were merged if args.remove_intermediate_output: temp_output_files += final_output_files final_output_files = [cat_output_file] else: final_output_files.append(cat_output_file) # Remove any temp output files, if set if not args.store_temp_output: for file in temp_output_files: utilities.remove_file(file) # Run fastqc if set to run at end of workflow if args.fastqc_end: run.fastqc(args.fastqc_path, args.output_dir, final_output_files, args.threads, args.verbose) if len(final_output_files) > 1: message = "\nFinal output files created: \n" else: message = "\nFinal output file created: \n" message = message + "\n".join(final_output_files) + "\n" logger.info(message) print(message)
def update_configuration(args): """ Update the run settings based on the arguments provided """ # get the full path for the output directory args.output_dir = os.path.abspath(args.output_dir) # set if temp output should be removed args.remove_temp_output = not args.store_temp_output # if intermediate output should be removed, then also remove temp output if args.remove_intermediate_output: args.remove_temp_output = True # check the input files are non-empty and readable args.input = [] if (args.input1 and args.input2): args.input.append(os.path.abspath(args.input1)) args.input.append(os.path.abspath(args.input2)) if (args.unpaired): args.input.append(os.path.abspath(args.unpaired)) utilities.is_file_readable(args.input[0], exit_on_error=True) if len(args.input) == 2: utilities.is_file_readable(args.input[1], exit_on_error=True) elif len(args.input) > 2: sys.exit("ERROR: Please provide at most 2 input files.") elif len(args.input) == 0: sys.exit( "ERROR: Please provide --input1/--input2 or --unpaired (input) files." ) #Store original file paths for FASTQC for input in args.input: original_input_files.append(input) # create the output directory if needed utilities.create_directory(args.output_dir) # set bowtie2 options if args.bowtie2_options: # parse the options from the user into any array of options args.bowtie2_options = utilities.format_options_to_list( args.bowtie2_options) else: # if not set by user, then set to default options args.bowtie2_options = config.bowtie2_options # add the quality scores to the bowtie2 options args.bowtie2_options += [ config.bowtie2_flag_start + args.trimmomatic_quality_scores ] # set the mode for single end input file if len(args.input) == 1: args.decontaminate_pairs = "unpaired" # set the bowtie2 mode based on the pairs input args.discordant = False if args.decontaminate_pairs != "lenient": args.discordant = True # update the quality score option into a flag for trimmomatic args.trimmomatic_quality_scores = config.trimmomatic_flag_start + args.trimmomatic_quality_scores # find the location of trimmomatic, trimmomatic does not need to be executable if not args.bypass_trim: args.trimmomatic_path = utilities.find_dependency( args.trimmomatic_path, config.trimmomatic_jar, "trimmomatic", "--trimmomatic", bypass_permissions_check=True) # find the location of bmtagger, if set to run if args.reference_db: if args.bmtagger: args.bmtagger_path = utilities.find_dependency( args.bmtagger_path, config.bmtagger_exe, "bmtagger", "--bmtagger", bypass_permissions_check=False) # add this folder to path, so as to be able to find other dependencies like bmfilter utilities.add_exe_to_path(os.path.dirname(args.bmtagger_path)) else: # find the location of bowtie2, if not running with bmtagger args.bowtie2_path = utilities.find_dependency( args.bowtie2_path, config.bowtie2_exe, "bowtie2", "--bowtie2", bypass_permissions_check=False) # find the location of trf, if set to run if not args.bypass_trf: args.trf_path = utilities.find_dependency( args.trf_path, config.trf_exe, "trf", "--trf", bypass_permissions_check=False) # if fastqc is set to be run, check if the executable can be found if args.fastqc_start or args.fastqc_end: args.fastqc_path = utilities.find_dependency( args.fastqc_path, config.fastqc_exe, "fastqc", "--fastqc", bypass_permissions_check=False) # set the default output prefix if args.output_prefix == None: if args.input[0].endswith(".gz") or args.input[0].endswith(".bz2"): # remove compression extension if present infile_base = os.path.splitext( os.path.splitext(os.path.basename(args.input[0]))[0])[0] else: infile_base = os.path.splitext(os.path.basename(args.input[0]))[0] args.output_prefix = infile_base + "_kneaddata" # find the bowtie2 indexes for each of the reference databases # reference database inputs can be directories, indexes, or index files if args.reference_db: reference_indexes = [] database_type = "bowtie2" if args.bmtagger: database_type = "bmtagger" for directory in args.reference_db: reference_indexes.append( utilities.find_database_index(os.path.abspath(directory), database_type)) args.reference_db = reference_indexes return args