def customize(cls, config, input_bams, output_bam, output_metrics=None, keep_dupes=False, dependencies=()): jar_file = os.path.join(config.jar_root, "MarkDuplicates.jar") params = AtomicJavaCmdBuilder(jar_file, jre_options=config.jre_options) # Create .bai index, since it is required by a lot of other programs params.set_option("CREATE_INDEX", "True", sep="=") params.set_option("OUTPUT", "%(OUT_BAM)s", sep="=") params.set_option("METRICS_FILE", "%(OUT_METRICS)s", sep="=") params.add_multiple_options("I", input_bams, sep="=") if not keep_dupes: # Remove duplicates from output by default to save disk-space params.set_option("REMOVE_DUPLICATES", "True", sep="=", fixed=False) output_metrics = output_metrics or swap_ext(output_bam, ".metrics") params.set_kwargs(OUT_BAM=output_bam, OUT_BAI=swap_ext(output_bam, ".bai"), OUT_METRICS=output_metrics, CHECK_JAR=_picard_version(config, jar_file)) return {"command": params, "dependencies": dependencies}
def customize(cls, config, input_bams, output_bam, output_metrics=None, keep_dupes=False, dependencies=()): params = picard_command(config, "MarkDuplicates") # Create .bai index, since it is required by a lot of other programs params.set_option("CREATE_INDEX", "True", sep="=") params.set_option("OUTPUT", "%(OUT_BAM)s", sep="=") params.set_option("METRICS_FILE", "%(OUT_METRICS)s", sep="=") params.add_multiple_options("I", input_bams, sep="=") if not keep_dupes: # Remove duplicates from output by default to save disk-space params.set_option("REMOVE_DUPLICATES", "True", sep="=", fixed=False) output_metrics = output_metrics or swap_ext(output_bam, ".metrics") params.set_kwargs(OUT_BAM=output_bam, OUT_BAI=swap_ext(output_bam, ".bai"), OUT_METRICS=output_metrics) return {"command": params, "dependencies": dependencies}
def customize(cls, config, input_bams, output_bam, output_metrics = None, dependencies = ()): jar_file = os.path.join(config.jar_root, "MarkDuplicates.jar") params = AtomicJavaCmdBuilder(config, jar_file) # Create .bai index, since it is required by a lot of other programs params.set_option("CREATE_INDEX", "True", sep = "=") params.set_option("OUTPUT", "%(OUT_BAM)s", sep = "=") params.set_option("METRICS_FILE", "%(OUT_METRICS)s", sep = "=") input_bams = safe_coerce_to_tuple(input_bams) for (index, filename) in enumerate(input_bams): params.add_option("I", "%%(IN_BAM_%02i)s" % index, sep = "=") params.set_kwargs(**{("IN_BAM_%02i" % index) : filename}) # Remove duplicates from output by default to save disk-space params.set_option("REMOVE_DUPLICATES", "True", sep = "=", fixed = False) params.set_kwargs(OUT_BAM = output_bam, OUT_BAI = swap_ext(output_bam, ".bai"), OUT_METRICS = output_metrics or swap_ext(output_bam, ".metrics"), CHECK_JAR = _picard_version(jar_file)) return {"command" : params, "dependencies" : dependencies}
def _build_examl_bootstraps(options, phylo, destination, input_alignment, input_partition, dependencies): bootstraps = [] num_bootstraps = phylo["ExaML"]["Bootstraps"] bootstrap_destination = os.path.join(destination, "bootstraps") bootstrap_template = os.path.join(bootstrap_destination, "bootstrap.%04i.phy") for bootstrap_num in xrange(num_bootstraps): bootstrap_alignment = bootstrap_template % (bootstrap_num,) bootstrap = PHYLIPBootstrapNode(input_alignment = input_alignment, input_partition = input_partition, output_alignment = bootstrap_alignment, seed = random.randint(1, 2**32 - 1), dependencies = dependencies) bootstrap_binary = swap_ext(bootstrap_alignment, ".binary") bootstrap_final = swap_ext(bootstrap_alignment, ".%s") bs_binary = ExaMLParserNode(input_alignment = bootstrap_alignment, input_partition = input_partition, output_file = bootstrap_binary, dependencies = bootstrap) bootstraps.append(_examl_nodes(options = options, settings = phylo, input_alignment = bootstrap_alignment, input_partitions = input_partition, input_binary = bootstrap_binary, output_template = bootstrap_final, dependencies = bs_binary)) if bootstraps: meta = MetaNode(description = "Bootstraps", subnodes = bootstraps, dependencies = dependencies) return _build_rerooted_trees(meta, phylo["RootTreesOn"]) return None
def customize(cls, config, input_bams, output_bam, output_metrics=None, dependencies=()): jar_file = os.path.join(config.jar_root, "MarkDuplicates.jar") params = AtomicJavaCmdBuilder(config, jar_file) # Create .bai index, since it is required by a lot of other programs params.set_option("CREATE_INDEX", "True", sep="=") params.set_option("OUTPUT", "%(OUT_BAM)s", sep="=") params.set_option("METRICS_FILE", "%(OUT_METRICS)s", sep="=") input_bams = safe_coerce_to_tuple(input_bams) for (index, filename) in enumerate(input_bams): params.add_option("I", "%%(IN_BAM_%02i)s" % index, sep="=") params.set_kwargs(**{("IN_BAM_%02i" % index): filename}) # Remove duplicates from output by default to save disk-space params.set_option("REMOVE_DUPLICATES", "True", sep="=", fixed=False) params.set_kwargs(OUT_BAM=output_bam, OUT_BAI=swap_ext(output_bam, ".bai"), OUT_METRICS=output_metrics or swap_ext(output_bam, ".metrics"), CHECK_JAR=_picard_version(jar_file)) return {"command": params, "dependencies": dependencies}
def __init__(self, config, input_bams, pipename="input.bam"): self.pipe = pipename self.files = safe_coerce_to_tuple(input_bams) self.commands = [] self.kwargs = {"TEMP_IN_BAM": self.pipe} if len(self.files) > 1: jar_file = os.path.join(config.jar_root, "MergeSamFiles.jar") params = AtomicJavaCmdBuilder(jar=jar_file, temp_root=config.temp_root, jre_options=config.jre_options) params.set_option("SO", "coordinate", sep="=", fixed=False) params.set_option("CREATE_INDEX", "False", sep="=") params.set_option("COMPRESSION_LEVEL", 0, sep="=") params.set_option("OUTPUT", "%(TEMP_OUT_BAM)s", sep="=") params.add_multiple_options("I", input_bams, sep="=") params.set_kwargs(CHECK_JAR=_picard_version(config, jar_file), TEMP_OUT_BAM=self.pipe) self.commands = [params.finalize()] else: # Ensure that the actual command depends on the input self.kwargs["IN_FILE_00"] = self.files[0] self.kwargs["IN_FILE_01"] = swap_ext(self.files[0], ".bai")
def customize(cls, input_alignment, input_partition, output_alignment, dependencies=()): command = AtomicCmdBuilder("raxmlHPC", set_cwd=True) # Read and (in the case of empty columns) reduce input command.set_option("-f", "j") # Output files are saved with a .Pypeline postfix, and subsequently renamed command.set_option("-n", "Pypeline") # Model required, but not used command.set_option("-m", "GTRGAMMA") # Set random seed for bootstrap generation. May be set to a fixed value to allow replicability. command.set_option("-b", int(random.random() * 2**31 - 1), fixed=False) # Generate a single bootstrap alignment (makes growing the number of bootstraps easier). command.set_option("-N", 1, fixed=False) # Symlink to sequence and partitions, to prevent the creation of *.reduced files outside temp folder # In addition, it may be nessesary to remove the .reduced files if created command.set_option("-s", "input.alignment") command.set_option("-q", "input.partition") command.set_kwargs(IN_ALIGNMENT=input_alignment, IN_PARTITION=input_partition, OUT_ALIGNMENT=output_alignment, OUT_INFO=fileutils.swap_ext(output_alignment, ".info")) return {"command": command}
def customize(cls, input_alignment, input_partition, output_alignment, dependencies = ()): command = AtomicCmdBuilder("raxmlHPC", set_cwd = True) # Read and (in the case of empty columns) reduce input command.set_option("-f", "j") # Output files are saved with a .Pypeline postfix, and subsequently renamed command.set_option("-n", "Pypeline") # Model required, but not used command.set_option("-m", "GTRGAMMA") # Set random seed for bootstrap generation. May be set to a fixed value to allow replicability. command.set_option("-b", int(random.random() * 2**31 - 1), fixed = False) # Generate a single bootstrap alignment (makes growing the number of bootstraps easier). command.set_option("-N", 1, fixed = False) # Symlink to sequence and partitions, to prevent the creation of *.reduced files outside temp folder # In addition, it may be nessesary to remove the .reduced files if created command.set_option("-s", "input.alignment") command.set_option("-q", "input.partition") command.set_kwargs(IN_ALIGNMENT = input_alignment, IN_PARTITION = input_partition, OUT_ALIGNMENT = output_alignment, OUT_INFO = fileutils.swap_ext(output_alignment, ".info")) return {"command" : command}
def __init__(self, config, reference, intervals, infiles, outfile, dependencies=()): self._basename = os.path.basename(outfile) infiles = safe_coerce_to_tuple(infiles) jar_file = os.path.join(config.jar_root, "GenomeAnalysisTK.jar") command = AtomicJavaCmdBuilder(jar_file, jre_options=config.jre_options) command.set_option("-T", "IndelRealigner") command.set_option("-R", "%(IN_REFERENCE)s") command.set_option("-targetIntervals", "%(IN_INTERVALS)s") command.set_option("-o", "%(OUT_BAMFILE)s") command.set_option("--bam_compression", 0) command.set_option("--disable_bam_indexing") _set_input_files(command, infiles) command.set_kwargs(IN_REFERENCE=reference, IN_REF_DICT=fileutils.swap_ext(reference, ".dict"), IN_INTERVALS=intervals, OUT_BAMFILE=outfile, CHECK_GATK=_get_gatk_version_check(config)) calmd = AtomicCmd(["samtools", "calmd", "-b", "%(TEMP_IN_BAM)s", "%(IN_REF)s"], TEMP_IN_BAM=self._basename, IN_REF=reference, TEMP_OUT_STDOUT=self._basename + ".calmd", CHECK_VERSION=SAMTOOLS_VERSION) description = "<Indel Realigner (aligning): %s -> %r>" \ % (describe_files(infiles), outfile) CommandNode.__init__(self, description=description, command=ParallelCmds([command.finalize(), calmd]), dependencies=dependencies)
def __init__(self, config, target_name, input_files, output_file, intervals_file = None, print_stats = False, max_contigs = _MAX_CONTIGS, dependencies = ()): self._target_name = target_name self._input_files = safe_coerce_to_tuple(input_files) self._output_file = output_file self._intervals = intervals_file self._print_stats = print_stats self._max_contigs = max_contigs self._max_contigs_reached = False input_files = [] input_files.extend(self._input_files) input_files.extend(swap_ext(input_file, ".bai") for input_file in self._input_files) if intervals_file: input_files.append(intervals_file) executables = ["coverageBed"] if intervals_file else ["genomeCoverageBed"] auxiliary_files = [] for cmd in concatenate_input_bams(config, self._input_files)[0]: executables.extend(cmd.executables) auxiliary_files.extend(cmd.auxiliary_files) Node.__init__(self, description = "<DepthHistogram: %s -> '%s'>" \ % (describe_files(self._input_files), self._output_file), input_files = input_files, output_files = self._output_file, dependencies = dependencies, executables = executables, auxiliary_files = auxiliary_files)
def __init__(self, config, reference, intervals, infiles, outfile, dependencies = ()): self._basename = os.path.basename(outfile) infiles = safe_coerce_to_tuple(infiles) jar_file = os.path.join(config.jar_root, "GenomeAnalysisTK.jar") command = AtomicJavaCmdBuilder(config, jar_file) command.set_option("-T", "IndelRealigner") command.set_option("-R", "%(IN_REFERENCE)s") command.set_option("-targetIntervals", "%(IN_INTERVALS)s") command.set_option("-o", "%(OUT_BAMFILE)s") command.set_option("--bam_compression", 0) command.set_option("--disable_bam_indexing") _set_input_files(command, infiles) command.set_kwargs(IN_REFERENCE = reference, IN_REF_DICT = fileutils.swap_ext(reference, ".dict"), IN_INTERVALS = intervals, OUT_BAMFILE = outfile) calmd = AtomicCmd(["samtools", "calmd", "-b", "%(TEMP_IN_BAM)s", "%(IN_REF)s"], TEMP_IN_BAM = self._basename, IN_REF = reference, TEMP_OUT_STDOUT = self._basename + ".calmd") description = "<Indel Realign: %i file(s) -> '%s'>" \ % (len(infiles), outfile) CommandNode.__init__(self, description = description, command = ParallelCmds([command.finalize(), calmd]), dependencies = dependencies)
def _collect_subsets(roi, subset, path): if roi not in subsets_by_regions: raise MakefileError("Subset of unknown region (%r) requested at %r" % (roi, path)) roi_fname = swap_ext(subsets_by_regions[roi]["BED"], subset + ".names") if not os.path.isfile(roi_fname): raise MakefileError( ("Subset file does not exist for Regions Of Interest:\n" " Region = %r\n Subset = %r\n Path = %r") % (roi, subset, roi_fname) ) sequences = set() with open(roi_fname) as handle: for line in handle: line = line.strip() if line and not line.startswith("#"): sequences.add(line) known_seqs = subsets_by_regions[roi]["Sequences"][None] unknown_seqs = sequences - known_seqs if unknown_seqs: message = ( "Unknown sequences in subset file:\n" " File = %r\n Region = %r\n Subset = %r\n" " Unknown sequence names =" ) % (roi_fname, roi, subset) unknown_seqs = list(sorted(unknown_seqs)) if len(unknown_seqs) > 5: unknown_seqs = unknown_seqs[:5] + ["..."] message = "\n - ".join([message] + unknown_seqs) raise MakefileError(message) subsets_by_regions[roi]["SubsetFiles"][subset] = (roi_fname,) subsets_by_regions[roi]["Sequences"][subset] = frozenset(sequences)
def _collect_subsets(roi, subset, path): if roi not in subsets_by_regions: raise MakefileError( "Subset of unknown region (%r) requested at %r" % (roi, path)) roi_fname = swap_ext(subsets_by_regions[roi]["BED"], subset + ".names") if not os.path.isfile(roi_fname): raise MakefileError( ("Subset file does not exist for Regions Of Interest:\n" " Region = %r\n Subset = %r\n Path = %r") % (roi, subset, roi_fname)) sequences = set() with open(roi_fname) as handle: for line in handle: line = line.strip() if line and not line.startswith("#"): sequences.add(line) known_seqs = subsets_by_regions[roi]["Sequences"][None] unknown_seqs = sequences - known_seqs if unknown_seqs: message = ("Unknown sequences in subset file:\n" " File = %r\n Region = %r\n Subset = %r\n" " Unknown sequence names =") \ % (roi_fname, roi, subset) unknown_seqs = list(sorted(unknown_seqs)) if len(unknown_seqs) > 5: unknown_seqs = unknown_seqs[:5] + ["..."] message = "\n - ".join([message] + unknown_seqs) raise MakefileError(message) subsets_by_regions[roi]["SubsetFiles"][subset] = (roi_fname, ) subsets_by_regions[roi]["Sequences"][subset] = frozenset(sequences)
def parse_arguments(argv, ext): parser = argparse.ArgumentParser(prog="paleomix %s" % (ext.strip("."),)) parser.add_argument("infile", metavar="BAM", help="Filename of a sorted BAM file. If set to '-' " "the file is read from STDIN.") parser.add_argument("outfile", metavar="OUTPUT", nargs='?', help="Filename of output table; defaults to name of " "the input BAM with a '%s' extension. If " "set to '-' the table is printed to STDOUT." % (ext,)) parser.add_argument("--target-name", default=None, help="Name used for 'Target' column; defaults to the " "filename of the BAM file.") parser.add_argument("--regions-file", default=None, dest="regions_fpath", help="BED file containing regions of interest; %s " "is calculated only for these grouping by the " "name used in the BED file, or the contig name " "if no name has been specified for a record." % (ext.strip("."),)) parser.add_argument('--max-contigs', default=100, type=int, help="The maximum number of contigs allowed in a BAM " "file. If this number is exceeded, the entire " "set of contigs is aggregated into one pseudo-" "contig named '<Genome>'. This is done to " "limit table sizes [default: %(default)s]") parser.add_argument('--ignore-readgroups', default=False, action="store_true", help="Ignore readgroup information in reads, and only " "provide aggregated statistics; this is required " "if readgroup information is missing or partial " "[default: %(default)s]") parser.add_argument('--overwrite-output', default=False, action="store_true", help="Overwrite output file if it it exists; by " "default, the script will terminate if the file " "already exists.") args = parser.parse_args(argv) if not args.outfile: args.outfile = swap_ext(args.infile, ext) if args.ignore_readgroups: args.get_readgroup_func = _get_readgroup_ignored else: args.get_readgroup_func = _get_readgroup if not args.target_name: if args.infile == "-": args.target_name = "<STDIN>" else: args.target_name = os.path.basename(args.infile) if os.path.exists(args.outfile) and not args.overwrite_output: parser.error("Destination filename already exists (%r); use option " "--overwrite-output to allow overwriting of this file." % (args.outfile,)) return args
def _set_input_files(command, input_files): keys = {} for (index, filename) in enumerate(input_files): command.add_option("-I", "%%(IN_BAMFILE_%02i)s" % index) keys["IN_BAMFILE_%02i" % index] = filename keys["IN_BAIFILE_%02i" % index] = swap_ext(filename, ".bai") command.set_kwargs(**keys)
def customize(cls, config, input_bam, output_log=None, dependencies=()): params = picard_command(config, "ValidateSamFile") params.set_option("I", "%(IN_BAM)s", sep="=") output_log = output_log or swap_ext(input_bam, ".validated") params.set_kwargs(IN_BAM=input_bam, OUT_STDOUT=output_log) return {"command": params, "dependencies": dependencies}
def customize(cls, config, reference, dependencies=()): params = picard_command(config, "CreateSequenceDictionary") params.set_option("R", "%(TEMP_OUT_REF)s", sep="=") params.set_option("O", "%(OUT_DICT)s", sep="=") params.set_kwargs(IN_REF=reference, TEMP_OUT_REF=os.path.basename(reference), OUT_DICT=swap_ext(reference, ".dict")) return {"command": params, "dependencies": dependencies}
def _teardown(self, config, temp): fileutils.move_file(os.path.join(temp, "RAxML_info.Pypeline"), os.path.join(temp, fileutils.swap_ext(self._output_alignment, ".info"))) fileutils.move_file(os.path.join(temp, "input.alignment.BS0"), os.path.join(temp, self._output_alignment)) os.remove(os.path.join(temp, "input.alignment")) os.remove(os.path.join(temp, "input.partition")) CommandNode._teardown(self, config, temp)
def __init__(self, infile, dependencies = ()): cmd_index = AtomicCmd(["samtools", "index", "%(IN_BAM)s", "%(OUT_BAI)s"], IN_BAM = infile, OUT_BAI = swap_ext(infile, ".bai"), CHECK_SAM = SAMTOOLS_VERSION) CommandNode.__init__(self, description = "<BAMIndex: '%s'>" % (infile,), command = cmd_index, dependencies = dependencies)
def _setup(self, config, temp_root): CommandNode._setup(self, config, temp_root) dst_fname = os.path.join(temp_root, self._bam_input.pipe) if len(self._bam_input.files) > 1: os.mkfifo(dst_fname) else: src_fname, = self._bam_input.files os.symlink(os.path.join(os.getcwd(), src_fname), dst_fname) src_fname = os.path.join(os.getcwd(), swap_ext(src_fname, ".bai")) os.symlink(src_fname, dst_fname + ".bai")
def __init__(self, input_file, target_name, output_file=None, intervals_file=None, max_contigs=_MAX_CONTIGS, dependencies=()): self._target_name = target_name self._input_file = input_file self._output_file = output_file or swap_ext(input_file, ".coverage") self._intervals_file = intervals_file self._max_contigs = max_contigs self._max_contigs_reached = False Node.__init__(self, description = "<Coverage: '%s' -> '%s'>" \ % (input_file, self._output_file), input_files = (input_file, swap_ext(input_file, ".bai")), output_files = self._output_file, dependencies = dependencies)
def customize(cls, config, input_bam, output_log=None, dependencies=()): jar_file = os.path.join(config.jar_root, "ValidateSamFile.jar") params = AtomicJavaCmdBuilder(config, jar_file) params.set_option("I", "%(IN_BAM)s", sep="=") params.set_kwargs(IN_BAM=input_bam, OUT_STDOUT=output_log or swap_ext(input_bam, ".validated"), CHECK_JAR=_picard_version(jar_file)) return {"command": params, "dependencies": dependencies}
def customize(cls, config, input_bam, output_log = None, dependencies = ()): jar_file = os.path.join(config.jar_root, "ValidateSamFile.jar") params = AtomicJavaCmdBuilder(config, jar_file) params.set_option("I", "%(IN_BAM)s", sep = "=") params.set_kwargs(IN_BAM = input_bam, OUT_STDOUT = output_log or swap_ext(input_bam, ".validated"), CHECK_JAR = _picard_version(jar_file)) return {"command" : params, "dependencies" : dependencies}
def customize(cls, config, reference, dependencies=()): jar_file = os.path.join(config.jar_root, "CreateSequenceDictionary.jar") params = AtomicJavaCmdBuilder(config, jar_file) params.set_option("R", "%(IN_REF)s", sep="=") params.set_option("O", "%(OUT_DICT)s", sep="=") params.set_kwargs(IN_REF=reference, OUT_DICT=swap_ext(reference, ".dict"), CHECK_JAR=_picard_version(jar_file)) return {"command": params, "dependencies": dependencies}
def customize(cls, config, input_bams, output_bam, dependencies=()): params = picard_command(config, "MergeSamFiles") params.set_option("OUTPUT", "%(OUT_BAM)s", sep="=") params.set_option("CREATE_INDEX", "True", sep="=") params.set_option("SO", "coordinate", sep="=", fixed=False) params.add_multiple_options("I", input_bams, sep="=") params.set_kwargs(OUT_BAM=output_bam, OUT_BAI=swap_ext(output_bam, ".bai")) return {"command": params, "dependencies": dependencies}
def customize(cls, config, reference, dependencies = ()): jar_file = os.path.join(config.jar_root, "CreateSequenceDictionary.jar") params = AtomicJavaCmdBuilder(config, jar_file) params.set_option("R", "%(IN_REF)s", sep = "=") params.set_option("O", "%(OUT_DICT)s", sep = "=") params.set_kwargs(IN_REF = reference, OUT_DICT = swap_ext(reference, ".dict"), CHECK_JAR = _picard_version(jar_file)) return {"command" : params, "dependencies" : dependencies}
def _teardown(self, config, temp): fileutils.move_file( os.path.join(temp, "RAxML_info.Pypeline"), os.path.join(temp, fileutils.swap_ext(self._output_alignment, ".info"))) fileutils.move_file(os.path.join(temp, "input.alignment.BS0"), os.path.join(temp, self._output_alignment)) os.remove(os.path.join(temp, "input.alignment")) os.remove(os.path.join(temp, "input.partition")) CommandNode._teardown(self, config, temp)
def _build_examl_bootstraps(options, phylo, destination, input_alignment, input_partition, dependencies): bootstraps = [] num_bootstraps = phylo["ExaML"]["Bootstraps"] bootstrap_destination = os.path.join(destination, "bootstraps") bootstrap_template = os.path.join(bootstrap_destination, "bootstrap.%04i.phy") for bootstrap_num in xrange(num_bootstraps): bootstrap_alignment = bootstrap_template % (bootstrap_num, ) bootstrap = PHYLIPBootstrapNode(input_alignment=input_alignment, input_partition=input_partition, output_alignment=bootstrap_alignment, seed=random.randint(1, 2**32 - 1), dependencies=dependencies) bootstrap_binary = swap_ext(bootstrap_alignment, ".binary") bootstrap_final = swap_ext(bootstrap_alignment, ".%s") bs_binary = ExaMLParserNode(input_alignment=bootstrap_alignment, input_partition=input_partition, output_file=bootstrap_binary, dependencies=bootstrap) bootstraps.append( _examl_nodes(options=options, settings=phylo, input_alignment=bootstrap_alignment, input_partitions=input_partition, input_binary=bootstrap_binary, output_template=bootstrap_final, dependencies=bs_binary)) if bootstraps: meta = MetaNode(description="Bootstraps", subnodes=bootstraps, dependencies=dependencies) return _build_rerooted_trees(meta, phylo["RootTreesOn"]) return None
def customize(cls, config, input_bams, output_bam, dependencies=()): jar_file = os.path.join(config.jar_root, "MergeSamFiles.jar") params = AtomicJavaCmdBuilder(jar_file, jre_options=config.jre_options) params.set_option("OUTPUT", "%(OUT_BAM)s", sep="=") params.set_option("CREATE_INDEX", "True", sep="=") params.set_option("SO", "coordinate", sep="=", fixed=False) params.add_multiple_options("I", input_bams, sep="=") params.set_kwargs(OUT_BAM=output_bam, OUT_BAI=swap_ext(output_bam, ".bai"), CHECK_JAR=_picard_version(config, jar_file)) return {"command": params, "dependencies": dependencies}
def customize(cls, config, input_bams, output_bam, dependencies=()): jar_file = os.path.join(config.jar_root, "MergeSamFiles.jar") params = AtomicJavaCmdBuilder(config, jar_file) params.set_option("OUTPUT", "%(OUT_BAM)s", sep="=") params.set_option("CREATE_INDEX", "True", sep="=") params.set_kwargs(OUT_BAM=output_bam, OUT_BAI=swap_ext(output_bam, ".bai"), CHECK_JAR=_picard_version(jar_file)) for (index, filename) in enumerate(input_bams, start=1): params.add_option("I", "%%(IN_BAM_%02i)s" % index, sep="=") params.set_kwargs(**{("IN_BAM_%02i" % index): filename}) params.set_option("SO", "coordinate", sep="=", fixed=False) return {"command": params, "dependencies": dependencies}
def __init__(self, config, reference, infiles, outfile, dependencies=()): infiles = safe_coerce_to_tuple(infiles) jar_file = os.path.join(config.jar_root, "GenomeAnalysisTK.jar") command = AtomicJavaCmdBuilder(jar_file, jre_options=config.jre_options) command.set_option("-T", "RealignerTargetCreator") command.set_option("-R", "%(IN_REFERENCE)s") command.set_option("-o", "%(OUT_INTERVALS)s") _set_input_files(command, infiles) command.set_kwargs( IN_REFERENCE=reference, IN_REF_DICT=fileutils.swap_ext(reference, ".dict"), OUT_INTERVALS=outfile, CHECK_GATK=_get_gatk_version_check(config), ) description = "<Indel Realigner (training): %s -> %r>" % (describe_files(infiles), outfile) CommandNode.__init__(self, description=description, command=command.finalize(), dependencies=dependencies)
def customize(cls, config, input_bams, output_bam, dependencies = ()): jar_file = os.path.join(config.jar_root, "MergeSamFiles.jar") params = AtomicJavaCmdBuilder(config, jar_file) params.set_option("OUTPUT", "%(OUT_BAM)s", sep = "=") params.set_option("CREATE_INDEX", "True", sep = "=") params.set_kwargs(OUT_BAM = output_bam, OUT_BAI = swap_ext(output_bam, ".bai"), CHECK_JAR = _picard_version(jar_file)) for (index, filename) in enumerate(input_bams, start = 1): params.add_option("I", "%%(IN_BAM_%02i)s" % index, sep = "=") params.set_kwargs(**{("IN_BAM_%02i" % index) : filename}) params.set_option("SO", "coordinate", sep = "=", fixed = False) return {"command" : params, "dependencies" : dependencies}
def _build_coverage_nodes_cached(files_and_nodes, target_name, aoi_name, aoi_filename, cache): output_ext = ".coverage" if aoi_name: output_ext = ".%s.coverage" % aoi_name coverages = {} for (input_filename, node) in files_and_nodes.iteritems(): output_filename = swap_ext(input_filename, output_ext) cache_key = (aoi_filename, input_filename) if cache_key not in cache: cache[cache_key] = CoverageNode(input_file = input_filename, output_file = output_filename, target_name = target_name, intervals_file = aoi_filename, dependencies = node) coverages[output_filename] = cache[cache_key] return coverages
def __init__(self, config, reference, infiles, outfile, dependencies=()): infiles = safe_coerce_to_tuple(infiles) jar_file = os.path.join(config.jar_root, "GenomeAnalysisTK.jar") command = AtomicJavaCmdBuilder(config, jar_file) command.set_option("-T", "RealignerTargetCreator") command.set_option("-R", "%(IN_REFERENCE)s") command.set_option("-o", "%(OUT_INTERVALS)s") _set_input_files(command, infiles) command.set_kwargs(IN_REFERENCE=reference, IN_REF_DICT=fileutils.swap_ext(reference, ".dict"), OUT_INTERVALS=outfile) description = "<Train Indel Realigner: %i file(s) -> '%s'>" \ % (len(infiles), outfile) CommandNode.__init__(self, description=description, command=command.finalize(), dependencies=dependencies)
def __init__(self, config, reference, infiles, outfile, dependencies = ()): infiles = safe_coerce_to_tuple(infiles) jar_file = os.path.join(config.jar_root, "GenomeAnalysisTK.jar") command = AtomicJavaCmdBuilder(config, jar_file) command.set_option("-T", "RealignerTargetCreator") command.set_option("-R", "%(IN_REFERENCE)s") command.set_option("-o", "%(OUT_INTERVALS)s") _set_input_files(command, infiles) command.set_kwargs(IN_REFERENCE = reference, IN_REF_DICT = fileutils.swap_ext(reference, ".dict"), OUT_INTERVALS = outfile) description = "<Train Indel Realigner: %i file(s) -> '%s'>" \ % (len(infiles), outfile) CommandNode.__init__(self, description = description, command = command.finalize(), dependencies = dependencies)
def _build_coverage_nodes_cached(files_and_nodes, target_name, aoi_name, aoi_filename, cache): output_ext = ".coverage" if aoi_name: output_ext = ".%s.coverage" % aoi_name coverages = {} for (input_filename, node) in files_and_nodes.iteritems(): output_filename = swap_ext(input_filename, output_ext) cache_key = (aoi_filename, input_filename) if cache_key not in cache: cache[cache_key] = CoverageNode(input_file=input_filename, output_file=output_filename, target_name=target_name, intervals_file=aoi_filename, dependencies=node) coverages[output_filename] = cache[cache_key] return coverages
def __init__(self, config, reference, infiles, outfile, dependencies=()): infiles = safe_coerce_to_tuple(infiles) jar_file = os.path.join(config.jar_root, "GenomeAnalysisTK.jar") command = AtomicJavaCmdBuilder(jar_file, jre_options=config.jre_options) command.set_option("-T", "RealignerTargetCreator") command.set_option("-R", "%(IN_REFERENCE)s") command.set_option("-o", "%(OUT_INTERVALS)s") _set_input_files(command, infiles) command.set_kwargs(IN_REFERENCE=reference, IN_REF_DICT=fileutils.swap_ext(reference, ".dict"), OUT_INTERVALS=outfile, CHECK_GATK=_get_gatk_version_check(config)) description = "<Indel Realigner (training): %s -> %r>" \ % (describe_files(infiles), outfile) CommandNode.__init__(self, description=description, command=command.finalize(), dependencies=dependencies)
def __init__(self, config, target_name, input_files, output_file, intervals_file=None, print_stats=False, max_contigs=_MAX_CONTIGS, dependencies=()): self._target_name = target_name self._input_files = safe_coerce_to_tuple(input_files) self._output_file = output_file self._intervals = intervals_file self._print_stats = print_stats self._max_contigs = max_contigs self._max_contigs_reached = False input_files = [] input_files.extend(self._input_files) input_files.extend( swap_ext(input_file, ".bai") for input_file in self._input_files) if intervals_file: input_files.append(intervals_file) executables = ["coverageBed" ] if intervals_file else ["genomeCoverageBed"] auxiliary_files = [] for cmd in concatenate_input_bams(config, self._input_files)[0]: executables.extend(cmd.executables) auxiliary_files.extend(cmd.auxiliary_files) Node.__init__(self, description = "<DepthHistogram: %s -> '%s'>" \ % (describe_files(self._input_files), self._output_file), input_files = input_files, output_files = self._output_file, dependencies = dependencies, executables = executables, auxiliary_files = auxiliary_files)
def __init__(self, config, input_bams, pipename="input.bam"): self.pipe = pipename self.files = safe_coerce_to_tuple(input_bams) self.commands = [] self.kwargs = {"TEMP_IN_BAM": self.pipe} if len(self.files) > 1: params = picard_command(config, "MergeSamFiles") params.set_option("SO", "coordinate", sep="=", fixed=False) params.set_option("CREATE_INDEX", "False", sep="=") params.set_option("COMPRESSION_LEVEL", 0, sep="=") params.set_option("OUTPUT", "%(TEMP_OUT_BAM)s", sep="=") params.add_multiple_options("I", input_bams, sep="=") params.set_kwargs(TEMP_OUT_BAM=self.pipe) self.commands = [params.finalize()] else: # Ensure that the actual command depends on the input self.kwargs["IN_FILE_00"] = self.files[0] self.kwargs["IN_FILE_01"] = swap_ext(self.files[0], ".bai")
def build_sampling_nodes(options, genotyping, sample, regions, dependencies): fasta_file = regions["Genotypes"][sample] pileup_file = swap_ext(fasta_file, ".pileup.bgz") padding = genotyping["Padding"] slop, node = build_regions_nodes(regions, padding, dependencies) bam_file = "%s.%s.bam" % (sample, regions["Prefix"]) bam_file = os.path.join(options.samples_root, bam_file) if regions["Realigned"]: bam_file = add_postfix(bam_file, ".realigned") bai_node = build_bam_index_node(bam_file) genotype = GenotypeRegionsNode.customize( pileup_only=True, reference=regions["FASTA"], bedfile=slop, infile=bam_file, outfile=pileup_file, nbatches=options.samtools_max_threads, dependencies=node + (bai_node, )) apply_samtools_options(genotype.command, genotyping["MPileup"], "--mpileup-argument") genotype = genotype.build_node() tabix = TabixIndexNode(infile=pileup_file, preset="pileup", dependencies=genotype) builder = SampleRegionsNode(infile=pileup_file, bedfile=regions["BED"], outfile=fasta_file, dependencies=tabix) faidx = FastaIndexNode(infile=fasta_file, dependencies=builder) return (faidx, )
def build_sampling_nodes(options, genotyping, sample, regions, dependencies): fasta_file = regions["Genotypes"][sample] pileup_file = swap_ext(fasta_file, ".pileup.bgz") padding = genotyping["Padding"] slop, node = build_regions_nodes(regions, padding, dependencies) bam_file = "%s.%s.bam" % (sample, regions["Prefix"]) bam_file = os.path.join(options.samples_root, bam_file) if regions["Realigned"]: bam_file = add_postfix(bam_file, ".realigned") bai_node = build_bam_index_node(bam_file) genotype = GenotypeRegionsNode.customize(pileup_only=True, reference=regions["FASTA"], bedfile=slop, infile=bam_file, outfile=pileup_file, nbatches=options.samtools_max_threads, dependencies=node + (bai_node,)) apply_samtools_options(genotype.command, genotyping["MPileup"], "--mpileup-argument") genotype = genotype.build_node() tabix = TabixIndexNode(infile=pileup_file, preset="pileup", dependencies=genotype) builder = SampleRegionsNode(infile=pileup_file, bedfile=regions["BED"], outfile=fasta_file, dependencies=tabix) faidx = FastaIndexNode(infile=fasta_file, dependencies=builder) return (faidx,)
def parse_arguments(argv, ext): parser = argparse.ArgumentParser(prog="paleomix %s" % (ext.strip("."), )) parser.add_argument("infile", metavar="BAM", help="Filename of a sorted BAM file. If set to '-' " "the file is read from STDIN.") parser.add_argument("outfile", metavar="OUTPUT", nargs='?', help="Filename of output table; defaults to name of " "the input BAM with a '%s' extension. If " "set to '-' the table is printed to STDOUT." % (ext, )) parser.add_argument("--target-name", default=None, help="Name used for 'Target' column; defaults to the " "filename of the BAM file.") parser.add_argument("--regions-file", default=None, dest="regions_fpath", help="BED file containing regions of interest; %s " "is calculated only for these grouping by the " "name used in the BED file, or the contig name " "if no name has been specified for a record." % (ext.strip("."), )) parser.add_argument('--max-contigs', default=100, type=int, help="The maximum number of contigs allowed in a BAM " "file. If this number is exceeded, the entire " "set of contigs is aggregated into one pseudo-" "contig named '<Genome>'. This is done to " "limit table sizes [default: %(default)s]") parser.add_argument('--ignore-readgroups', default=False, action="store_true", help="Ignore readgroup information in reads, and only " "provide aggregated statistics; this is required " "if readgroup information is missing or partial " "[default: %(default)s]") parser.add_argument('--overwrite-output', default=False, action="store_true", help="Overwrite output file if it it exists; by " "default, the script will terminate if the file " "already exists.") args = parser.parse_args(argv) if not args.outfile: args.outfile = swap_ext(args.infile, ext) if args.ignore_readgroups: args.get_readgroup_func = _get_readgroup_ignored else: args.get_readgroup_func = _get_readgroup if not args.target_name: if args.infile == "-": args.target_name = "<STDIN>" else: args.target_name = os.path.basename(args.infile) if os.path.exists(args.outfile) and not args.overwrite_output: parser.error("Destination filename already exists (%r); use option " "--overwrite-output to allow overwriting of this file." % (args.outfile, )) return args
def test_swap_ext__empty_ext_vs_empty_ext(): assert_equal(swap_ext("name", ""), "name")
def test_swap_ext__dot_ext_vs_dot_ext(): assert_equal(swap_ext("name.", "."), "name")
def test_swap_ext__multiple__has_ext_vs_empty_ext(): assert_equal(swap_ext("name.foo.bar", ""), "name.foo")
def build_genotyping_nodes_cached(options, genotyping, sample, regions, dependencies): """Carries out genotyping, filtering of calls, and indexing of files for a given sample and prefix. If the option 'GenotypeEntirePrefix' is enabled, the BAM is genotyped once, and each set of RegionsOfInterest simply extract the relevant regions during construction of the consensus sequence. Parameters: options: An options object (c.f. pypeline.tools.phylo_pipeline.config). genotyping: Genotyping options defined for a specific set of areas of interest, corresponding to Genotyping:NAME in the makefile. sample: The name of the sample to be genotyped. egions: A dictionary for a 'RegionsOfInterest' from the makefile. dependencies: Depenencies that must be met before genotyping starts. Returns a tuple containing the filename of the filtered and tabix-indexed VCF file, and the top-level node generating this file. Multiple calls for the same BAM and prefix will return the same VCF and nodes if the option for 'GenotypeEntirePrefix' is enabled, otherwise each ROI is genotyped individiually. Output files are generated in ./results/PROJECT/genotyping. If the option for 'GenotypeEntirePrefix' is enabled, the following files are generated: SAMPLE.PREFIX.vcf.bgz: Unfiltered calls for variant/non-variant sites. SAMPLE.PREFIX.vcf.pileup.bgz: Pileup of sites containing SNPs. SAMPLE.PREFIX.vcf.pileup.bgz.tbi: Tabix index of the pileup. SAMPLE.PREFIX.filtered.vcf.bgz: Variant calls filtered with vcf_filter. SAMPLE.PREFIX.filtered.vcf.bgz.tbi: Tabix index for the filtered VCF. If 'GenotypeEntirePrefix' is not enabled for a given ROI, the following files are generated for that ROI (see descriptions above): SAMPLE.PREFIX.ROI.filtered.vcf.bgz SAMPLE.PREFIX.ROI.filtered.vcf.bgz.tbi SAMPLE.PREFIX.ROI.vcf.bgz SAMPLE.PREFIX.ROI.vcf.pileup.bgz SAMPLE.PREFIX.ROI.vcf.pileup.bgz.tbi In addition, the following files are generated for each set of RegionsOfInterest (ROI), regardless of the 'GenotypeEntirePrefix' option: SAMPLE.PREFIX.ROI.CDS.fasta: FASTA sequence of each feature in the ROI. SAMPLE.PREFIX.ROI.CDS.fasta.fai: FASTA index generated using SAMTools. """ output_prefix, bamfile, bedfile, dependencies \ = build_genotyping_bedfile_nodes(options, genotyping, sample, regions, dependencies) if (bamfile, output_prefix) in _VCF_CACHE: return _VCF_CACHE[(bamfile, output_prefix)] calls = swap_ext(output_prefix, ".vcf.bgz") pileups = swap_ext(output_prefix, ".vcf.pileup.bgz") filtered = swap_ext(output_prefix, ".filtered.vcf.bgz") # 1. Call samtools mpilup | bcftools view on the bam genotype = GenotypeRegionsNode.customize(reference=regions["FASTA"], bedfile=bedfile, infile=bamfile, outfile=calls, nbatches=options.samtools_max_threads, dependencies=dependencies) genotype.command.add_option("--mpileup-argument", "-f=%s" % (regions["FASTA"],), sep="=") apply_samtools_options(genotype.command, genotyping["MPileup"], "--mpileup-argument") apply_samtools_options(genotype.command, genotyping["BCFTools"], "--bcftools-argument") genotype = genotype.build_node() # 2. Collect pileups of sites with SNPs, to allow proper filtering by # frequency of the minor allele, as only the major non-ref allele is # counted in the VCF (c.f. field DP4). vcfpileup = VCFPileupNode.customize(reference=regions["FASTA"], infile_bam=bamfile, infile_vcf=calls, outfile=pileups, dependencies=genotype) apply_samtools_options(vcfpileup.command, genotyping["MPileup"], "--mpileup-argument") vcfpileup = vcfpileup.build_node() vcf_tabix = TabixIndexNode(infile=pileups, preset="pileup", dependencies=vcfpileup) # 3. Filter all sites using the 'vcf_filter' command vcffilter = VCFFilterNode.customize(infile=calls, pileup=pileups, outfile=filtered, regions=regions, dependencies=vcf_tabix) vcffilter = _apply_vcf_filter_options(vcffilter, genotyping, sample) # 4. Tabix index. This allows random-access to the VCF file when building # the consensus FASTA sequence later in the pipeline. tabix = TabixIndexNode(infile=filtered, preset="vcf", dependencies=vcffilter) _VCF_CACHE[(bamfile, output_prefix)] = (filtered, tabix) return filtered, tabix
def _setup(self, _config, temp): bam_filename = os.path.abspath(self._input_file) temp_filename = reroot_path(temp, bam_filename) os.symlink(bam_filename, temp_filename) os.symlink(swap_ext(bam_filename, ".bai"), temp_filename + ".bai")
def test_swap_ext__has_ext_vs_dot_ext(): assert_equal(swap_ext("name.foo", "."), "name")