def getProbeFragments(probe_bed, digest_bed, outfile, lookup_out): # First find the length of the restriction enzyme cut, required to obtain the start and end coordinates # from the pregenerated file. # First iteration, no comparison first_iteration = True length_RE_cut = 0 last_bed = None for bed_digest in Bed.iterator(IOTools.openFile(digest_bed)): if(first_iteration): first_iteration = False else: # If they are in the same contig they can be compared if(bed_digest.contig == last_bed.contig): length_RE_cut = bed_digest.start - last_bed.end break last_bed = bed_digest digest_fragments = pysam.TabixFile(digest_bed) bed = Bed.Bed() with IOTools.openFile(outfile, "w") as outf, \ IOTools.openFile(lookup_out,"w") as lookup: lookup.write("probe\tfragment\n") for probe in Bed.iterator(IOTools.openFile(probe_bed)): frag = digest_fragments.fetch(probe.contig, probe.start, probe.end, parser=pysam.asBed()) frag = list(frag) if not len(frag) == 1: E.warn("%i fragments found for probe %s, skipping" % (len(frag), probe.name)) continue frag = frag[0] # The restriction enzyme cut on the left side of the fragment # is the end site of the last restriction enzyme fragment + 1 # (+1 because according to the manual coordinates are specified # in 1-origin for the bed start.) bed.start = frag.start-length_RE_cut+1 bed.end = frag.end+length_RE_cut bed.contig = frag.contig bed["name"] = probe.name bed["score"] = "." bed["strand"] = "+" lookup.write("%s\t%s\n" % (probe.name, frag.name)) outf.write(str(bed) + "\n")
def exportSequencesFromBedFile( infile, outfile, masker = None, mode = "intervals" ): '''export sequences for intervals in :term:`bed`-formatted *infile* to :term:`fasta` formatted *outfile* ''' track = P.snip( infile, ".bed.gz" ) fasta = IndexedFasta.IndexedFasta( os.path.join( PARAMS["genome_dir"], PARAMS["genome"] ) ) outs = IOTools.openFile( outfile, "w") ids, seqs = [], [] for bed in Bed.setName(Bed.iterator( IOTools.openFile(infile) )): lcontig = fasta.getLength( bed.contig ) if mode == "intervals": seqs.append( fasta.getSequence( bed.contig, "+", bed.start, bed.end) ) ids.append( "%s_%s %s:%i..%i" % (track, bed.name, bed.contig, bed.start, bed.end) ) elif mode == "leftright": l = bed.end - bed.start start, end = max(0,bed.start-l), bed.end-l ids.append( "%s_%s_l %s:%i..%i" % (track, bed.name, bed.contig, start, end) ) seqs.append( fasta.getSequence( bed.contig, "+", start, end) ) start, end = bed.start+l, min(lcontig,bed.end+l) ids.append( "%s_%s_r %s:%i..%i" % (track, bed.name, bed.contig, start, end) ) seqs.append( fasta.getSequence( bed.contig, "+", start, end) ) masked = maskSequences( seqs, masker ) outs.write("\n".join( [ ">%s\n%s" % (x,y) for x,y in zip(ids, masked) ] ) ) outs.close()
def fetchProbeFragments(probe_bed, digest_bed, outfile, lookup_out): digest_fragments = pysam.TabixFile(digest_bed) bed = Bed.Bed() with IOTools.openFile(outfile, "w") as outf, \ IOTools.openFile(lookup_out,"w") as lookup: lookup.write("probe\tfragment\n") for probe in Bed.iterator(IOTools.openFile(probe_bed)): frag = digest_fragments.fetch(probe.contig, probe.start, probe.end, parser=pysam.asBed()) frag = list(frag) if not len(frag) == 1: E.warn("%i fragments found for probe %s, skipping" % (len(frag), probe.name)) continue frag = frag[0] bed.start = frag.start bed.end = frag.end bed.contig = frag.contig bed["name"] = probe.name bed["score"] = "." bed["strand"] = "+" lookup.write("%s\t%s\n" % (probe.name, frag.name)) outf.write(str(bed) + "\n")
def annotateCpGIslands( infiles, outfile ): '''annotate transcript by absence/presence of CpG islands ''' cpgfile, tssfile = infiles cpg = Bed.readAndIndex( IOTools.openFile( cpgfile ) ) extension_upstream = PARAMS["cpg_search_upstream"] extension_downstream = PARAMS["cpg_search_downstream"] c = E.Counter() outf = IOTools.openFile( outfile, "w" ) outf.write("transcript_id\tstrand\tstart\tend\trelative_start\trelative_end\n" ) for tss in Bed.iterator(IOTools.openFile( tssfile ) ): c.tss_total += 1 if tss.strand == "+": start, end = tss.start - extension_upstream, tss.start + extension_downstream else: start, end = tss.end - extension_downstream, tss.end + extension_upstream try: matches = list(cpg[tss.contig].find( start, end )) except KeyError: c.promotor_without_matches += 1 continue if len(matches) == 0: c.promotor_without_matches += 1 continue c.promotor_output += 1 for match in matches: c.matches_total += 1 genome_start, genome_end, x = match l = genome_end - genome_start # get relative location of match if tss.strand == "+": relative_start = genome_start - tss.start else: relative_start = tss.end - genome_end relative_end = relative_start + l outf.write( "\t".join( map(str, ( tss.name, tss.strand, genome_start, genome_end, relative_start, relative_end ))) + "\n" ) c.matches_output += 1 outf.close() with IOTools.openFile( outfile + ".summary", "w" ) as outf: outf.write ("category\tcounts\n" ) outf.write( c.asTable() + "\n" ) E.info( c )
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if not argv: argv = sys.argv # setup command line parser parser = E.OptionParser( version="%prog version: $Id: bed2graph.py 2861 2010-02-23 17:36:32Z andreas $", usage=globals()["__doc__"]) parser.add_option("-o", "--output-section", dest="output", type="choice", choices=("full", "name"), help="output either ``full`` overlapping entries, only the ``name``s. [default=%default].") parser.set_defaults( output="full", ) # add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv) if len(args) != 2: raise ValueError("two arguments required") if args[0] == "-": infile1 = options.stdin else: infile1 = IOTools.openFile(args[0], "r") infile2 = IOTools.openFile(args[1], "r") idx = Bed.readAndIndex(infile2, with_values=True) output = options.output outfile = options.stdout if output == "name": outfile.write("name1\tname2\n") outf = lambda x: x.fields[0] else: outf = str for bed in Bed.iterator(infile1): try: overlaps = idx[bed.contig].find(bed.start, bed.end) except (KeyError, IndexError): # ignore missing contig and zero length intervals continue for o in overlaps: outfile.write("\t".join((outf(bed), outf(o[2]))) + "\n") E.Stop()
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if not argv: argv = sys.argv # setup command line parser parser = E.OptionParser( version="%prog version: $Id: bed2graph.py 2861 2010-02-23 17:36:32Z andreas $", usage=globals()["__doc__"]) parser.add_option("-o", "--output", dest="output", type="choice", choices=("full", "name"), help="output either ``full`` overlapping entries, only the ``name``s. [default=%default].") parser.set_defaults( output="full", ) # add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv) if len(args) != 2: raise ValueError("two arguments required") if args[0] == "-": infile1 = options.stdin else: infile1 = IOTools.openFile(args[0], "r") infile2 = IOTools.openFile(args[1], "r") idx = Bed.readAndIndex(infile2, with_values=True) output = options.output outfile = options.stdout if output == "name": outfile.write("name1\tname2\n") outf = lambda x: x.fields[0] else: outf = str for bed in Bed.iterator(infile1): try: overlaps = idx[bed.contig].find(bed.start, bed.end) except (KeyError, IndexError): # ignore missing contig and zero length intervals continue for o in overlaps: outfile.write("\t".join((outf(bed), outf(o[2]))) + "\n") E.Stop()
def main( argv = None ): if argv == None: argv = sys.argv parser = E.OptionParser( version = "%prog version: $Id: gtf2table.py 2888 2010-04-07 08:48:36Z andreas $", usage = globals()["__doc__"]) parser.add_option("-g", "--genome-file", dest="genome_file", type="string", help="filename with genome [default=%default]." ) parser.add_option("-n", "--per-name", dest="per_name", action="store_true", help="compute counts per name [default=%default]." ) parser.add_option("-c", "--per-contig", dest="per_contig", action="store_true", help="compute counts per contig [default=%default]." ) parser.add_option("-t", "--per-track", dest="per_track", action="store_true", help="compute counts per track [default=%default]." ) parser.set_defaults( genome_file = None, per_name = False, per_track = False, ) (options, args) = E.Start( parser, argv ) # get files if options.genome_file: fasta = IndexedFasta.IndexedFasta( options.genome_file ) else: fasta = None counts = collections.defaultdict( Counter ) if options.per_track: keyf = lambda x: x.track elif options.per_name: keyf = lambda x: x.name elif options.per_contig: keyf = lambda x: x.contig else: keyf = lambda x: "all" for bed in Bed.iterator(options.stdin): counts[keyf(bed)].add( bed ) outf = options.stdout key = "track" outf.write( "%s\t%s\n" % (key, "\t".join( Counter.headers) )) for key, count in counts.iteritems(): outf.write( "%s\t%s\n" % ( key, str(count)) ) E.Stop()
def quantifyAnomolies(bam, probes, outfile): bamfile = pysam.AlignmentFile(bam) results = dict() mapped = bamfile.mapped total = 0 seeks = 0 for probe in Bed.iterator(IOTools.openFile(probes)): c = collections.Counter() for read in bamfile.fetch(probe.contig, probe.start, probe.end, multiple_iterators=True): if read.is_unmapped: continue c["total"] += 1 if not (read.is_secondary or read.is_supplementary): c["primary"] += 1 else: continue if read.pos < (probe.start-4) or read.aend > (probe.end +4): c["undigested"] += 1 if read.is_read1: c["read1"] += 1 if not read.mate_is_unmapped: c["paired"] += 1 if read.is_read1 and ( read.mpos >= probe.start and read.mpos <= probe.end): c["self_lig"] += 1 if (total + c["total"] % 10000) == 0: E.debug("%s/%s done" % (total + c["total"], mapped)) E.debug("%s processed, %i found" % (probe.name, c["total"])) results[probe.name] = c total += c["total"] headers = ["Probe", "total", "primary", "undigested", "read1", "paired", "self_lig"] with IOTools.openFile(outfile, "w") as outf: outf.write("\t".join(headers) + "\n") for probe in results: outf.write("\t".join( [probe]+[str(results[probe][col]) for col in headers[1:]]) + "\n")
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if argv is None: argv = sys.argv # setup command line parser parser = E.OptionParser(version="%prog version: $Id$", usage=globals()["__doc__"]) parser.add_option("-t", "--test", dest="test", type="string", help="supply help") parser.add_option("--regex-filename", dest="re_name", type="string", help="regex for filename component to be used " "as the annotation label") parser.set_defaults( re_name="(.+)", ) (options, args) = E.Start(parser, argv=argv) infiles = argv[-1] bed_files = infiles.split(",") if len(bed_files) == 1: raise IOError("Only one file detected, cannot merge " "a single bed file") else: pass # get the regex for annotation names, rx = re.compile(options.re_name) annot_names = [[y for y in rx.search(x).groups()] for x in bed_files] annot_names = list(itertools.chain(*annot_names)) # output as BED4 format: chr, start, end, name for fx in range(len(bed_files)): bfile = bed_files[fx] with IOTools.openFile(bfile, "r") as ofile: intervals = Bed.iterator(ofile) track_name = [bx for bx in annot_names if re.search(bx, bfile)][0] for entry in intervals: entry.__setitem__("name", track_name) options.stdout.write("%s\t%s\t%s\t%s\n" % (entry.contig, entry.start, entry.end, entry.name)) # write footer and output benchmark information. E.Stop()
def countTagsInClusters(bedfile, bamfile, outfile): bam = pysam.AlignmentFile(bamfile) outlines = [] for bed in Bed.iterator(IOTools.openFile(bedfile)): interval = (bed.start, bed.end) counts = iCLIP.count_intervals(bam, [interval], bed.contig).sum() outlines.append(["%s:%i-%i" % (bed.contig, bed.start, bed.end), str(counts)]) IOTools.writeLines(outfile, outlines, header=["position","count"])
def countTagsInClusters(bedfile, bamfile, outfile): bam = pysam.AlignmentFile(bamfile) outlines = [] for bed in Bed.iterator(IOTools.openFile(bedfile)): interval = (bed.start, bed.end) counts = iCLIP.count_intervals(bam, [interval], bed.contig).sum() outlines.append( ["%s:%i-%i" % (bed.contig, bed.start, bed.end), str(counts)]) IOTools.writeLines(outfile, outlines, header=["position", "count"])
def buildQuicksectMask(bed_file): '''return Quicksect object containing the regions specified takes a bed file listing the regions to mask ''' mask = IndexedGenome.Quicksect() n_regions = 0 for bed in Bed.iterator(IOTools.openFile(bed_file)): # it is neccessary to extend the region to make an accurate mask mask.add(bed.contig, (bed.start - 1), (bed.end + 1), 1) n_regions += 1 E.info("Built Quicksect mask for %i regions" % n_regions) return(mask)
def buildAlignmentSizes(infiles, outfile): ''' use bed files to sum the total number of bases that are aligned to the genomes ''' outf = open(outfile, "w") outf.write("genome\tsize\n") for infile in infiles: genome = P.snip(os.path.basename(infile), ".bed.gz") c = 0 inf = IOTools.openFile(infile) for bed in Bed.iterator(inf): c += bed.end - bed.start outf.write("%s\t%s\n" % (genome, str(c))) outf.close()
def sites2fragments(infile, genomefile, outfile): '''Convert bedfile of deigestion sites into bedfile of fragments''' contig_lengths = { line.split()[0]: int(line.split()[1][:-1]) for line in IOTools.openFile(genomefile) } last_end = 0 last_contig = None name = 0 new_bed = Bed.Bed() new_bed["strand"] = "+" new_bed["score"] = "." with IOTools.openFile(outfile, "w") as outf: for bed in Bed.iterator(IOTools.openFile(infile)): if last_contig is not None and not bed.contig == last_contig: name += 1 new_bed.start = last_end new_bed.contig = last_contig new_bed.end = contig_lengths[bed.contig] new_bed["name"] = str(name) outf.write(str(new_bed) + "\n") last_end = 0 last_contig = bed.contig new_bed.contig = last_contig new_bed.start = last_end new_bed.end = bed.start name += 1 new_bed["name"] = str(name) outf.write(str(new_bed) + "\n") last_end = bed.end name += 1 new_bed.start = last_end new_bed.contig = last_contig new_bed.end = contig_lengths[bed.contig] new_bed["name"] = str(name) outf.write(str(new_bed) + "\n") pysam.tabix_index(outfile, force=True, preset="bed")
def sites2fragments(infile, genomefile, outfile): '''Convert bedfile of deigestion sites into bedfile of fragments''' contig_lengths = {line.split()[0]: int(line.split()[1][:-1]) for line in IOTools.openFile(genomefile)} last_end = 0 last_contig = None name = 0 new_bed = Bed.Bed() new_bed["strand"] = "+" new_bed["score"] = "." with IOTools.openFile(outfile, "w") as outf: for bed in Bed.iterator(IOTools.openFile(infile)): if last_contig is not None and not bed.contig == last_contig: name += 1 new_bed.start = last_end new_bed.contig = last_contig new_bed.end = contig_lengths[bed.contig] new_bed["name"] = str(name) outf.write(str(new_bed) + "\n") last_end = 0 last_contig = bed.contig new_bed.contig = last_contig new_bed.start = last_end new_bed.end = bed.start name += 1 new_bed["name"] = str(name) outf.write(str(new_bed) + "\n") last_end = bed.end name += 1 new_bed.start = last_end new_bed.contig = last_contig new_bed.end = contig_lengths[bed.contig] new_bed["name"] = str(name) outf.write(str(new_bed) + "\n") pysam.tabix_index(outfile, force=True, preset="bed")
def makeIntervalCorrelation(infiles, outfile, field, reference): '''compute correlation of interval properties between sets ''' dbhandle = sqlite3.connect(PARAMS["database_name"]) tracks, idx = [], [] for infile in infiles: track = P.snip(infile, ".bed.gz") tablename = "%s_intervals" % P.tablequote(track) cc = dbhandle.cursor() statement = "SELECT contig, start, end, %(field)s FROM %(tablename)s" % locals( ) cc.execute(statement) ix = IndexedGenome.IndexedGenome() for contig, start, end, peakval in cc: ix.add(contig, start, end, peakval) idx.append(ix) tracks.append(track) outs = IOTools.openFile(outfile, "w") outs.write("contig\tstart\tend\tid\t" + "\t".join(tracks) + "\n") for bed in Bed.iterator(infile=IOTools.openFile(reference, "r")): row = [] for ix in idx: try: intervals = list(ix.get(bed.contig, bed.start, bed.end)) except KeyError: row.append("") continue if len(intervals) == 0: peakval = "" else: peakval = str((max([x[2] for x in intervals]))) row.append(peakval) outs.write(str(bed) + "\t" + "\t".join(row) + "\n") outs.close()
def exportSequencesFromBedFile(infile, outfile, masker=None, mode="intervals"): '''export sequences for intervals in :term:`bed`-formatted *infile* to :term:`fasta` formatted *outfile* ''' track = P.snip(infile, ".bed.gz") fasta = IndexedFasta.IndexedFasta( os.path.join(P.get_params()["genome_dir"], P.get_params()["genome"])) outs = IOTools.open_file(outfile, "w") ids, seqs = [], [] for bed in Bed.setName(Bed.iterator(IOTools.open_file(infile))): lcontig = fasta.getLength(bed.contig) if mode == "intervals": seqs.append(fasta.getSequence(bed.contig, "+", bed.start, bed.end)) ids.append("%s_%s %s:%i..%i" % (track, bed.name, bed.contig, bed.start, bed.end)) elif mode == "leftright": l = bed.end - bed.start start, end = max(0, bed.start - l), bed.end - l ids.append("%s_%s_l %s:%i..%i" % (track, bed.name, bed.contig, start, end)) seqs.append(fasta.getSequence(bed.contig, "+", start, end)) start, end = bed.start + l, min(lcontig, bed.end + l) ids.append("%s_%s_r %s:%i..%i" % (track, bed.name, bed.contig, start, end)) seqs.append(fasta.getSequence(bed.contig, "+", start, end)) masked = maskSequences(seqs, masker) outs.write("\n".join([">%s\n%s" % (x, y) for x, y in zip(ids, masked)])) outs.close()
def formatProbeFragments(probe_fragments_bed, outfile): # Create strings for output collisions probe_collisions = "" exclus_collisions = "" with IOTools.openFile(outfile, "w") as outf: # dictionaries for collisions per chr chr_probe = {} chr_exclus = {} for bed_digest in Bed.iterator(IOTools.openFile(probe_fragments_bed)): # chromosome needs to specify only number (remove chr) chromosome = re.sub('chr', '', bed_digest.contig) out_array = [] out_array.append(formatNameCompliance(bed_digest["name"])) out_array.append(str(chromosome)) out_array.append(str(bed_digest.start)) out_array.append(str(bed_digest.end)) out_array.append(str(chromosome)) out_array.append(str(bed_digest.start-1000)) out_array.append(str(bed_digest.end+1000)) out_array.append("1") out_array.append("A") outf.write("\t".join(out_array) + "\n") # Calculate collisions, all coordinates assumed in BED format # It needs to be done per chr # chr_probe -> probe_ranges -> probe_range # chr_exclus -> exclus_ranges -> exclus_range # Check if chromosome key already exists in one of the # dictionaries # (ranges introduced before on that chromosome if bed_digest.contig in chr_probe: # If it exists, get the array of probe ranges # and exclusion ranges already stored probe_ranges = chr_probe[bed_digest.contig] exclus_ranges = chr_exclus[bed_digest.contig] # If it doesn't create the arrays else: probe_ranges = [] exclus_ranges = [] # Create a range for probes and exclusion fragments probe_range = [] exclus_range = [] probe_range.append(bed_digest.start) probe_range.append(bed_digest.end) probe_ranges.append(probe_range) exclus_range.append(bed_digest.start-1000) exclus_range.append(bed_digest.end+1000) exclus_ranges.append(exclus_range) # Substitute the ranges back to the corresponding chr in the # dictionary chr_probe[bed_digest.contig] = probe_ranges chr_exclus[bed_digest.contig] = exclus_ranges for chr in chr_probe: probe_ranges = chr_probe[chr] exclus_ranges = chr_exclus[chr] probe_intersection = set.intersection(*(set(range(start, finish)) for start, finish in probe_ranges)) exclus_intersection = set.intersection(*(set(range(start, finish)) for start, finish in exclus_ranges)) if(len(probe_intersection) != 0): probe_collisions += "Probe collision " +str(chr) probe_collisions += " " +str(min(probe_intersection)) probe_collisions += " " +str(max(probe_intersection)) probe_collisions += "\n" if(len(exclus_intersection) != 0): exclus_collisions += "Exclusion collision " +str(chr) exclus_collisions += " " +str(min(exclus_intersection)) exclus_collisions += " " +str(max(exclus_intersection)) exclus_collisions += "\n" return (probe_collisions, exclus_collisions)
def main(argv=None): if argv == None: argv = sys.argv parser = E.OptionParser( version= "%prog version: $Id: gff2fasta.py 2861 2010-02-23 17:36:32Z andreas $") parser.add_option("-g", "--genome-file", dest="genome_file", type="string", help="filename with genome.") parser.add_option("-m", "--masker", dest="masker", type="choice", choices=("dust", "dustmasker", "softmask", "none"), help="apply masker [%default].") parser.add_option("-o", "--mode", dest="mode", type="choice", choices=("intervals", "leftright"), help="what to output [%default]") parser.add_option("--min-length", dest="min_length", type="int", help="require a minimum sequence length [%default]") parser.add_option("--max-length", dest="max_length", type="int", help="require a maximum sequence length [%default]") parser.add_option( "--extend-at", dest="extend_at", type="choice", choices=("none", "3", "5", "both", "3only", "5only"), help= "extend at no, 3', 5' or both ends. If 3only or 5only are set, only the added sequence is returned [default=%default]" ) parser.add_option("--extend-by", dest="extend_by", type="int", help="extend by # bases [default=%default]") parser.add_option( "--use-strand", dest="ignore_strand", action="store_false", help= "use strand information and return reverse complement [default=%default]" ) parser.set_defaults( genome_file=None, masker=None, mode="intervals", min_length=0, max_length=0, extend_at=None, extend_by=100, ignore_strand=True, ) (options, args) = E.Start(parser) if options.genome_file: fasta = IndexedFasta.IndexedFasta(options.genome_file) contigs = fasta.getContigSizes() fasta.setConverter(IndexedFasta.getConverter("zero-both-open")) counter = E.Counter() ids, seqs = [], [] E.info("collecting sequences") for bed in Bed.setName(Bed.iterator(options.stdin)): counter.input += 1 lcontig = fasta.getLength(bed.contig) if options.ignore_strand: strand = "+" else: strand = bed.strand if options.mode == "intervals": ids.append("%s %s:%i..%i (%s)" % (bed.name, bed.contig, bed.start, bed.end, strand)) seqs.append( fasta.getSequence(bed.contig, strand, bed.start, bed.end)) elif options.mode == "leftright": l = bed.end - bed.start start, end = max(0, bed.start - l), bed.end - l ids.append("%s_l %s:%i..%i (%s)" % (bed.name, bed.contig, start, end, strand)) seqs.append(fasta.getSequence(bed.contig, strand, start, end)) start, end = bed.start + l, min(lcontig, bed.end + l) ids.append("%s_r %s:%i..%i (%s)" % (bed.name, bed.contig, start, end, strand)) seqs.append(fasta.getSequence(bed.contig, strand, start, end)) E.info("collected %i sequences" % len(seqs)) masked = Masker.maskSequences(seqs, options.masker) options.stdout.write( "\n".join([">%s\n%s" % (x, y) for x, y in zip(ids, masked)]) + "\n") E.info("masked %i sequences" % len(seqs)) counter.output = len(seqs) E.info("%s" % counter) E.Stop()
def loadIntervalsFromBed(bedfile, track, outfile, bamfiles, offsets): '''load intervals from :term:`bed` formatted files into database. Re-evaluate the intervals by counting reads within the interval. In contrast to the initial pipeline, the genome is not binned. In particular, the meaning of the columns in the table changes to: nProbes: number of reads in interval PeakCenter: position with maximum number of reads in interval AvgVal: average coverage within interval ''' tmpfile = P.getTempFile(".") headers = ("AvgVal", "DisttoStart", "GeneList", "Length", "PeakCenter", "PeakVal", "Position", "interval_id", "nCpGs", "nGenes", "nPeaks", "nProbes", "nPromoters", "contig", "start", "end") tmpfile.write("\t".join(headers) + "\n") avgval, contig, disttostart, end, genelist, length, peakcenter, peakval, position, start, interval_id, ncpgs, ngenes, npeaks, nprobes, npromoters = \ 0, "", 0, 0, "", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, mlength = int(PARAMS["calling_merge_min_interval_length"]) c = E.Counter() # count tags for bed in Bed.iterator(IOTools.openFile(infile, "r")): c.input += 1 if "name" not in bed: bed.name = c.input # remove very short intervals if bed.end - bed.start < mlength: c.skipped_length += 1 continue if replicates: npeaks, peakcenter, length, avgval, peakval, nprobes = \ PipelineChipseq.countPeaks( bed.contig, bed.start, bed.end, samfiles, offsets) # nreads can be 0 if the intervals overlap only slightly # and due to the binning, no reads are actually in the # overlap region. However, most of these intervals should # be small and have already be deleted via the # merge_min_interval_length cutoff. do not output # intervals without reads. if nprobes == 0: c.skipped_reads += 1 else: npeaks, peakcenter, length, avgval, peakval, nprobes = ( 1, bed.start + (bed.end - bed.start) // 2, bed.end - bed.start, 1, 1, 1) c.output += 1 tmpfile.write("\t".join(map( str, (avgval, disttostart, genelist, length, peakcenter, peakval, position, bed.name, ncpgs, ngenes, npeaks, nprobes, npromoters, bed.contig, bed.start, bed.end))) + "\n") if c.output == 0: E.warn("%s - no intervals") tmpfile.close() tmpfilename = tmpfile.name tablename = "%s_intervals" % track.asTable() statement = ''' cgat csv2db %(csv2db_options)s --allow-empty-file --add-index=interval_id --table=%(tablename)s < %(tmpfilename)s > %(outfile)s ''' P.run() os.unlink(tmpfile.name) E.info("%s\n" % str(c))
def main(argv=sys.argv): parser = E.OptionParser(version="%prog version: $Id: bed2gff.py 2861 2010-02-23 17:36:32Z andreas $", usage=globals()["__doc__"]) parser.add_option("-a", "--as-gtf", dest="as_gtf", action="store_true", help="output as gtf.") parser.add_option("-f", "--id-format", dest="id_format", type="string", help="format for numeric identifier if --as-gtf is set and no name in bed file [%default].") parser.set_defaults(as_gtf=False, id_format="%08i", test=None) (options, args) = E.Start(parser, add_pipe_options=True) as_gtf = options.as_gtf id_format = options.id_format if as_gtf: gff = GTF.Entry() else: gff = GTF.Entry() gff.source = "bed" gff.feature = "exon" ninput, noutput, nskipped = 0, 0, 0 id = 0 for bed in Bed.iterator(options.stdin): ninput += 1 gff.contig = bed.contig gff.start = bed.start gff.end = bed.end if bed.fields and len(bed.fields) >= 3: gff.strand = bed.fields[2] else: gff.strand = "." if bed.fields and len(bed.fields) >= 2: gff.score = bed.fields[1] if as_gtf: if bed.fields: gff.gene_id = bed.fields[0] gff.transcript_id = bed.fields[0] else: id += 1 gff.gene_id = id_format % id gff.transcript_id = id_format % id else: if bed.fields: gff.source = bed.fields[0] options.stdout.write(str(gff) + "\n") noutput += 1 E.info("ninput=%i, noutput=%i, nskipped=%i" % (ninput, noutput, nskipped)) E.Stop()
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if not argv: argv = sys.argv # setup command line parser parser = E.OptionParser( version= "%prog version: $Id: bed2psl.py 2899 2010-04-13 14:37:37Z andreas $", usage=globals()["__doc__"]) parser.add_option("-q", "--query", dest="query", type="string", help="sequence to use for query [default=%default].") parser.add_option("-t", "--target", dest="target", type="string", help="sequence to use for target [default=%default].") parser.add_option("-g", "--genome-file", dest="genome_file", type="string", help="filename with genome.") parser.set_defaults( genome_file=None, query=None, target=None, ) ## add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv) ## do sth ninput, nskipped, noutput = 0, 0, 0 if options.genome_file: fasta = IndexedFasta.IndexedFasta(options.genome_file) else: fasta = None psl = Blat.Match() for bed in Bed.iterator(options.stdin): ninput += 1 start, end = bed.start, bed.end if "blockSizes" in bed: psl.mQueryId = bed["name"] blocksizes = [int(x) for x in bed["blockSizes"].split(",")[:-1]] sbjctblockstarts = [ int(x) + start for x in bed["blockStarts"].split(",")[:-1] ] strand = bed["strand"] else: psl.mQueryId = "%i" % ninput blocksizes = [end - start] sbjctblockstarts = [ start, ] strand = "+" psl.mSbjctId = bed.contig psl.mSbjctFrom, psl.mSbjctTo = start, end psl.mQueryFrom, psl.mQueryTo = 0, end - start psl.mBlockSizes = blocksizes psl.mNBlocks = len(blocksizes) psl.strand = strand q, qp = [], 0 for x in blocksizes: q.append(qp) qp += x psl.mQueryBlockStarts = q psl.mSbjctBlockStarts = sbjctblockstarts psl.mQueryLength = sum(psl.mBlockSizes) if fasta: psl.mSbjctLength = fasta.getLength(bed.contig) options.stdout.write("%s\n" % str(psl)) noutput += 1 E.info("ninput=%i, noutput=%i, nskipped=%i" % (ninput, noutput, nskipped)) ## write footer and output benchmark information. E.Stop()
def loadIntervals(infile, outfile): '''load intervals from :term:`bed` formatted files into the database. If a :term:`bam` file is associated with a :term:`bed` file, re-evaluate the intervals by counting reads within the interval. In contrast to the initial pipeline, the genome is not binned. nprobes: number of reads in interval peakcenter: position with maximum number of reads in interval avgval: average coverage within interval ''' tmpfile = P.getTempFile(".") headers = ("avgval", "disttostart", "genelist", "length", "peakcenter", "peakval", "position", "interval_id", "npeaks", "nprobes", "contig", "start", "end", "score", "strand") tmpfile.write("\t".join(headers) + "\n") (avgval, contig, disttostart, end, genelist, length, peakcenter, peakval, position, start, interval_id, npeaks, nprobes) = \ 0, "", 0, 0, "", 0, 0, 0, 0, 0, 0, 0, 0 track = Sample(filename=P.snip(infile, ".bed.gz")) bamfiles, offsets = getAssociatedBAMFiles(track) if bamfiles: E.info("%s: associated bamfiles = %s" % (track, bamfiles)) else: E.info("%s: no bamfiles associated" % (track)) # open all bamfiles samfiles = [pysam.Samfile(fn, "rb") for fn in bamfiles] c = E.Counter() # count tags for bed in Bed.iterator(IOTools.openFile(infile, "r")): c.input += 1 if "name" not in bed: bed.name = c.input try: strand = bed["strand"] except IndexError: strand = "." # The fifth field of a bed file can be used to supply a # score. Our iterator returns the optional fields as a "fields # array". The first of these is the interval name, and the # second the score. The score may be more is better or less is # better. if len(bed.fields) > 1: value = bed.fields[1] if value != "": score = value else: score = 1 else: score = 1 if samfiles: npeaks, peakcenter, length, avgval, peakval, nprobes = \ PipelinePeakcalling.countPeaks( bed.contig, bed.start, bed.end, samfiles, offsets) if nprobes == 0: c.skipped_reads += 1 else: # deal with bed12 bed_intervals = bed.toIntervals() length = sum([e - s for s, e in bed_intervals]) mid_point = length / 2 for s, e in bed_intervals: peakcenter = s + mid_point if peakcenter >= e: mid_point = peakcenter - e else: break npeaks, avgval, peakval, nprobes = \ (1, 1, 1, 1) c.output += 1 tmpfile.write("\t".join( map(str, (avgval, disttostart, genelist, length, peakcenter, peakval, position, bed.name, npeaks, nprobes, bed.contig, bed.start, bed.end, score, strand))) + "\n") if c.output == 0: E.warn("%s - no aggregate intervals") tmpfile.close() P.load(tmpfile.name, outfile, tablename=os.path.basename("%s_intervals" % track.asTable()), options="--allow-empty-file " "--add-index=interval_id") os.unlink(tmpfile.name) E.info("%s\n" % str(c))
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if argv is None: argv = sys.argv parser = E.OptionParser( version="%prog version: $Id$", usage=globals()["__doc__"]) parser.add_option("-b", "--bin-size", dest="bin_size", type="string", help="bin size.") parser.add_option("--min-value", dest="min_value", type="float", help="minimum value for histogram.") parser.add_option( "--max-value", dest="max_value", type="float", help="maximum value for histogram.") parser.add_option( "--no-empty-bins", dest="no_empty_bins", action="store_true", help="do not display empty bins.") parser.add_option( "--with-empty-bins", dest="no_empty_bins", action="store_false", help="display empty bins.") parser.add_option( "--ignore-out-of-range", dest="ignore_out_of_range", action="store_true", help="ignore values that are out of range (as opposed to truncating " "them to range border.") parser.add_option("--missing-value", dest="missing_value", type="string", help="entry for missing values [%default].") parser.add_option("--use-dynamic-bins", dest="dynamic_bins", action="store_true", help="each value constitutes its own bin.") parser.add_option("--format", dest="format", type="choice", choices=("gff", "gtf", "bed"), help="input file format [%default].") parser.add_option("--method", dest="methods", type="choice", action="append", choices=("all", "hist", "stats", "overlaps", "values"), help="methods to apply [%default].") parser.add_option("--output-section", dest="output_section", type="choice", choices=("all", "size", "distance"), help="data to compute [%default].") parser.set_defaults( no_empty_bins=True, bin_size=None, dynamic_bins=False, ignore_out_of_range=False, min_value=None, max_value=None, nonull=None, missing_value="na", output_filename_pattern="%s", methods=[], output_section="all", format="gff", ) (options, args) = E.Start(parser, add_output_options=True) if "all" in options.methods: options.methods = ("hist", "stats", "overlaps") if not options.output_filename_pattern: options.output_filename_pattern = "%s" if len(options.methods) == 0: raise ValueError( "please provide counting method using --method option") if options.format in ("gff", "gtf"): gffs = GTF.iterator(options.stdin) elif options.format == "bed": gffs = Bed.iterator(options.stdin) values_between = [] values_within = [] values_overlaps = [] if "overlaps" in options.methods: if not options.output_filename_pattern: options.output_filename_pattern = "%s" outfile_overlaps = E.openOutputFile("overlaps") else: outfile_overlaps = None last = None ninput, noverlaps = 0, 0 for this in gffs: ninput += 1 values_within.append(this.end - this.start) if last and last.contig == this.contig: if this.start < last.end: noverlaps += 1 if outfile_overlaps: outfile_overlaps.write("%s\t%s\n" % (str(last), str(this))) values_overlaps.append( min(this.end, last.end) - max(last.start, this.start)) if this.end > last.end: last = this continue else: values_between.append(this.start - last.end) # if this.start - last.end < 10: # print str(last) # print str(this) # print "==" values_overlaps.append(0) last = this if "hist" in options.methods: outfile = E.openOutputFile("hist") h_within = Histogram.Calculate( values_within, no_empty_bins=options.no_empty_bins, increment=options.bin_size, min_value=options.min_value, max_value=options.max_value, dynamic_bins=options.dynamic_bins, ignore_out_of_range=options.ignore_out_of_range) h_between = Histogram.Calculate( values_between, no_empty_bins=options.no_empty_bins, increment=options.bin_size, min_value=options.min_value, max_value=options.max_value, dynamic_bins=options.dynamic_bins, ignore_out_of_range=options.ignore_out_of_range) if "all" == options.output_section: outfile.write("residues\tsize\tdistance\n") combined_histogram = Histogram.Combine( [h_within, h_between], missing_value=options.missing_value) Histogram.Write(outfile, combined_histogram, nonull=options.nonull) elif options.output_section == "size": outfile.write("residues\tsize\n") Histogram.Write(outfile, h_within, nonull=options.nonull) elif options.output_section == "distance": outfile.write("residues\tdistance\n") Histogram.Write(outfile, h_between, nonull=options.nonull) outfile.close() if "stats" in options.methods: outfile = E.openOutputFile("stats") outfile.write("data\t%s\n" % Stats.Summary().getHeader()) if options.output_section in ("size", "all"): outfile.write("size\t%s\n" % str(Stats.Summary(values_within))) if options.output_section in ("distance", "all"): outfile.write("distance\t%s\n" % str(Stats.Summary(values_between))) outfile.close() if "values" in options.methods: outfile = E.openOutputFile("distances") outfile.write("distance\n%s\n" % "\n".join(map(str, values_between))) outfile.close() outfile = E.openOutputFile("sizes") outfile.write("size\n%s\n" % "\n".join(map(str, values_within))) outfile.close() outfile = E.openOutputFile("overlaps") outfile.write("overlap\n%s\n" % "\n".join(map(str, values_overlaps))) outfile.close() E.info("ninput=%i, ndistance=%i, nsize=%i, noverlap=%i" % (ninput, len(values_between), len(values_within), noverlaps)) E.Stop()
def main(argv=None): if argv is None: argv = sys.argv parser = E.OptionParser(version="%prog version: $Id$", usage=globals()["__doc__"]) parser.add_option("-g", "--genome-file", dest="genome_file", type="string", help="filename with genome [default=%default].") parser.add_option("-a", "--aggregate-by", dest="aggregate", type="choice", choices=("name", "contig", "track", "none"), help="aggregate counts by feature [default=%default].") parser.add_option("-p", "--add-percent", dest="add_percent", action="store_true", help="add percentages [default=%default].") parser.set_defaults( genome_file=None, aggregate="none", add_percent=False, ) (options, args) = E.Start(parser, argv) # get files if options.genome_file: fasta = IndexedFasta.IndexedFasta(options.genome_file) else: if options.add_percent: raise ValueError("--add-percent option requires --genome-file") fasta = None if options.add_percent and not options.aggregate == "contig": raise NotImplementedError( "--add-percent option requires --aggregate=contig") counts = collections.defaultdict(Counter) total = Counter() output_totals = True if options.aggregate == "track": keyf = lambda x: x.track elif options.aggregate == "name": keyf = lambda x: x.name elif options.aggregate == "contig": keyf = lambda x: x.contig else: keyf = lambda x: "all" output_totals = False for bed in Bed.iterator(options.stdin): counts[keyf(bed)].add(bed) total.add(bed) outf = options.stdout key = "track" if options.add_percent: outf.write("%s\t%s\n" % (key, "\t".join(Counter.headers_percent))) else: outf.write("%s\t%s\n" % (key, "\t".join(Counter.headers))) total_bases = 0 for key, count in sorted(counts.items()): if options.add_percent: total_bases += fasta.getLength(key) count.setSize(fasta.getLength(key)) outf.write("%s\t%s\n" % (key, str(count))) if output_totals: if options.add_percent: count.setSize(total_bases) outf.write("%s\t%s\n" % ("total", str(total))) E.Stop()
def main(argv=None): if argv is None: argv = sys.argv parser = E.OptionParser(version="%prog version: $Id$", usage=globals()["__doc__"]) parser.add_option("-g", "--genome-file", dest="genome_file", type="string", help="filename with genome [default=%default].") parser.add_option( "-b", "--bam-file", dest="bam_files", type="string", help="filename with read mapping information. Multiple files can be " "submitted in a comma-separated list [default=%default].") parser.add_option( "--control-bam-file", dest="control_bam_files", type="string", help="filename with read mapping information for input/control. " "Multiple files can be submitted in a comma-separated list " "[default=%default].") parser.add_option("--filename-format", dest="filename_format", type="choice", choices=("bed", "gff", "gtf"), help="format of secondary stream [default=%default].") parser.add_option("-c", "--counter", dest="counters", type="choice", action="append", choices=("length", "overlap", "peaks", "composition-na", "composition-cpg", "classifier-chipseq", "motif"), help="select counters to apply [default=%default].") parser.add_option("--motif-sequence", dest="motif_sequence", type="string", help="specify a sequence to search for" "[default=%default].") parser.add_option( "-o", "--offset", dest="offsets", type="int", action="append", help="tag offsets for tag counting - supply as many as there " "are bam-files [default=%default].") parser.add_option( "--control-offset", dest="control_offsets", type="int", action="append", help="control tag offsets for tag counting - supply as many as " "there are bam-files [default=%default].") parser.add_option( "-a", "--output-all-fields", dest="all_fields", action="store_true", help="output all fields in original bed file, by default only " "the first 4 are output [default=%default].") parser.add_option( "--output-bed-headers", dest="bed_headers", type="string", help="supply ',' separated list of headers for bed component " "[default=%default].") parser.add_option( "-f", "--gff-file", dest="filename_gff", type="string", action="append", metavar='bed', help="filename with extra gff files. The order is important " "[default=%default].") parser.add_option( "--has-header", dest="has_header", action="store_true", help="bed file with headers. Headers and first columns are " "preserved [default=%default]") parser.set_defaults(genome_file=None, counters=[], bam_files=None, offsets=[], control_bam_files=None, control_offsets=[], all_fields=False, filename_format=None, bed_headers=None, filename_gff=[], has_header=False, motif_sequence=None) (options, args) = E.Start(parser) if options.bed_headers is not None: bed_headers = [x.strip() for x in options.bed_headers.split(",")] if len(bed_headers) < 3: raise ValueError("a bed file needs at least three columns") else: bed_headers = None if options.has_header: while 1: line = options.stdin.readline() if not line: E.warn("empty bed file with no header") E.Stop() return if not line.startswith("#"): break bed_headers = line[:-1].split("\t") if "motif" in options.counters and not options.motif_sequence: raise ValueError("if using motif must specify a motif-sequence") # get files if options.genome_file: fasta = IndexedFasta.IndexedFasta(options.genome_file) else: fasta = None if options.bam_files: bam_files = [] for bamfile in options.bam_files.split(","): bam_files.append(pysam.Samfile(bamfile, "rb")) else: bam_files = None if options.control_bam_files: control_bam_files = [] for bamfile in options.control_bam_files.split(","): control_bam_files.append(pysam.Samfile(bamfile, "rb")) else: control_bam_files = None counters = [] for c in options.counters: if c == "length": counters.append(CounterLength(fasta=fasta, options=options)) elif c == "overlap": counters.append( CounterOverlap(filename=options.filename_gff[0], fasta=fasta, options=options)) del options.filename_gff[0] elif c == "peaks": counters.append( CounterPeaks(bam_files, options.offsets, control_bam_files, options.control_offsets, options=options)) elif c == "composition-na": counters.append( CounterCompositionNucleotides(fasta=fasta, options=options)) elif c == "composition-cpg": counters.append(CounterCompositionCpG(fasta=fasta, options=options)) elif c == "classifier-chipseq": counters.append( ClassifierChIPSeq(filename_gff=options.filename_gff, fasta=fasta, options=options, prefix=None)) del options.filename_gff[0] elif c == "motif": counters.append( CounterMotif(fasta=fasta, motif=options.motif_sequence)) extra_fields = None for bed in Bed.iterator(options.stdin): if extra_fields is None: # output explicitely given headers if bed_headers: if len(bed_headers) > bed.columns: raise ValueError( "insufficient columns (%i, expected %i) in %s" % (bed.columns, len(bed_headers), str(bed))) else: bed_headers = Bed.Headers[:bed.columns] options.stdout.write("\t".join(bed_headers)) options.stdout.write("\t" + "\t".join([x.getHeader() for x in counters]) + "\n") extra_fields = list(range(len(bed_headers) - 3)) for counter in counters: counter.update(bed) if options.all_fields: options.stdout.write(str(bed)) else: options.stdout.write("\t".join( [bed.contig, str(bed.start), str(bed.end)] + [bed.fields[x] for x in extra_fields])) for counter in counters: options.stdout.write("\t%s" % str(counter)) options.stdout.write("\n") E.Stop()
def annotateCpGIslands(infiles, outfile): '''annotate transcript by absence/presence of CpG islands ''' cpgfile, tssfile = infiles cpg = Bed.readAndIndex(IOTools.openFile(cpgfile)) extension_upstream = PARAMS["cpg_search_upstream"] extension_downstream = PARAMS["cpg_search_downstream"] c = E.Counter() outf = IOTools.openFile(outfile, "w") outf.write( "transcript_id\tstrand\tstart\tend\trelative_start\trelative_end\n") for tss in Bed.iterator(IOTools.openFile(tssfile)): c.tss_total += 1 if tss.strand == "+": start, end = tss.start - \ extension_upstream, tss.start + extension_downstream else: start, end = tss.end - \ extension_downstream, tss.end + extension_upstream try: matches = list(cpg[tss.contig].find(start, end)) except KeyError: c.promotor_without_matches += 1 continue if len(matches) == 0: c.promotor_without_matches += 1 continue c.promotor_output += 1 for match in matches: c.matches_total += 1 genome_start, genome_end, x = match l = genome_end - genome_start # get relative location of match if tss.strand == "+": relative_start = genome_start - tss.start else: relative_start = tss.end - genome_end relative_end = relative_start + l outf.write("\t".join( map(str, (tss.name, tss.strand, genome_start, genome_end, relative_start, relative_end))) + "\n") c.matches_output += 1 outf.close() with IOTools.openFile(outfile + ".summary", "w") as outf: outf.write("category\tcounts\n") outf.write(c.asTable() + "\n") E.info(c)
def main(argv=sys.argv): # setup command line parser parser = E.OptionParser(version="%prog version: $Id$", usage=globals()["__doc__"]) parser.add_option("-m", "--method", dest="method", type="choice", choices=("igv", ), help="method to create plots with [%default]") parser.add_option("-d", "--snapshot-dir", dest="snapshotdir", type="string", help="directory to save snapshots in [%default]") parser.add_option("-f", "--format", dest="format", type="choice", choices=("png", "eps", "svg"), help="output file format [%default]") parser.add_option("-o", "--host", dest="host", type="string", help="host that IGV is running on [%default]") parser.add_option("-p", "--port", dest="port", type="int", help="port that IGV listens at [%default]") parser.add_option( "-e", "--extend", dest="extend", type="int", help="extend each interval by a number of bases [%default]") parser.add_option("-x", "--expand", dest="expand", type="float", help="expand each region by a certain factor [%default]") parser.set_defaults( method="igv", host='127.0.0.1', port=61111, snapshotdir=os.getcwd(), extend=0, format="png", expand=1.0, ) # add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv) E.info("connection to session on %s:%s" % (options.host, options.port)) E.info("saving images in %s" % options.snapshotdir) igv = IGV.IGV(host=options.host, port=options.port, snapshot_dir=os.path.abspath(options.snapshotdir)) c = E.Counter() for bed in Bed.iterator(options.stdin): c.input += 1 # IGV can not deal with white-space in filenames name = re.sub("\s", "_", bed.name) E.info("going to %s:%i-%i for %s" % (bed.contig, bed.start, bed.end, name)) start, end = bed.start, bed.end extend = options.extend if options.expand: d = end - start extend = max(extend, (options.expand * d - d) // 2) start -= extend end += extend igv.go("%s:%i-%i" % (bed.contig, start, end)) fn = "%s.%s" % (name, options.format) E.info("writing snapshot to '%s'" % fn) igv.save(fn) c.snapshots += 1 E.info(c) E.Stop()
def main(argv=sys.argv): # setup command line parser parser = E.OptionParser(version="%prog version: $Id$", usage=globals()["__doc__"]) parser.add_option("-n", "--new-instance", dest="new_instance", action="store_true", help="create a new IGV instance [%default]") parser.add_option("-s", "--session", dest="session", type="string", help="load session before creating plots " "[%default]") parser.add_option("-d", "--snapshot-dir", dest="snapshotdir", type="string", help="directory to save snapshots in [%default]") parser.add_option("-f", "--format", dest="format", type="choice", choices=("png", "eps", "svg"), help="output file format [%default]") parser.add_option("-o", "--host", dest="host", type="string", help="host that IGV is running on [%default]") parser.add_option("-p", "--port", dest="port", type="int", help="port that IGV listens at [%default]") parser.add_option("-e", "--extend", dest="extend", type="int", help="extend each interval by a number of bases " "[%default]") parser.add_option("-x", "--expand", dest="expand", type="float", help="expand each region by a certain factor " "[%default]") parser.add_option("--session-only", dest="session_only", action="store_true", help="plot session after opening, " "ignore intervals " "[%default]") parser.add_option("--keep", dest="keep_open", action="store_true", help="keep a newly created IGV session open " "[%default]") parser.set_defaults( command="igv.sh", host='127.0.0.1', port=61111, snapshotdir=os.getcwd(), extend=0, format="png", expand=1.0, session=None, session_only=False, keep_open=False, ) # add common options (-h/--help, ...) and parse command line (options, args) = E.start(parser, argv=argv, add_output_options=True) igv_process = None if options.new_instance: E.info("starting new IGV process") igv_process = IGV.startIGV(command=options.command, port=options.port) E.info("new IGV process started") E.info("connection to process on %s:%s" % (options.host, options.port)) E.info("saving images in %s" % options.snapshotdir) igv = IGV.IGV(host=options.host, port=options.port, snapshot_dir=os.path.abspath(options.snapshotdir)) if options.session: E.info('loading session from %s' % options.session) igv.load(options.session) E.info('loaded session') if options.session_only: E.info('plotting session only ignoring any intervals') fn = "%s.%s" % (os.path.basename(options.session), options.format) E.info("writing snapshot to '%s'" % os.path.join(options.snapshotdir, fn)) igv.save(fn) else: c = E.Counter() for bed in Bed.iterator(options.stdin): c.input += 1 # IGV can not deal with white-space in filenames name = re.sub("\s", "_", bed.name) E.info("going to %s:%i-%i for %s" % (bed.contig, bed.start, bed.end, name)) start, end = bed.start, bed.end extend = options.extend if options.expand: d = end - start extend = max(extend, (options.expand * d - d) // 2) start -= extend end += extend igv.go("%s:%i-%i" % (bed.contig, start, end)) fn = E.getOutputFile("%s.%s" % (name, options.format)) E.info("writing snapshot to '%s'" % fn) igv.save(fn) c.snapshots += 1 E.info(c) if igv_process is not None and not options.keep_open: E.info('shutting down IGV') igv_process.send_signal(signal.SIGKILL) E.stop()
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if argv == None: argv = sys.argv parser = E.OptionParser( version= "%prog version: $Id: bed2annotator2tsv.py 2885 2010-04-07 08:46:50Z andreas $", usage=globals()["__doc__"]) parser.add_option("-g", "--genome-file", dest="genome_file", type="string", help="filename with genome.") parser.add_option("-f", "--features", dest="features", type="string", help="feature to collect [default=None].") parser.add_option("-i", "--files", dest="files", action="append", help="use multiple annotations [default=None].") parser.add_option( "-a", "--annotations", dest="annotations", type="string", help= "aggregate name for annotations if only single file is provided from STDIN [default=None]." ) parser.add_option( "--input-filename-map", dest="input_filename_map", type="string", help="filename with a map of gene_ids to categories [default=None].") parser.add_option("-l", "--max-length", dest="max_length", type="string", help="maximum segment length [default=None].") parser.add_option( "-m", "--merge", dest="merge", action="store_true", help="merge overlapping bed segments [default=%default].") parser.add_option("-s", "--section", dest="section", type="choice", choices=("segments", "annotations", "workspace"), help="annotator section [default=None].") parser.add_option( "--subset", dest="subsets", type="string", action="append", help= "add filenames to delimit subsets within the gff files. The syntax is filename.gff,label,filename.ids [default=None]." ) parser.set_defaults( genome_file=None, feature=None, remove_random=True, section="segments", annotations="annotations", max_length=100000, files=[], subsets=[], input_filename_map=None, merge=False, ) (options, args) = E.Start(parser) options.files += args if len(options.files) == 0: options.files.append("-") options.files = list( itertools.chain(*[re.split("[,; ]+", x) for x in options.files])) if options.subsets: subsets = collections.defaultdict(list) for s in options.subsets: filename_gff, label, filename_ids = s.split(",") subsets[filename_gff].append((label, filename_ids)) options.subsets = subsets if options.genome_file: fasta = IndexedFasta.IndexedFasta(options.genome_file) else: fasta = None if options.section == "segments": prefix = "##Segs" elif options.section == "annotations": prefix = "##Id" elif options.section == "workspace": prefix = "##Work" else: raise ValueError("unknown section %s" % options.section) if options.max_length: max_length = options.max_length else: max_length = 0 ninput, ntracks, ncontigs, nsegments, ndiscarded = 0, 0, 0, 0, 0 if options.section in ("annotations"): contigs = set() it = itertools.groupby(Bed.iterator(options.stdin), key=lambda x: x.track["name"]) map_track2segments = {} for track, beds in it: ntracks += 1 map_track2segments[track] = [] first_segment = nsegments beds = list(beds) if options.merge: beds = Bed.merge(beds) for bed in beds: contig, start, end = bed.contig, bed.start, bed.end if options.remove_random and "random" in contig: continue if max_length > 0 and end - start > max_length: ndiscarded += 1 continue contigs.add(contig) map_track2segments[track].append(nsegments) options.stdout.write("%s\t%i\t%s\t(%i,%i)\n" % (prefix, nsegments, contig, start, end)) nsegments += 1 options.stdout.write("##Ann\t%s\t%s\n" % (track, "\t".join( ["%i" % x for x in range(first_segment, nsegments)]))) E.info("track %s: annotated with %i segments" % (track, nsegments - first_segment)) ncontigs = len(contigs) E.info( "ninput=%i, ntracks=%i, ncontigs=%i, nsegments=%i, ndiscarded=%i" % (ninput, ntracks, ncontigs, nsegments, ndiscarded)) E.Stop()
def main(argv=None): if argv is None: argv = sys.argv parser = E.OptionParser( version="%prog version: $Id$", usage=globals()["__doc__"]) parser.add_option( "-g", "--genome-file", dest="genome_file", type="string", help="filename with genome [default=%default].") parser.add_option( "-a", "--aggregate-by", dest="aggregate", type="choice", choices=("name", "contig", "track", "none"), help="aggregate counts by feature [default=%default].") parser.add_option( "-p", "--add-percent", dest="add_percent", action="store_true", help="add percentages [default=%default].") parser.set_defaults( genome_file=None, aggregate="none", add_percent=False, ) (options, args) = E.Start(parser, argv) # get files if options.genome_file: fasta = IndexedFasta.IndexedFasta(options.genome_file) else: if options.add_percent: raise ValueError("--add-percent option requires --genome-file") fasta = None if options.add_percent and not options.aggregate == "contig": raise NotImplementedError( "--add-percent option requires --aggregate=contig") counts = collections.defaultdict(Counter) total = Counter() output_totals = True if options.aggregate == "track": keyf = lambda x: x.track elif options.aggregate == "name": keyf = lambda x: x.name elif options.aggregate == "contig": keyf = lambda x: x.contig else: keyf = lambda x: "all" output_totals = False for bed in Bed.iterator(options.stdin): counts[keyf(bed)].add(bed) total.add(bed) outf = options.stdout key = "track" if options.add_percent: outf.write("%s\t%s\n" % (key, "\t".join(Counter.headers_percent))) else: outf.write("%s\t%s\n" % (key, "\t".join(Counter.headers))) total_bases = 0 for key, count in sorted(counts.items()): if options.add_percent: total_bases += fasta.getLength(key) count.setSize(fasta.getLength(key)) outf.write("%s\t%s\n" % (key, str(count))) if output_totals: if options.add_percent: count.setSize(total_bases) outf.write("%s\t%s\n" % ("total", str(total))) E.Stop()
def main(argv=sys.argv): parser = E.OptionParser(version="%prog version: $Id$", usage=globals()["__doc__"]) parser.add_option("-a", "--as-gtf", dest="as_gtf", action="store_true", help="output as gtf.") parser.add_option( "-f", "--id-format", dest="id_format", type="string", help="format for numeric identifier if --as-gtf is set and " "no name in bed file [%default].") parser.set_defaults(as_gtf=False, id_format="%08i", test=None) (options, args) = E.start(parser, add_pipe_options=True) as_gtf = options.as_gtf id_format = options.id_format if as_gtf: gff = GTF.Entry() else: gff = GTF.Entry() gff.source = "bed" gff.feature = "exon" ninput, noutput, nskipped = 0, 0, 0 id = 0 for bed in Bed.iterator(options.stdin): ninput += 1 gff.contig = bed.contig gff.start = bed.start gff.end = bed.end if bed.fields and len(bed.fields) >= 3: gff.strand = bed.fields[2] else: gff.strand = "." if bed.fields and len(bed.fields) >= 2: gff.score = bed.fields[1] if as_gtf: if bed.fields: gff.gene_id = bed.fields[0] gff.transcript_id = bed.fields[0] else: id += 1 gff.gene_id = id_format % id gff.transcript_id = id_format % id else: if bed.fields: gff.source = bed.fields[0] options.stdout.write(str(gff) + "\n") noutput += 1 E.info("ninput=%i, noutput=%i, nskipped=%i" % (ninput, noutput, nskipped)) E.stop()
def main(argv=None): if argv == None: argv = sys.argv parser = E.OptionParser( version= "%prog version: $Id: gtf2table.py 2888 2010-04-07 08:48:36Z andreas $", usage=globals()["__doc__"]) parser.add_option("-g", "--genome-file", dest="genome_file", type="string", help="filename with genome [default=%default].") parser.add_option("-n", "--per-name", dest="per_name", action="store_true", help="compute counts per name [default=%default].") parser.add_option("-c", "--per-contig", dest="per_contig", action="store_true", help="compute counts per contig [default=%default].") parser.add_option("-t", "--per-track", dest="per_track", action="store_true", help="compute counts per track [default=%default].") parser.add_option("-p", "--add-percent", dest="add_percent", action="store_true", help="add percentages [default=%default].") parser.set_defaults( genome_file=None, per_name=False, per_track=False, add_percent=False, ) (options, args) = E.Start(parser, argv) # get files if options.genome_file: fasta = IndexedFasta.IndexedFasta(options.genome_file) else: if options.add_percent: raise ValueError("--add-percent option requires --genome-file") fasta = None if options.add_percent and not options.per_contig: raise NotImplementedError("--add-percent option requires --per-contig") counts = collections.defaultdict(Counter) if options.per_track: keyf = lambda x: x.track elif options.per_name: keyf = lambda x: x.name elif options.per_contig: keyf = lambda x: x.contig else: keyf = lambda x: "all" for bed in Bed.iterator(options.stdin): counts[keyf(bed)].add(bed) outf = options.stdout key = "track" if options.add_percent: outf.write("%s\t%s\n" % (key, "\t".join(Counter.headers_percent))) else: outf.write("%s\t%s\n" % (key, "\t".join(Counter.headers))) total_bases = 0 for key, count in counts.iteritems(): if options.add_percent: total_bases += fasta.getLength(key) count.setSize(fasta.getLength(key)) outf.write("%s\t%s\n" % (key, str(count))) E.Stop()
def main( argv = None ): if argv == None: argv = sys.argv parser = E.OptionParser( version = "%prog version: $Id: gtf2table.py 2888 2010-04-07 08:48:36Z andreas $", usage = globals()["__doc__"]) parser.add_option("-g", "--genome-file", dest="genome_file", type="string", help="filename with genome [default=%default]." ) parser.add_option("-b", "--bam-file", dest="bam_files", type="string", help="filename with read mapping information. Multiple files can be submitted in a comma-separated list [default=%default]." ) parser.add_option( "--control-bam-file", dest="control_bam_files", type="string", help="filename with read mapping information for input/control. Multiple files can be submitted in a comma-separated list [default=%default]." ) parser.add_option( "--filename-format", dest="filename_format", type="choice", choices=("bed", "gff", "gtf" ), help="format of secondary stream [default=%default]." ) parser.add_option("-c", "--counter", dest="counters", type="choice", action="append", choices=( "overlap", "peaks", "composition-na", "composition-cpg", "classifier-chipseq"), help="select counters to apply [default=%default]." ) parser.add_option("-o", "--offset", dest="offsets", type="int", action="append", help="tag offsets for tag counting - supply as many as there are bam-files [default=%default]." ) parser.add_option( "--control-offset", dest="control_offsets", type="int", action="append", help="control tag offsets for tag counting - supply as many as there are bam-files [default=%default]." ) parser.add_option("-a", "--all-fields", dest="all_fields", action = "store_true", help="output all fields in original bed file, by default only the first 4 are output [default=%default]." ) parser.add_option("--bed-headers", dest="bed_headers", type="string", help="supply ',' separated list of headers for bed component [default=%default]." ) parser.add_option("-f", "--filename-gff", dest="filename_gff", type="string", action="append", metavar='bed', help="filename with extra gff files. The order is important [default=%default]." ) parser.set_defaults( genome_file = None, counters = [], bam_files = None, offsets = [], control_bam_files = None, control_offsets = [], all_fields = False, filename_format = None, bed_headers = "contig,start,end,name", filename_gff = [], ) (options, args) = E.Start( parser ) # get files if options.genome_file: fasta = IndexedFasta.IndexedFasta( options.genome_file ) else: fasta = None if options.bam_files: bam_files = [] for bamfile in options.bam_files.split(","): bam_files.append( pysam.Samfile(bamfile, "rb" ) ) else: bam_files = None if options.control_bam_files: control_bam_files = [] for bamfile in options.control_bam_files.split(","): control_bam_files.append( pysam.Samfile(bamfile, "rb" ) ) else: control_bam_files = None counters = [] for c in options.counters: if c == "overlap": counters.append( CounterOverlap( filename = options.filename_bed, fasta=fasta, options = options) ) elif c == "peaks": counters.append( CounterPeaks( bam_files, options.offsets, control_bam_files, options.control_offsets, options = options ) ) elif c == "composition-na": counters.append( CounterCompositionNucleotides( fasta=fasta, options = options )) elif c == "composition-cpg": counters.append( CounterCompositionCpG( fasta=fasta, options = options ) ) elif c == "classifier-chipseq": counters.append( ClassifierChIPSeq( filename_gff = options.filename_gff, fasta = fasta, options = options, prefix = None) ) options.stdout.write( "\t".join( [x.strip() for x in options.bed_headers.split(",") ] ) ) options.stdout.write( "\t" + "\t".join( [ x.getHeader() for x in counters] ) + "\n" ) for bed in Bed.iterator(options.stdin): for counter in counters: counter.update(bed) if options.all_fields: options.stdout.write( str(bed) ) else: options.stdout.write( "\t".join( (bed.contig, str(bed.start), str(bed.end), bed.fields[0]) ) ) for counter in counters: options.stdout.write("\t%s" % str(counter) ) options.stdout.write("\n") E.Stop()
def buildResults(bedfile, fg_file, control_file, counter, options): '''compute densities and peakshape parameters.''' options.stdout.write("\t".join(("contig", "start", "end", "name", "\t".join(_bam2peakshape.PeakShapeResult._fields))) + "\n") if options.window_size: # bins are centered at peak-center and then stretching outwards. bins = numpy.arange(-options.window_size + options.bin_size // 2, +options.window_size, options.bin_size) #contigs = set(pysam_in.references) strand_specific = options.strand_specific result = [] c = E.Counter() c.input = 0 for bed in Bed.iterator(IOTools.openFile(bedfile)): c.input += 1 # if bed.contig not in contigs: # c.skipped += 1 # continue if c.input % options.report_step == 0: E.info("iteration: %i" % c.input) features = counter.countInInterval( fg_file, bed.contig, bed.start, bed.end, window_size=options.window_size, bins=bins, only_interval=options.only_interval, centring_method=options.centring_method) if features is None: c.skipped += 1 continue if control_file: control = counter.countAroundPos(control_file, bed.contig, features.peak_center, bins=features.bins) else: control = None if options.random_shift: direction = numpy.random.randint(0, 2) if direction: pos = features.peak_center + 2 * bins[0] else: pos = features.peak_center + 2 * bins[-1] shifted = counter.countAroundPos(fg_file, bed.contig, pos, bins=features.bins) else: shifted = None if strand_specific and bed.strand == "-": features._replace(hist=hist[::-1]) if control: control._replace(hist=hist[::-1]) if shifted: shift._replace(hist=hist[::-1]) result.append((features, bed, control, shifted)) c.added += 1 E.info("interval processing: %s" % c) return result, bins
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if not argv: argv = sys.argv # setup command line parser parser = E.OptionParser(version="%prog version: $Id$", usage=globals()["__doc__"]) parser.add_option("-m", "--min-overlap", dest="min_overlap", type="float", help="minimum overlap [%default]") parser.add_option("-k", "--keep-temp", dest="keep_temp", action="store_true", help="do not delete temporary files [%default]") parser.add_option("-a", "--filename-bam", dest="filename_bam", metavar="bam", type="string", help="bam-file to use [%default]") parser.add_option("-b", "--filename-bed", dest="filename_bed", metavar="bed", type="string", help="bed-file to use [%default]") parser.set_defaults( min_overlap=0.5, keep_temp=False, filename_bam=None, filename_bed=None, ) # add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv) filename_bam = options.filename_bam filename_bed = options.filename_bed if filename_bam is None and filename_bed is None: if len(args) != 2: raise ValueError( "please supply a bam and a bed file or two bed-files.") filename_bam, filename_bed = args if filename_bed is None: raise ValueError("please supply a bed file to compare to.") if filename_bam is None: raise ValueError("please supply a bam file to compare with.") E.info("intersecting the two files") tmpfile = tempfile.NamedTemporaryFile(delete=False) tmpfile.close() tmpfilename = tmpfile.name min_overlap = options.min_overlap options.stdout.write("category\talignments\n") # get number of columns of reference bed file for bed in Bed.iterator(IOTools.openFile(filename_bed)): ncolumns_bed = bed.columns break E.info("assuming %s is bed%i format" % (filename_bed, ncolumns_bed)) if ncolumns_bed < 4: raise ValueError("please supply a name attribute in the bed file") # get information about if filename_bam.endswith(".bam"): format = "-abam" samfile = pysam.Samfile(filename_bam, "rb") total = samfile.mapped # latest bedtools uses bed12 format when bam is input ncolumns_bam = 12 # count per read sort_key = lambda x: x.name else: format = "-a" total = IOTools.getNumLines(filename_bam) # get bed format ncolumns_bam = 0 for bed in Bed.iterator(IOTools.openFile(filename_bam)): ncolumns_bam = bed.columns break if ncolumns_bam > 0: E.info("assuming %s is bed%i fomat" % (filename_bam, ncolumns_bam)) if ncolumns_bam == 3: # count per interval sort_key = lambda x: (x.contig, x.start, x.end) else: # count per interval category sort_key = lambda x: x.name # use fields for bam/bed file (regions to count with) data_fields = [ "contig", "start", "end", "name", "score", "strand", "thickstart", "thickend", "rgb", "blockcount", "blockstarts", "blockends"][:ncolumns_bam] # add fields for second bed (regions to count in) data_fields.extend([ "contig2", "start2", "end2", "name2", "score2", "strand2", "thickstart2", "thickend2", "rgb2", "blockcount2", "blockstarts2", "blockends2"][:ncolumns_bed]) # add bases overlap data_fields.append("bases_overlap") data = collections.namedtuple("data", data_fields) options.stdout.write("total\t%i\n" % total) if total == 0: E.warn("no data in %s" % filename_bam) return # IMS: newer versions of intersectBed have a very high memory # requirement unless passed sorted bed files. statement = """intersectBed %(format)s %(filename_bam)s -b <( zcat %(filename_bed)s | sort -k1,1 -k2,2n) -sorted -bed -wo -f %(min_overlap)f > %(tmpfilename)s""" % locals() E.info("running %s" % statement) retcode = E.run(statement) if retcode != 0: raise ValueError("error while executing statement %s" % statement) infile = open(tmpfilename, "r") counts_per_alignment = collections.defaultdict(int) E.info("counting") take_columns = len(data._fields) def iter(infile): for line in infile: if not line.strip(): continue yield data._make(line[:-1].split()[:take_columns]) for read, overlaps in itertools.groupby(iter(infile), key=sort_key): annotations = [x.name2 for x in overlaps] for anno in annotations: counts_per_alignment[anno] += 1 infile.close() for key, counts in counts_per_alignment.iteritems(): options.stdout.write("%s\t%i\n" % (key, counts)) if not options.keep_temp: os.unlink(tmpfilename) # write footer and output benchmark information. E.Stop()
def readSegments(infile, indexed_workspace, truncate=False, format="gtf", keep_ambiguous=False, remove_overhangs=False): """read segments from infile. segments not overlapping with indexed_workspace are removed. If :attr: truncate is given, segments extending beyond the workspace are truncated. returns a list of segments for each contig in a dictionary """ counter = E.Counter() segments = collections.defaultdict(list) def addSegment(contig, start, end, counter): if contig in indexed_workspace: r = indexed_workspace[contig].find(start, end) if not r: counter.nskipped += 1 return if len(r) > 1: counter.nambiguous += 1 if not keep_ambiguous: return if truncate: for x in r: wstart, wend = x.start, x.end rstart, rend = max(start, wstart), min(end, wend) if start < wstart or end > wend: counter.ntruncated += 1 segments[contig].append((rstart, rend)) counter.added += 1 elif remove_overhangs: for x in r: wstart, wend = x.start, x.end rstart, rend = max(start, wstart), min(end, wend) if start < wstart or end > wend: counter.overhangs += 1 break else: segments[contig].append((start, end)) else: segments[contig].append((start, end)) counter.added += 1 counter.nkept += 1 if format == "gtf": gtf_iterator = GTF.flat_gene_iterator(GTF.iterator(infile)) for gene in gtf_iterator: # get start and end ignoring introns # contig, start, end = gene[0].contig, min( [x.start for x in gene] ), max( [x.end for x in gene] ) contig, coords = gene[0].contig, [(x.start, x.end) for x in gene] counter.ninput += 1 for start, end in coords: addSegment(contig, start, end, counter) elif format == "bed": bed_iterator = Bed.iterator(infile) for bed in bed_iterator: counter.ninput += 1 addSegment(bed.contig, bed.start, bed.end, counter) E.info("read segments: %s" % str(counter)) return segments
def main(argv=sys.argv): parser = E.OptionParser( version= "%prog version: $Id: bed2bed.py 2861 2010-02-23 17:36:32Z andreas $", usage=globals()["__doc__"]) # IMS: new method: extend intervals by set amount parser.add_option("-m", "--method", dest="methods", type="choice", action="append", choices=("merge", "filter-genome", "bins", "block", "sanitize-genome", "shift", "extend", "filter-names"), help="method to apply [default=%default]") parser.add_option("--num-bins", dest="num_bins", type="int", help="number of bins into which to merge (used for " "method `bins) [default=%default]") parser.add_option("--bin-edges", dest="bin_edges", type="string", help="bin_edges for binning method [default=%default]") parser.add_option( "--binning-method", dest="binning_method", type="choice", choices=("equal-bases", "equal-intervals", "equal-range"), help="method used for binning (used for method `bins` if no " "bin_edges is given) [default=%default]") parser.add_option( "--merge-distance", dest="merge_distance", type="int", help="distance in bases over which to merge that are not " "directly adjacent [default=%default]") parser.add_option( "--merge-min-intervals", dest="merge_min_intervals", type="int", help="only output merged intervals that are build from at least " "x intervals [default=%default]") parser.add_option( "--merge-by-name", dest="merge_by_name", action="store_true", help="only merge intervals with the same name [default=%default]") parser.add_option( "--merge-and-resolve-blocks", dest="resolve_blocks", action="store_true", help="When merging bed12 entrys, should blocks be resolved?") parser.add_option("--merge-stranded", dest="stranded", action="store_true", help="Only merge intervals on the same strand") parser.add_option( "--remove-inconsistent-names", dest="remove_inconsistent_names", action="store_true", help="when merging, do not output intervals where the names of " "overlapping intervals do not match [default=%default]") parser.add_option("--offset", dest="offset", type="int", help="offset for shifting intervals [default=%default]") parser.add_option("-g", "--genome-file", dest="genome_file", type="string", help="filename with genome.") parser.add_option("-b", "--bam-file", dest="bam_file", type="string", help="bam-formatted filename with genome.") parser.add_option("--filter-names-file", dest="names", type="string", help="list of names to keep. One per line") parser.set_defaults(methods=[], merge_distance=0, binning_method="equal-bases", merge_by_name=False, genome_file=None, bam_file=None, num_bins=5, merge_min_intervals=1, bin_edges=None, offset=10000, test=None, extend_distance=1000, remove_inconsistent_names=False, resolve_blocks=False) (options, args) = E.Start(parser, add_pipe_options=True) contigs = None # Why provide full indexed genome, when a tsv of contig sizes would do? if options.genome_file: genome_fasta = IndexedFasta.IndexedFasta(options.genome_file) contigs = genome_fasta.getContigSizes() if options.bam_file: samfile = pysam.AlignmentFile(options.bam_file) contigs = dict(list(zip(samfile.references, samfile.lengths))) processor = Bed.iterator(options.stdin) for method in options.methods: if method == "filter-genome": if not contigs: raise ValueError("please supply contig sizes") processor = filterGenome(processor, contigs) elif method == "sanitize-genome": if not contigs: raise ValueError("please supply contig sizes") processor = sanitizeGenome(processor, contigs) elif method == "merge": processor = merge( processor, options.merge_distance, by_name=options.merge_by_name, min_intervals=options.merge_min_intervals, remove_inconsistent=options.remove_inconsistent_names, resolve_blocks=options.resolve_blocks, stranded=options.stranded) elif method == "bins": if options.bin_edges: bin_edges = list(map(float, options.bin_edges.split(","))) # IMS: check bin edges are valid if not (len(bin_edges) == options.num_bins + 1): raise ValueError( "Number of bin edge must be one more than " "number of bins") else: bin_edges = None processor, bin_edges = Bed.binIntervals( processor, num_bins=options.num_bins, method=options.binning_method, bin_edges=bin_edges) E.info("# split bed: bin_edges=%s" % (str(bin_edges))) elif method == "block": processor = Bed.blocked_iterator(processor) elif method == "shift": # IMS: test that contig sizes are availible if not contigs: raise ValueError("please supply genome file") processor = shiftIntervals(processor, contigs, offset=options.offset) # IMS: new method: extend intervals by set amount elif method == "extend": if not contigs: raise ValueError("please supply genome file") processor = extendInterval(processor, contigs, options.offset) elif method == "filter-names": if not options.names: raise ValueError("please supply list of names to filter") names = [name.strip() for name in open(options.names)] processor = filterNames(processor, names) noutput = 0 for bed in processor: options.stdout.write(str(bed) + "\n") noutput += 1 E.info("noutput=%i" % (noutput)) E.Stop()
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if not argv: argv = sys.argv # setup command line parser parser = E.OptionParser(version="%prog version: $Id$", usage=globals()["__doc__"]) parser.add_option( "-g", "--gtf-file", dest="filename_gtf", type="string", help="filename with gene models in gtf format [%default]") parser.add_option( "-m", "--filename-mismapped", dest="filename_mismapped", type="string", help="output bam file for mismapped reads [%default]") parser.add_option( "-j", "--junctions-bed-file", dest="filename_junctions", type="string", help="bam file with reads mapped across junctions [%default]") parser.add_option( "-r", "--filename-regions", dest="filename_regions", type="string", help="filename with regions to remove in bed format [%default]") parser.add_option( "-t", "--transcripts-gtf-file", dest="filename_transcriptome", type="string", help="bam file with reads mapped against transcripts [%default]") parser.add_option( "-p", "--map-tsv-file", dest="filename_map", type="string", help="filename mapping transcript numbers (used by " "--filename-transciptome) to transcript names " "(used by --filename-gtf) [%default]") parser.add_option( "-s", "--filename-stats", dest="filename_stats", type="string", help="filename to output stats to [%default]") parser.add_option( "-o", "--colour", dest="colour_mismatches", action="store_true", help="mismatches will use colour differences (CM tag) [%default]") parser.add_option( "-i", "--ignore-mismatches", dest="ignore_mismatches", action="store_true", help="ignore mismatches [%default]") parser.add_option( "-c", "--remove-contigs", dest="remove_contigs", type="string", help="','-separated list of contigs to remove [%default]") parser.add_option( "-f", "--force-output", dest="force", action="store_true", help="force overwriting of existing files [%default]") parser.add_option("-u", "--unique", dest="unique", action="store_true", help="remove reads not matching uniquely [%default]") parser.add_option("--output-sam", dest="output_sam", action="store_true", help="output in sam format [%default]") parser.set_defaults( filename_gtf=None, filename_mismapped=None, filename_junctions=None, filename_transcriptome=None, filename_map=None, remove_contigs=None, force=False, unique=False, colour_mismatches=False, ignore_mismatches=False, output_sam=False, filename_table=None, ) # add common options (-h/--help, ...) and parse command line (options, args) = E.start(parser, argv=argv) if len(args) != 1: raise ValueError("please supply one bam file") bamfile_genome = args[0] genome_samfile = pysam.AlignmentFile(bamfile_genome, "rb") if options.remove_contigs: options.remove_contigs = options.remove_contigs.split(",") if options.filename_map: E.info("reading map") id_map = IOTools.read_map( IOTools.open_file(options.filename_map), has_header=True) id_map = dict([(y, x) for x, y in id_map.items()]) else: id_map = None transcripts = {} if options.filename_gtf: E.info("indexing geneset") mapped, missed = 0, 0 for gtf in GTF.transcript_iterator( GTF.iterator(IOTools.open_file(options.filename_gtf))): gtf.sort(key=lambda x: x.start) transcript_id = gtf[0].transcript_id if id_map: try: transcript_id = id_map[transcript_id] mapped += 1 except KeyError: missed += 1 continue transcripts[transcript_id] = gtf E.info("read %i transcripts from geneset (%i mapped, %i missed)" % (len(transcripts), mapped, missed)) regions_to_remove = None if options.filename_regions: E.info("indexing regions") regions_to_remove = IndexedGenome.Simple() for bed in Bed.iterator(IOTools.open_file(options.filename_regions)): regions_to_remove.add(bed.contig, bed.start, bed.end) E.info("read %i regions" % len(regions_to_remove)) if options.filename_transcriptome: transcripts_samfile = pysam.AlignmentFile(options.filename_transcriptome, "rb") else: transcripts_samfile = None if options.output_sam: output_samfile = pysam.AlignmentFile("-", "wh", template=genome_samfile) else: output_samfile = pysam.AlignmentFile("-", "wb", template=genome_samfile) if options.filename_mismapped: if not options.force and os.path.exists(options.filename_mismapped): raise IOError("output file %s already exists" % options.filename_mismapped) output_mismapped = pysam.AlignmentFile(options.filename_mismapped, "wb", template=genome_samfile) else: output_mismapped = None if options.filename_junctions: junctions_samfile = pysam.AlignmentFile(options.filename_junctions, "rb") else: junctions_samfile = None c = bams2bam_filter(genome_samfile, output_samfile, output_mismapped, transcripts_samfile, junctions_samfile, transcripts, regions=regions_to_remove, unique=options.unique, remove_contigs=options.remove_contigs, colour_mismatches=options.colour_mismatches, ignore_mismatches=options.ignore_mismatches, ignore_transcripts=transcripts_samfile is None, ignore_junctions=junctions_samfile is None) if options.filename_stats: outf = IOTools.open_file(options.filename_stats, "w") outf.write("category\tcounts\n%s\n" % c.asTable()) outf.close() if options.filename_transcriptome: transcripts_samfile.close() genome_samfile.close() output_samfile.close() if output_mismapped: output_mismapped.close() # write footer and output benchmark information. E.stop()
def main(argv=None): if argv is None: argv = sys.argv parser = E.OptionParser(version="%prog version: $Id$", usage=globals()["__doc__"]) parser.add_option("-g", "--genome-file", dest="genome_file", type="string", help="filename with genomic sequence to retrieve " "sequences from.") parser.add_option("-m", "--masker", dest="masker", type="choice", choices=("dust", "dustmasker", "softmask", "none"), help="apply masker to mask output sequences " "[%default].") parser.add_option("--output-mode", dest="output_mode", type="choice", choices=("intervals", "leftright", "segments"), help="what to output. " "'intervals' generates a single sequence for " "each bed interval. 'leftright' generates two " "sequences, one in each direction, for each bed " "interval. 'segments' can be used to output " "sequence from bed12 files so that sequence only covers " "the segements [%default]") parser.add_option("--min-sequence-length", dest="min_length", type="int", help="require a minimum sequence length [%default]") parser.add_option("--max-sequence-length", dest="max_length", type="int", help="require a maximum sequence length [%default]") parser.add_option( "--extend-at", dest="extend_at", type="choice", choices=("none", "3", "5", "both", "3only", "5only"), help="extend at 3', 5' or both or no ends. If 3only or 5only " "are set, only the added sequence is returned [default=%default]") parser.add_option("--extend-by", dest="extend_by", type="int", help="extend by # bases [default=%default]") parser.add_option( "--use-strand", dest="ignore_strand", action="store_false", help="use strand information and return reverse complement " "on intervals located on the negative strand. " "[default=%default]") parser.set_defaults( genome_file=None, masker=None, output_mode="intervals", min_length=0, max_length=0, extend_at=None, extend_by=100, ignore_strand=True, ) (options, args) = E.start(parser) if options.genome_file: fasta = IndexedFasta.IndexedFasta(options.genome_file) contigs = fasta.getContigSizes() fasta.setConverter(IndexedFasta.getConverter("zero-both-open")) counter = E.Counter() ids, seqs = [], [] E.info("collecting sequences") for bed in Bed.setName(Bed.iterator(options.stdin)): counter.input += 1 lcontig = fasta.getLength(bed.contig) if options.ignore_strand: strand = "+" else: strand = bed.strand if options.output_mode == "segments" and bed.columns == 12: ids.append("%s %s:%i..%i (%s) %s %s" % (bed.name, bed.contig, bed.start, bed.end, strand, bed["blockSizes"], bed["blockStarts"])) seg_seqs = [ fasta.getSequence(bed.contig, strand, start, end) for start, end in bed.toIntervals() ] seqs.append("".join(seg_seqs)) elif (options.output_mode == "intervals" or options.output_mode == "segments"): ids.append("%s %s:%i..%i (%s)" % (bed.name, bed.contig, bed.start, bed.end, strand)) seqs.append( fasta.getSequence(bed.contig, strand, bed.start, bed.end)) elif options.output_mode == "leftright": l = bed.end - bed.start start, end = max(0, bed.start - l), bed.end - l ids.append("%s_l %s:%i..%i (%s)" % (bed.name, bed.contig, start, end, strand)) seqs.append(fasta.getSequence(bed.contig, strand, start, end)) start, end = bed.start + l, min(lcontig, bed.end + l) ids.append("%s_r %s:%i..%i (%s)" % (bed.name, bed.contig, start, end, strand)) seqs.append(fasta.getSequence(bed.contig, strand, start, end)) E.info("collected %i sequences" % len(seqs)) masked = Masker.maskSequences(seqs, options.masker) options.stdout.write( "\n".join([">%s\n%s" % (x, y) for x, y in zip(ids, masked)]) + "\n") E.info("masked %i sequences" % len(seqs)) counter.output = len(seqs) E.info("%s" % counter) E.stop()
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if not argv: argv = sys.argv # setup command line parser parser = E.OptionParser(version="%prog version: $Id$", usage=globals()["__doc__"]) parser.add_option("-m", "--min-overlap", dest="min_overlap", type="float", help="minimum overlap [%default]") parser.add_option("-a", "--bam-file", dest="filename_bam", metavar="bam", type="string", help="bam-file to use (required) [%default]") parser.add_option("-b", "--bed-file", dest="filename_bed", metavar="bed", type="string", help="bed-file to use (required) [%default]") parser.add_option( "-s", "--sort-bed", dest="sort_bed", action="store_true", help="sort the bed file by chromosomal location before " "processing. " "[%default]") parser.add_option( "--assume-sorted", dest="sort_bed", action="store_false", help="assume that the bed-file is sorted by chromosomal location. " "[%default]") parser.add_option( "--split-intervals", dest="split_intervals", action="store_true", help="treat split BAM intervals, for example spliced intervals, " "as separate intervals. Note that a single alignment might be " "counted several times as a result. " "[%default]") parser.set_defaults( min_overlap=0.5, filename_bam=None, filename_bed=None, sort_bed=True, split_intervals=False, ) # add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv) filename_bam = options.filename_bam filename_bed = options.filename_bed if filename_bam is None and filename_bed is None: if len(args) != 2: raise ValueError( "please supply a bam and a bed file or two bed-files.") filename_bam, filename_bed = args if filename_bed is None: raise ValueError("please supply a bed file to compare to.") if filename_bam is None: raise ValueError("please supply a bam file to compare with.") E.info("intersecting the two files") min_overlap = options.min_overlap options.stdout.write("category\talignments\n") # get number of columns of reference bed file for bed in Bed.iterator(IOTools.openFile(filename_bed)): ncolumns_bed = bed.columns break E.info("assuming %s is bed%i format" % (filename_bed, ncolumns_bed)) if ncolumns_bed < 4: raise ValueError("please supply a name attribute in the bed file") # get information about if filename_bam.endswith(".bam"): format = "-abam" samfile = pysam.Samfile(filename_bam, "rb") total = samfile.mapped # latest bedtools uses bed12 format when bam is input ncolumns_bam = 12 # count per read sort_key = lambda x: x.name else: format = "-a" total = IOTools.getNumLines(filename_bam) # get bed format ncolumns_bam = 0 for bed in Bed.iterator(IOTools.openFile(filename_bam)): ncolumns_bam = bed.columns break if ncolumns_bam > 0: E.info("assuming %s is bed%i fomat" % (filename_bam, ncolumns_bam)) if ncolumns_bam == 3: # count per interval sort_key = lambda x: (x.contig, x.start, x.end) else: # count per interval category sort_key = lambda x: x.name # use fields for bam/bed file (regions to count with) data_fields = [ "contig", "start", "end", "name", "score", "strand", "thickstart", "thickend", "rgb", "blockcount", "blockstarts", "blockends"][:ncolumns_bam] # add fields for second bed (regions to count in) data_fields.extend([ "contig2", "start2", "end2", "name2", "score2", "strand2", "thickstart2", "thickend2", "rgb2", "blockcount2", "blockstarts2", "blockends2"][:ncolumns_bed]) # add bases overlap data_fields.append("bases_overlap") data = collections.namedtuple("data", data_fields) options.stdout.write("total\t%i\n" % total) if total == 0: E.warn("no data in %s" % filename_bam) return # SNS: sorting optional, off by default if options.sort_bed: bedcmd = "<( zcat %s | sort -k1,1 -k2,2n)" % filename_bed else: bedcmd = filename_bed if options.split_intervals: split = "-split" else: split = "" # IMS: newer versions of intersectBed have a very high memory # requirement unless passed sorted bed files. statement = """bedtools intersect %(format)s %(filename_bam)s -b %(bedcmd)s %(split)s -sorted -bed -wo -f %(min_overlap)f""" % locals() E.info("starting counting process: %s" % statement) proc = E.run(statement, return_popen=True, stdout=subprocess.PIPE) E.info("counting") counts_per_alignment = collections.defaultdict(int) take_columns = len(data._fields) def iterate(infile): for line in infile: if not line.strip(): continue yield data._make(line[:-1].split()[:take_columns]) for read, overlaps in itertools.groupby( iterate(IOTools.force_str(proc.stdout)), key=sort_key): annotations = [x.name2 for x in overlaps] for anno in annotations: counts_per_alignment[anno] += 1 for key, counts in sorted(counts_per_alignment.items()): options.stdout.write("%s\t%i\n" % (key, counts)) # write footer and output benchmark information. E.Stop()
def buildResults(bedfile, fg_file, control_file, counter, options): '''compute densities and peakshape parameters.''' options.stdout.write("\t".join( ("contig", "start", "end", "name", "\t".join(_bam2peakshape.PeakShapeResult._fields))) + "\n") if options.window_size: # bins are centered at peak-center and then stretching outwards. bins = numpy.arange(-options.window_size + options.bin_size // 2, +options.window_size, options.bin_size) #contigs = set(pysam_in.references) strand_specific = options.strand_specific result = [] c = E.Counter() c.input = 0 for bed in Bed.iterator(IOTools.openFile(bedfile)): c.input += 1 #if bed.contig not in contigs: # c.skipped += 1 # continue if c.input % options.report_step == 0: E.info("iteration: %i" % c.input) features = counter.countInInterval( fg_file, bed.contig, bed.start, bed.end, window_size=options.window_size, bins=bins, only_interval=options.only_interval, centring_method=options.centring_method) if control_file: control = counter.countAroundPos(control_file, bed.contig, features.peak_center, bins=features.bins) else: control = None if options.random_shift: direction = numpy.random.randint(0, 2) if direction: pos = features.peak_center + 2 * bins[0] else: pos = features.peak_center + 2 * bins[-1] shifted = counter.countAroundPos(fg_file, bed.contig, pos, bins=features.bins) else: shifted = None if strand_specific and bed.strand == "-": features._replace(hist=hist[::-1]) if control: control._replace(hist=hist[::-1]) if shifted: shift._replace(hist=hist[::-1]) result.append((features, bed, control, shifted)) c.added += 1 E.info("interval processing: %s" % c) return result, bins
def main( argv = None ): """script main. parses command line options in sys.argv, unless *argv* is given. """ if argv == None: argv = sys.argv parser = E.OptionParser( version = "%prog version: $Id: gff2histogram.py 2781 2009-09-10 11:33:14Z andreas $") parser.add_option("-b", "--bin-size", dest="bin_size", type="string", help="bin size." ) parser.add_option("--min-value", dest="min_value", type="float", help="minimum value for histogram.") parser.add_option("--max-value", dest="max_value", type="float", help="maximum value for histogram.") parser.add_option("--no-empty-bins", dest="no_empty_bins", action="store_true", help="do not display empty bins.") parser.add_option("--with-empty-bins", dest="no_empty_bins", action="store_false", help="display empty bins.") parser.add_option("--ignore-out-of-range", dest="ignore_out_of_range", action="store_true", help="ignore values that are out of range (as opposed to truncating them to range border.") parser.add_option("--missing", dest="missing_value", type="string", help="entry for missing values [%default]." ) parser.add_option("--dynamic-bins", dest="dynamic_bins", action="store_true", help="each value constitutes its own bin." ) parser.add_option( "--format", dest="format", type="choice", choices=( "gff", "gtf", "bed" ), help="input file format [%default].") parser.add_option( "--method", dest="methods", type="choice", action="append", choices=( "all", "hist", "stats", "overlaps", "values" ), help="methods to apply [%default].") parser.add_option( "--data", dest="data", type="choice", choices=( "all", "size", "distance" ), help="data to compute [%default].") parser.set_defaults( no_empty_bins = True, bin_size = None, dynamic_bins = False, ignore_out_of_range = False, min_value = None, max_value = None, nonull = None, missing_value = "na", output_filename_pattern="%s", methods = [], data = "all", format = "gff", ) (options, args) = E.Start( parser, add_output_options = True ) if "all" in options.methods: options.methods = ("hist", "stats", "overlaps") if not options.output_filename_pattern: options.output_filename_pattern = "%s" if len(options.methods) == 0: raise ValueError( "please provide counting method using --method option" ) if options.format in ( "gff", "gtf" ): gffs = GTF.iterator( options.stdin ) elif options.format == "bed": gffs = Bed.iterator( options.stdin ) values_between = [] values_within = [] values_overlaps = [] if "overlaps" in options.methods: if not options.output_filename_pattern: options.output_filename_pattern = "%s" outfile_overlaps = E.openOutputFile( "overlaps" ) else: outfile_overlaps = None last = None ninput, noverlaps = 0,0 for this in gffs: ninput += 1 values_within.append( this.end - this.start ) if last and last.contig == this.contig: if this.start < last.end: noverlaps += 1 if outfile_overlaps: outfile_overlaps.write( "%s\t%s\n" % (str(last), str(this)) ) values_overlaps.append( min(this.end, last.end) - max(last.start, this.start) ) if this.end > last.end: last = this continue else: values_between.append( this.start - last.end ) # if this.start - last.end < 10: # print str(last) # print str(this) # print "==" values_overlaps.append( 0 ) last = this if "hist" in options.methods: outfile = E.openOutputFile( "hist" ) h_within = Histogram.Calculate( values_within, no_empty_bins = options.no_empty_bins, increment = options.bin_size, min_value = options.min_value, max_value = options.max_value, dynamic_bins = options.dynamic_bins, ignore_out_of_range = options.ignore_out_of_range ) h_between = Histogram.Calculate( values_between, no_empty_bins = options.no_empty_bins, increment = options.bin_size, min_value = options.min_value, max_value = options.max_value, dynamic_bins = options.dynamic_bins, ignore_out_of_range = options.ignore_out_of_range ) if "all" == options.data: outfile.write( "residues\tsize\tdistance\n" ) combined_histogram = Histogram.Combine( [h_within, h_between], missing_value = options.missing_value ) Histogram.Write( outfile, combined_histogram, nonull = options.nonull ) elif options.data == "size": outfile.write( "residues\tsize\n" ) Histogram.Write( outfile, h_within, nonull = options.nonull ) elif options.data == "distance": outfile.write( "residues\tdistance\n" ) Histogram.Write( outfile, h_between, nonull = options.nonull ) outfile.close() if "stats" in options.methods: outfile = E.openOutputFile( "stats" ) outfile.write( "data\t%s\n" % Stats.Summary().getHeader() ) if options.data in ("size", "all"): outfile.write( "size\t%s\n" % str(Stats.Summary(values_within)) ) if options.data in ("distance", "all"): outfile.write( "distance\t%s\n" % str(Stats.Summary(values_between)) ) outfile.close() if "values" in options.methods: outfile = E.openOutputFile( "distances" ) outfile.write( "distance\n%s\n" % "\n".join( map(str, values_between) ) ) outfile.close() outfile = E.openOutputFile( "sizes" ) outfile.write( "size\n%s\n" % "\n".join( map(str, values_within) ) ) outfile.close() outfile = E.openOutputFile( "overlaps" ) outfile.write( "overlap\n%s\n" % "\n".join( map(str, values_overlaps) ) ) outfile.close() E.info( "ninput=%i, ndistance=%i, nsize=%i, noverlap=%i" % (ninput, len(values_between), len(values_within), noverlaps) ) E.Stop()
def main(argv=sys.argv): parser = E.OptionParser(version="%prog version: $Id: bed2bed.py 2861 2010-02-23 17:36:32Z andreas $", usage=globals()["__doc__"]) # IMS: new method: extend intervals by set amount parser.add_option("-m", "--method", dest="methods", type="choice", action="append", choices=("merge", "filter-genome", "bins", "block", "sanitize-genome", "shift", "extend"), help="method to apply [default=%default]") parser.add_option("--num-bins", dest="num_bins", type="int", help="number of bins into which to merge (used for method `bins) [default=%default]") parser.add_option("--bin-edges", dest="bin_edges", type="string", help="bin_edges for binning method [default=%default]") parser.add_option("--binning-method", dest="binning_method", type="choice", choices=( "equal-bases", "equal-intervals", "equal-range"), help="method used for binning (used for method `bins` if no bin_edges is given) [default=%default]") parser.add_option("--merge-distance", dest="merge_distance", type="int", help="distance in bases over which to merge that are not directly adjacent [default=%default]") parser.add_option("--merge-min-intervals", dest="merge_min_intervals", type="int", help="only output merged intervals that are build from at least x intervals [default=%default]") parser.add_option("--merge-by-name", dest="merge_by_name", action="store_true", help="only merge intervals with the same name [default=%default]") parser.add_option("--remove-inconsistent", dest="remove_inconsistent", action="store_true", help="when merging, do not output intervals where the names of overlapping intervals " "do not match [default=%default]") parser.add_option("--offset", dest="offset", type="int", help="offset for shifting intervals [default=%default]") parser.add_option("-g", "--genome-file", dest="genome_file", type="string", help="filename with genome.") parser.add_option("-b", "--bam-file", dest="bam_file", type="string", help="bam-formatted filename with genome.") parser.set_defaults(methods=[], merge_distance=0, binning_method="equal-bases", merge_by_name=False, genome_file=None, bam_file=None, num_bins=5, merge_min_intervals=1, bin_edges=None, offset=10000, test=None, extend_distance=1000, remove_inconsistent=False) (options, args) = E.Start(parser, add_pipe_options=True) contigs = None # Why provide full indexed genome, when a tsv of contig sizes would do? if options.genome_file: genome_fasta = IndexedFasta.IndexedFasta(options.genome_file) contigs = genome_fasta.getContigSizes() if options.bam_file: samfile = pysam.Samfile(options.bam_file) contigs = dict(zip(samfile.references, samfile.lengths)) processor = Bed.iterator(options.stdin) for method in options.methods: if method == "filter-genome": if not contigs: raise ValueError("please supply contig sizes") processor = filterGenome(processor, contigs) elif method == "sanitize-genome": if not contigs: raise ValueError("please supply contig sizes") processor = sanitizeGenome(processor, contigs) elif method == "merge": processor = merge(processor, options.merge_distance, by_name=options.merge_by_name, min_intervals=options.merge_min_intervals, remove_inconsistent=options.remove_inconsistent) elif method == "bins": if options.bin_edges: bin_edges = map(float, options.bin_edges.split(",")) # IMS: check bin edges are valid if not(len(bin_edges) == options.num_bins + 1): raise ValueError( "Number of bin edge must be one more than number of bins") else: bin_edges = None processor, bin_edges = Bed.binIntervals(processor, num_bins=options.num_bins, method=options.binning_method, bin_edges=bin_edges) E.info("# split bed: bin_edges=%s" % (str(bin_edges))) elif method == "block": processor = Bed.blocked_iterator(processor) elif method == "shift": # IMS: test that contig sizes are availible if not contigs: raise ValueError("please supply genome file") processor = shiftIntervals( processor, contigs, offset=options.offset) # IMS: new method: extend intervals by set amount elif method == "extend": if not contigs: raise ValueError("please supply genome file") processor = extendInterval(processor, contigs, options.offset) noutput = 0 for bed in processor: options.stdout.write(str(bed) + "\n") noutput += 1 E.info("noutput=%i" % (noutput)) E.Stop()
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if not argv: argv = sys.argv # setup command line parser parser = E.OptionParser(version="%prog version: $Id", usage=globals()["__doc__"]) parser.add_option("--bed-file", dest="infiles", type="string", metavar="bed", help="supply list of bed files", action="append") parser.set_defaults(infiles=[]) # add common options (-h/--help, ...) and parse command line (options, args) = E.start(parser, argv=argv) options.infiles.extend(args) if len(options.infiles) == 0: raise ValueError('please provide at least 1 bed file') E.info("concatenating bed files") # concatenate the list of files tmp = tempfile.NamedTemporaryFile(delete=False, mode="w") tmp_merge = tempfile.NamedTemporaryFile(delete=False, mode="w") infs = options.infiles for inf in infs: for bed in Bed.iterator(IOTools.open_file(inf)): tmp.write("%s\n" % bed) tmp.close() E.info("merging bed entries") # merge the bed entries in the file name = tmp.name tmp_bed = pybedtools.BedTool(name) tmp_bed.sort().merge().saveas(tmp_merge.name) tmp_merge.close() E.info("indexing bed entries") # index the bed entries merged = IndexedGenome.Simple() for bed in Bed.iterator(IOTools.open_file(tmp_merge.name)): merged.add(bed.contig, bed.start, bed.end) counts = collections.defaultdict(int) # list of samples samples = options.infiles E.info("counting no. samples overlapping each interval") for sample in samples: found = set() for bed in Bed.iterator(IOTools.open_file(sample)): if merged.contains(bed.contig, bed.start, bed.end): key = [bed.contig] + \ [x for x in merged.get(bed.contig, bed.start, bed.end)] key = (key[0], key[1][0], key[1][1]) if key in found: continue found.add(key) # tuple of interval description as key - (contig, start, end) counts[key] += 1 # open outfile options.stdout.write("contig\tstart\tend\tcount\n") E.info("outputting result") for interval, count in sorted(counts.items()): options.stdout.write("\t".join(map(str, interval)) + "\t" + str(count) + "\n") # write footer and output benchmark information. E.stop()
def main(argv=sys.argv): # setup command line parser parser = E.OptionParser( version="%prog version: $Id$", usage=globals()["__doc__"]) parser.add_option("-n", "--new-instance", dest="new_instance", action="store_true", help="create a new IGV instance [%default]") parser.add_option("-s", "--session", dest="session", type="string", help="load session before creating plots " "[%default]") parser.add_option("-d", "--snapshot-dir", dest="snapshotdir", type="string", help="directory to save snapshots in [%default]") parser.add_option("-f", "--format", dest="format", type="choice", choices=("png", "eps", "svg"), help="output file format [%default]") parser.add_option("-o", "--host", dest="host", type="string", help="host that IGV is running on [%default]") parser.add_option("-p", "--port", dest="port", type="int", help="port that IGV listens at [%default]") parser.add_option("-e", "--extend", dest="extend", type="int", help="extend each interval by a number of bases " "[%default]") parser.add_option("-x", "--expand", dest="expand", type="float", help="expand each region by a certain factor " "[%default]") parser.add_option("--session-only", dest="session_only", action="store_true", help="plot session after opening, " "ignore intervals " "[%default]") parser.add_option("--keep", dest="keep_open", action="store_true", help="keep a newly created IGV session open " "[%default]") parser.set_defaults( command="igv.sh", host='127.0.0.1', port=61111, snapshotdir=os.getcwd(), extend=0, format="png", expand=1.0, session=None, session_only=False, keep_open=False, ) # add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv) igv_process = None if options.new_instance: E.info("starting new IGV process") igv_process = IGV.startIGV(command=options.command, port=options.port) E.info("new IGV process started") E.info("connection to process on %s:%s" % (options.host, options.port)) E.info("saving images in %s" % options.snapshotdir) igv = IGV.IGV(host=options.host, port=options.port, snapshot_dir=os.path.abspath(options.snapshotdir)) if options.session: E.info('loading session from %s' % options.session) igv.load(options.session) E.info('loaded session') if options.session_only: E.info('plotting session only ignoring any intervals') fn = "%s.%s" % (os.path.basename(options.session), options.format) E.info("writing snapshot to '%s'" % os.path.join(options.snapshotdir, fn)) igv.save(fn) else: c = E.Counter() for bed in Bed.iterator(options.stdin): c.input += 1 # IGV can not deal with white-space in filenames name = re.sub("\s", "_", bed.name) E.info("going to %s:%i-%i for %s" % (bed.contig, bed.start, bed.end, name)) start, end = bed.start, bed.end extend = options.extend if options.expand: d = end - start extend = max(extend, (options.expand * d - d) // 2) start -= extend end += extend igv.go("%s:%i-%i" % (bed.contig, start, end)) fn = "%s.%s" % (name, options.format) E.info("writing snapshot to '%s'" % fn) igv.save(fn) c.snapshots += 1 E.info(c) if igv_process is not None and not options.keep_open: E.info('shutting down IGV') igv_process.send_signal(signal.SIGKILL) E.Stop()
def main(argv=None): if argv is None: argv = sys.argv parser = E.OptionParser( version="%prog version: $Id: gff2fasta.py 2861 2010-02-23 17:36:32Z andreas $") parser.add_option("-g", "--genome-file", dest="genome_file", type="string", help="filename with genome.") parser.add_option("-m", "--masker", dest="masker", type="choice", choices=("dust", "dustmasker", "softmask", "none"), help="apply masker [%default].") parser.add_option("-o", "--mode", dest="mode", type="choice", choices=("intervals", "leftright"), help="what to output [%default]") parser.add_option("--min-length", dest="min_length", type="int", help="require a minimum sequence length [%default]") parser.add_option("--max-length", dest="max_length", type="int", help="require a maximum sequence length [%default]") parser.add_option("--extend-at", dest="extend_at", type="choice", choices=("none", "3", "5", "both", "3only", "5only"), help="extend at no, 3', 5' or both ends. If 3only or 5only are set, only the added sequence is returned [default=%default]") parser.add_option("--extend-by", dest="extend_by", type="int", help="extend by # bases [default=%default]") parser.add_option("--use-strand", dest="ignore_strand", action="store_false", help="use strand information and return reverse complement [default=%default]") parser.set_defaults( genome_file=None, masker=None, mode="intervals", min_length=0, max_length=0, extend_at=None, extend_by=100, ignore_strand=True, ) (options, args) = E.Start(parser) if options.genome_file: fasta = IndexedFasta.IndexedFasta(options.genome_file) contigs = fasta.getContigSizes() fasta.setConverter(IndexedFasta.getConverter("zero-both-open")) counter = E.Counter() ids, seqs = [], [] E.info("collecting sequences") for bed in Bed.setName(Bed.iterator(options.stdin)): counter.input += 1 lcontig = fasta.getLength(bed.contig) if options.ignore_strand: strand = "+" else: strand = bed.strand if options.mode == "intervals": ids.append("%s %s:%i..%i (%s)" % (bed.name, bed.contig, bed.start, bed.end, strand)) seqs.append( fasta.getSequence(bed.contig, strand, bed.start, bed.end)) elif options.mode == "leftright": l = bed.end - bed.start start, end = max(0, bed.start - l), bed.end - l ids.append("%s_l %s:%i..%i (%s)" % (bed.name, bed.contig, start, end, strand)) seqs.append(fasta.getSequence(bed.contig, strand, start, end)) start, end = bed.start + l, min(lcontig, bed.end + l) ids.append("%s_r %s:%i..%i (%s)" % (bed.name, bed.contig, start, end, strand)) seqs.append(fasta.getSequence(bed.contig, strand, start, end)) E.info("collected %i sequences" % len(seqs)) masked = Masker.maskSequences(seqs, options.masker) options.stdout.write( "\n".join([">%s\n%s" % (x, y) for x, y in zip(ids, masked)]) + "\n") E.info("masked %i sequences" % len(seqs)) counter.output = len(seqs) E.info("%s" % counter) E.Stop()
def main(argv=None): if argv is None: argv = sys.argv parser = E.OptionParser( version="%prog version: $Id$", usage=globals()["__doc__"]) parser.add_option("-g", "--genome-file", dest="genome_file", type="string", help="filename with genomic sequence to retrieve " "sequences from.") parser.add_option("-m", "--masker", dest="masker", type="choice", choices=("dust", "dustmasker", "softmask", "none"), help="apply masker to mask output sequences " "[%default].") parser.add_option("--output-mode", dest="output_mode", type="choice", choices=("intervals", "leftright", "segments"), help="what to output. " "'intervals' generates a single sequence for " "each bed interval. 'leftright' generates two " "sequences, one in each direction, for each bed " "interval. 'segments' can be used to output " "sequence from bed12 files so that sequence only covers " "the segements [%default]") parser.add_option("--min-sequence-length", dest="min_length", type="int", help="require a minimum sequence length [%default]") parser.add_option("--max-sequence-length", dest="max_length", type="int", help="require a maximum sequence length [%default]") parser.add_option( "--extend-at", dest="extend_at", type="choice", choices=("none", "3", "5", "both", "3only", "5only"), help="extend at 3', 5' or both or no ends. If 3only or 5only " "are set, only the added sequence is returned [default=%default]") parser.add_option( "--extend-by", dest="extend_by", type="int", help="extend by # bases [default=%default]") parser.add_option( "--use-strand", dest="ignore_strand", action="store_false", help="use strand information and return reverse complement " "on intervals located on the negative strand. " "[default=%default]") parser.set_defaults( genome_file=None, masker=None, output_mode="intervals", min_length=0, max_length=0, extend_at=None, extend_by=100, ignore_strand=True, ) (options, args) = E.Start(parser) if options.genome_file: fasta = IndexedFasta.IndexedFasta(options.genome_file) contigs = fasta.getContigSizes() fasta.setConverter(IndexedFasta.getConverter("zero-both-open")) counter = E.Counter() ids, seqs = [], [] E.info("collecting sequences") for bed in Bed.setName(Bed.iterator(options.stdin)): counter.input += 1 lcontig = fasta.getLength(bed.contig) if options.ignore_strand: strand = "+" else: strand = bed.strand if options.output_mode == "segments" and bed.columns == 12: ids.append("%s %s:%i..%i (%s) %s %s" % (bed.name, bed.contig, bed.start, bed.end, strand, bed["blockSizes"], bed["blockStarts"])) seg_seqs = [fasta.getSequence(bed.contig, strand, start, end) for start, end in bed.toIntervals()] seqs.append("".join(seg_seqs)) elif (options.output_mode == "intervals" or options.output_mode == "segments"): ids.append("%s %s:%i..%i (%s)" % (bed.name, bed.contig, bed.start, bed.end, strand)) seqs.append( fasta.getSequence(bed.contig, strand, bed.start, bed.end)) elif options.output_mode == "leftright": l = bed.end - bed.start start, end = max(0, bed.start - l), bed.end - l ids.append("%s_l %s:%i..%i (%s)" % (bed.name, bed.contig, start, end, strand)) seqs.append(fasta.getSequence(bed.contig, strand, start, end)) start, end = bed.start + l, min(lcontig, bed.end + l) ids.append("%s_r %s:%i..%i (%s)" % (bed.name, bed.contig, start, end, strand)) seqs.append(fasta.getSequence(bed.contig, strand, start, end)) E.info("collected %i sequences" % len(seqs)) masked = Masker.maskSequences(seqs, options.masker) options.stdout.write( "\n".join([">%s\n%s" % (x, y) for x, y in zip(ids, masked)]) + "\n") E.info("masked %i sequences" % len(seqs)) counter.output = len(seqs) E.info("%s" % counter) E.Stop()
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if not argv: argv = sys.argv # setup command line parser parser = E.OptionParser(version="%prog version: $Id$", usage=globals()["__doc__"]) parser.add_option("-m", "--min-overlap", dest="min_overlap", type="float", help="minimum overlap [%default]") parser.add_option("-a", "--bam-file", dest="filename_bam", metavar="bam", type="string", help="bam-file to use (required) [%default]") parser.add_option("-b", "--bed-file", dest="filename_bed", metavar="bed", type="string", help="bed-file to use (required) [%default]") parser.add_option("-s", "--sort-bed", dest="sort_bed", action="store_true", help="sort the bed file by chromosomal location before " "processing. " "[%default]") parser.add_option( "--assume-sorted", dest="sort_bed", action="store_false", help="assume that the bed-file is sorted by chromosomal location. " "[%default]") parser.add_option( "--split-intervals", dest="split_intervals", action="store_true", help="treat split BAM intervals, for example spliced intervals, " "as separate intervals. Note that a single alignment might be " "counted several times as a result. " "[%default]") parser.set_defaults( min_overlap=0.5, filename_bam=None, filename_bed=None, sort_bed=True, split_intervals=False, ) # add common options (-h/--help, ...) and parse command line (options, args) = E.start(parser, argv=argv) filename_bam = options.filename_bam filename_bed = options.filename_bed if filename_bam is None and filename_bed is None: if len(args) != 2: raise ValueError( "please supply a bam and a bed file or two bed-files.") filename_bam, filename_bed = args if filename_bed is None: raise ValueError("please supply a bed file to compare to.") if filename_bam is None: raise ValueError("please supply a bam file to compare with.") E.info("intersecting the two files") min_overlap = options.min_overlap options.stdout.write("category\talignments\n") # get number of columns of reference bed file for bed in Bed.iterator(IOTools.open_file(filename_bed)): ncolumns_bed = bed.columns break E.info("assuming %s is bed%i format" % (filename_bed, ncolumns_bed)) if ncolumns_bed < 4: raise ValueError("please supply a name attribute in the bed file") # get information about if filename_bam.endswith(".bam"): format = "-abam" samfile = pysam.AlignmentFile(filename_bam, "rb") total = samfile.mapped # latest bedtools uses bed12 format when bam is input ncolumns_bam = 12 # count per read sort_key = lambda x: x.name else: format = "-a" total = IOTools.get_num_lines(filename_bam) # get bed format ncolumns_bam = 0 for bed in Bed.iterator(IOTools.open_file(filename_bam)): ncolumns_bam = bed.columns break if ncolumns_bam > 0: E.info("assuming %s is bed%i fomat" % (filename_bam, ncolumns_bam)) if ncolumns_bam == 3: # count per interval sort_key = lambda x: (x.contig, x.start, x.end) else: # count per interval category sort_key = lambda x: x.name # use fields for bam/bed file (regions to count with) data_fields = [ "contig", "start", "end", "name", "score", "strand", "thickstart", "thickend", "rgb", "blockcount", "blockstarts", "blockends" ][:ncolumns_bam] # add fields for second bed (regions to count in) data_fields.extend([ "contig2", "start2", "end2", "name2", "score2", "strand2", "thickstart2", "thickend2", "rgb2", "blockcount2", "blockstarts2", "blockends2" ][:ncolumns_bed]) # add bases overlap data_fields.append("bases_overlap") data = collections.namedtuple("data", data_fields) options.stdout.write("total\t%i\n" % total) if total == 0: E.warn("no data in %s" % filename_bam) return # SNS: sorting optional, off by default if options.sort_bed: bedcmd = "<( gunzip < %s | sort -k1,1 -k2,2n)" % filename_bed else: bedcmd = filename_bed if options.split_intervals: split = "-split" else: split = "" # IMS: newer versions of intersectBed have a very high memory # requirement unless passed sorted bed files. statement = """bedtools intersect %(format)s %(filename_bam)s -b %(bedcmd)s %(split)s -sorted -bed -wo -f %(min_overlap)f""" % locals() E.info("starting counting process: %s" % statement) proc = E.run(statement, return_popen=True, stdout=subprocess.PIPE) E.info("counting") counts_per_alignment = collections.defaultdict(int) take_columns = len(data._fields) def iterate(infile): for line in infile: if not line.strip(): continue yield data._make(line[:-1].split()[:take_columns]) for read, overlaps in itertools.groupby(iterate( IOTools.force_str(proc.stdout)), key=sort_key): annotations = [x.name2 for x in overlaps] for anno in annotations: counts_per_alignment[anno] += 1 for key, counts in sorted(counts_per_alignment.items()): options.stdout.write("%s\t%i\n" % (key, counts)) # write footer and output benchmark information. E.stop()
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ parser = buildOptionParser(argv) # add common options (-h/--help, ...) and parse command line (options, args) = E.start(parser, argv=argv, add_output_options=True) if len(args) != 2: raise ValueError( "please specify one bam- or wig-file and one bed file") if options.control_files: E.info("using control files: %s" % ",".join(options.control_files)) infile, bedfile = args control_files = [] if options.format == "bigwig": fg_file = pyBigWig.open(infile) for control_file in options.control_files: control_files.append(pyBigWig.open(control_file)) counter = _bam2peakshape.CounterBigwig( smooth_method=options.smooth_method) elif options.format == "bam": fg_file = pysam.AlignmentFile(infile, "rb") for control_file in options.control_files: control_files.append(pysam.AlignmentFile(control_file, "rb")) counter = _bam2peakshape.CounterBam( shift=options.shift, smooth_method=options.smooth_method) features_per_interval, bins = buildDensityMatrices( Bed.iterator(IOTools.open_file(bedfile)), fg_file, control_files, counter, window_size=options.window_size, bin_size=options.bin_size, strand_specific=options.strand_specific, centring_method=options.centring_method, use_interval=options.use_interval, random_shift=options.random_shift, smooth_method=options.smooth_method, report_step=options.report_step) if len(features_per_interval) == 0: E.warn("no data - no output") E.stop() return outputFeatureTable(options.stdout, features_per_interval, bins) # apply normalization # Note: does not normalize control? # Needs reworking, currently it does not normalize across # all samples nor does the work "sum" reflect the per million # normalization. if options.normalization == "sum": E.info("starting sum normalization") # get total counts across all intervals norm = 0.0 for foreground, bed, controls, shifted in features_per_interval: norm += sum(foreground.counts) # per million norm /= float(1000000) E.info("sum/million normalization with %f" % norm) # normalise new_data = [] for foreground, bed, controls, shifted in features_per_interval: foreground = foreground._replace( counts=numpy.array(foreground.counts, dtype=numpy.float) / norm) new_controls = [] for control in controls: new_controls.append( control._replace( counts=numpy.array(control.counts, dtype=numpy.float) / norm)) if shifted: shifted = shifted._replace( counts=numpy.array(shifted.counts, dtype=numpy.float) / norm) new_data.append( IntervalData._make((foreground, bed, new_controls, shifted))) features_per_interval = new_data else: E.info("no normalization performed") # center bins out_bins = bins[:-1] + options.bin_size # build tracks def _toTrack(filename): return os.path.splitext(os.path.basename(filename))[0] outputMatrices(features_per_interval, out_bins, foreground_track=_toTrack(infile), control_tracks=[_toTrack(x) for x in options.control_files], shifted=options.random_shift, sort_orders=options.sort_orders) # write footer and output benchmark information. E.stop()
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if not argv: argv = sys.argv # setup command line parser parser = E.OptionParser(version="%prog version: $Id$", usage=globals()["__doc__"]) parser.add_option( "-g", "--gtf-file", dest="filename_gtf", type="string", help="filename with gene models in gtf format [%default]") parser.add_option( "-m", "--filename-mismapped", dest="filename_mismapped", type="string", help="output bam file for mismapped reads [%default]") parser.add_option( "-j", "--junctions-bed-file", dest="filename_junctions", type="string", help="bam file with reads mapped across junctions [%default]") parser.add_option( "-r", "--filename-regions", dest="filename_regions", type="string", help="filename with regions to remove in bed format [%default]") parser.add_option( "-t", "--transcripts-gtf-file", dest="filename_transcriptome", type="string", help="bam file with reads mapped against transcripts [%default]") parser.add_option( "-p", "--map-tsv-file", dest="filename_map", type="string", help="filename mapping transcript numbers (used by " "--filename-transciptome) to transcript names " "(used by --filename-gtf) [%default]") parser.add_option( "-s", "--filename-stats", dest="filename_stats", type="string", help="filename to output stats to [%default]") parser.add_option( "-o", "--colour", dest="colour_mismatches", action="store_true", help="mismatches will use colour differences (CM tag) [%default]") parser.add_option( "-i", "--ignore-mismatches", dest="ignore_mismatches", action="store_true", help="ignore mismatches [%default]") parser.add_option( "-c", "--remove-contigs", dest="remove_contigs", type="string", help="','-separated list of contigs to remove [%default]") parser.add_option( "-f", "--force-output", dest="force", action="store_true", help="force overwriting of existing files [%default]") parser.add_option("-u", "--unique", dest="unique", action="store_true", help="remove reads not matching uniquely [%default]") parser.add_option("--output-sam", dest="output_sam", action="store_true", help="output in sam format [%default]") parser.set_defaults( filename_gtf=None, filename_mismapped=None, filename_junctions=None, filename_transcriptome=None, filename_map=None, remove_contigs=None, force=False, unique=False, colour_mismatches=False, ignore_mismatches=False, output_sam=False, filename_table=None, ) # add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv) if len(args) != 1: raise ValueError("please supply one bam file") bamfile_genome = args[0] genome_samfile = pysam.Samfile(bamfile_genome, "rb") if options.remove_contigs: options.remove_contigs = options.remove_contigs.split(",") if options.filename_map: E.info("reading map") id_map = IOTools.readMap( IOTools.openFile(options.filename_map), has_header=True) id_map = dict([(y, x) for x, y in id_map.iteritems()]) else: id_map = None transcripts = {} if options.filename_gtf: E.info("indexing geneset") mapped, missed = 0, 0 for gtf in GTF.transcript_iterator( GTF.iterator(IOTools.openFile(options.filename_gtf))): gtf.sort(key=lambda x: x.start) transcript_id = gtf[0].transcript_id if id_map: try: transcript_id = id_map[transcript_id] mapped += 1 except KeyError: missed += 1 continue transcripts[transcript_id] = gtf E.info("read %i transcripts from geneset (%i mapped, %i missed)" % (len(transcripts), mapped, missed)) regions_to_remove = None if options.filename_regions: E.info("indexing regions") regions_to_remove = IndexedGenome.Simple() for bed in Bed.iterator(IOTools.openFile(options.filename_regions)): regions_to_remove.add(bed.contig, bed.start, bed.end) E.info("read %i regions" % len(regions_to_remove)) if options.filename_transcriptome: transcripts_samfile = pysam.Samfile(options.filename_transcriptome, "rb") else: transcripts_samfile = None if options.output_sam: output_samfile = pysam.Samfile("-", "wh", template=genome_samfile) else: output_samfile = pysam.Samfile("-", "wb", template=genome_samfile) if options.filename_mismapped: if not options.force and os.path.exists(options.filename_mismapped): raise IOError("output file %s already exists" % options.filename_mismapped) output_mismapped = pysam.Samfile(options.filename_mismapped, "wb", template=genome_samfile) else: output_mismapped = None if options.filename_junctions: junctions_samfile = pysam.Samfile(options.filename_junctions, "rb") else: junctions_samfile = None c = _bams2bam.filter(genome_samfile, output_samfile, output_mismapped, transcripts_samfile, junctions_samfile, transcripts, regions=regions_to_remove, unique=options.unique, remove_contigs=options.remove_contigs, colour_mismatches=options.colour_mismatches, ignore_mismatches=options.ignore_mismatches, ignore_transcripts=transcripts_samfile is None, ignore_junctions=junctions_samfile is None) if options.filename_stats: outf = IOTools.openFile(options.filename_stats, "w") outf.write("category\tcounts\n%s\n" % c.asTable()) outf.close() if options.filename_transcriptome: transcripts_samfile.close() genome_samfile.close() output_samfile.close() if output_mismapped: output_mismapped.close() # write footer and output benchmark information. E.Stop()
def main( argv = None ): """script main. parses command line options in sys.argv, unless *argv* is given. """ if argv == None: argv = sys.argv parser = E.OptionParser( version = "%prog version: $Id: bed2annotator2tsv.py 2885 2010-04-07 08:46:50Z andreas $", usage = globals()["__doc__"]) parser.add_option( "-g", "--genome-file", dest="genome_file", type="string", help="filename with genome." ) parser.add_option( "-f", "--features", dest="features", type="string", help="feature to collect [default=None]." ) parser.add_option( "-i", "--files", dest="files", action="append", help="use multiple annotations [default=None]." ) parser.add_option( "-a", "--annotations", dest="annotations", type="string", help="aggregate name for annotations if only single file is provided from STDIN [default=None]." ) parser.add_option( "--input-filename-map", dest="input_filename_map", type="string", help="filename with a map of gene_ids to categories [default=None]." ) parser.add_option( "-l", "--max-length", dest="max_length", type="string", help="maximum segment length [default=None]." ) parser.add_option( "-m", "--merge", dest="merge", action="store_true", help="merge overlapping bed segments [default=%default]." ) parser.add_option( "-s", "--section", dest="section", type="choice", choices=("segments", "annotations", "workspace" ), help="annotator section [default=None]." ) parser.add_option( "--subset", dest="subsets", type="string", action="append", help="add filenames to delimit subsets within the gff files. The syntax is filename.gff,label,filename.ids [default=None]." ) parser.set_defaults( genome_file = None, feature = None, remove_random = True, section = "segments", annotations = "annotations", max_length = 100000, files = [], subsets = [], input_filename_map = None, merge = False, ) (options, args) = E.Start( parser ) options.files += args if len(options.files) == 0: options.files.append("-") options.files = list( itertools.chain( *[ re.split( "[,; ]+", x) for x in options.files ] ) ) if options.subsets: subsets = collections.defaultdict( list ) for s in options.subsets: filename_gff,label,filename_ids = s.split( "," ) subsets[filename_gff].append( (label,filename_ids) ) options.subsets = subsets if options.genome_file: fasta = IndexedFasta.IndexedFasta( options.genome_file ) else: fasta = None if options.section == "segments": prefix = "##Segs" elif options.section == "annotations": prefix = "##Id" elif options.section == "workspace": prefix = "##Work" else: raise ValueError("unknown section %s" % options.section) if options.max_length: max_length = options.max_length else: max_length = 0 ninput, ntracks, ncontigs, nsegments, ndiscarded = 0, 0, 0, 0, 0 if options.section in ("annotations"): contigs = set() it = itertools.groupby( Bed.iterator( options.stdin ), key=lambda x: x.track["name"]) map_track2segments = {} for track, beds in it: ntracks += 1 map_track2segments[track] = [] first_segment = nsegments beds = list(beds) if options.merge: beds = Bed.merge( beds ) for bed in beds: contig, start, end = bed.contig, bed.start, bed.end if options.remove_random and "random" in contig: continue if max_length > 0 and end - start > max_length: ndiscarded += 1 continue contigs.add(contig) map_track2segments[track].append( nsegments ) options.stdout.write( "%s\t%i\t%s\t(%i,%i)\n" % (prefix, nsegments, contig, start, end ) ) nsegments += 1 options.stdout.write("##Ann\t%s\t%s\n" % (track, "\t".join( ["%i" % x for x in range(first_segment, nsegments) ] ) ) ) E.info( "track %s: annotated with %i segments" % (track, nsegments - first_segment) ) ncontigs = len(contigs) E.info( "ninput=%i, ntracks=%i, ncontigs=%i, nsegments=%i, ndiscarded=%i" % (ninput, ntracks, ncontigs, nsegments, ndiscarded)) E.Stop()
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if not argv: argv = sys.argv # setup command line parser parser = E.OptionParser(version="%prog version: $Id: cgat_script_template.py 2871 2010-03-03 10:20:44Z andreas $", usage=globals()["__doc__"]) parser.add_option("-i", "--infiles", dest="infiles", type="string", metavar="bed", action="append", help="supply list of bed files") parser.set_defaults(infiles=[]) # add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv) options.infiles.extend(args) if len(options.infiles) == 0: raise ValueError('please provide at least 1 bed file') E.info("concatenating bed files") # concatenate the list of files tmp = tempfile.NamedTemporaryFile(delete=False) tmp_merge = tempfile.NamedTemporaryFile(delete=False) infs = options.infiles for inf in infs: for bed in Bed.iterator(IOTools.openFile(inf)): tmp.write("%s\n" % bed) tmp.close() E.info("merging bed entries") # merge the bed entries in the file name = tmp.name tmp_bed = pybedtools.BedTool(name) tmp_bed.merge().saveas(tmp_merge.name) tmp_merge.close() E.info("indexing bed entries") # index the bed entries merged = IndexedGenome.Simple() for bed in Bed.iterator(IOTools.openFile(tmp_merge.name)): merged.add(bed.contig, bed.start, bed.end) counts = collections.defaultdict(int) # list of samples samples = options.infiles E.info("counting no. samples overlapping each interval") for sample in samples: found = set() for bed in Bed.iterator(IOTools.openFile(sample)): if merged.contains(bed.contig, bed.start, bed.end): key = [bed.contig] + \ [x for x in merged.get(bed.contig, bed.start, bed.end)] key = (key[0], key[1][0], key[1][1]) if key in found: continue found.add(key) # tuple of interval description as key - (contig, start, end) counts[key] += 1 # open outfile options.stdout.write("contig\tstart\tend\tcount\n") E.info("outputting result") for interval, count in counts.iteritems(): options.stdout.write( "\t".join(map(str, interval)) + "\t" + str(count) + "\n") # write footer and output benchmark information. E.Stop()
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if not argv: argv = sys.argv # setup command line parser parser = E.OptionParser(version="%prog version: $Id: cgat_script_template.py 2871 2010-03-03 10:20:44Z andreas $", usage=globals()["__doc__"]) parser.add_option("-m", "--min-overlap", dest="min_overlap", type="float", help="minimum overlap [%default]") parser.add_option("-k", "--keep-temp", dest="keep_temp", action="store_true", help="do not delete temporary files [%default]") parser.add_option("-a", "--filename-bam", dest="filename_bam", metavar="bam", type="string", help="bam-file to use [%default]") parser.add_option("-b", "--filename-bed", dest="filename_bed", metavar="bam", type="string", help="bed-file to use [%default]") parser.set_defaults( min_overlap=0.5, keep_temp=False, filename_bam=None, filename_bed=None, ) # add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv) filename_bam = options.filename_bam filename_bed = options.filename_bed if filename_bam is None and filename_bed is None: if len(args) != 2: raise ValueError( "please supply a bam and a bed file or two bed-files.") filename_bam, filename_bed = args if filename_bed is None: raise ValueError("please supply a bed file to compare to.") if filename_bam is None: raise ValueError("please supply a bam file to compare with.") E.info("intersecting the two files") tmpfile = tempfile.NamedTemporaryFile(delete=False) tmpfile.close() tmpfilename = tmpfile.name min_overlap = options.min_overlap options.stdout.write("category\talignments\n") # get number of columns of reference bed file for bed in Bed.iterator(IOTools.openFile(filename_bed)): ncolumns_bed = bed.columns break E.info("assuming %s is bed%i format" % (filename_bed, ncolumns_bed)) if ncolumns_bed < 4: raise ValueError("please supply a name attribute in the bed file") # get information about if filename_bam.endswith(".bam"): format = "-abam" samfile = pysam.Samfile(filename_bam, "rb") total = samfile.mapped # latest bedtools uses bed12 format when bam is input ncolumns_bam = 12 # count per read sort_key = lambda x: x.name else: format = "-a" total = IOTools.getNumLines(filename_bam) # get bed format ncolumns_bam = 0 for bed in Bed.iterator(IOTools.openFile(filename_bam)): ncolumns_bam = bed.columns break if ncolumns_bam > 0: E.info("assuming %s is bed%i fomat" % (filename_bam, ncolumns_bam)) if ncolumns_bam == 3: # count per interval sort_key = lambda x: (x.contig, x.start, x.end) else: # count per interval category sort_key = lambda x: x.name # use fields for bam/bed file (regions to count with) data_fields = ["contig", "start", "end", "name", "score", "strand", "thickstart", "thickend", "rgb", "blockcount", "blockstarts", "blockends"][:ncolumns_bam] # add fields for second bed (regions to count in) data_fields.extend(["contig2", "start2", "end2", "name2", "score2", "strand2", "thickstart2", "thickend2", "rgb2", "blockcount2", "blockstarts2", "blockends2"][:ncolumns_bed]) # add bases overlap data_fields.append("bases_overlap") data = collections.namedtuple("data", data_fields) options.stdout.write("total\t%i\n" % total) if total == 0: E.warn("no data in %s" % filename_bam) return # IMS: newer versions of intersectBed have a very high memory requirement unless # passed sorted bed files. statement = """intersectBed %(format)s %(filename_bam)s -b <( zcat %(filename_bed)s | sort -k1,1 -k2,2n) -sorted -bed -wo -f %(min_overlap)f > %(tmpfilename)s""" % locals() E.info("running %s" % statement) retcode = E.run(statement) if retcode != 0: raise ValueError("error while executing statement %s" % statement) infile = open(tmpfilename, "r") counts_per_alignment = collections.defaultdict(int) E.info("counting") take_columns = len(data._fields) def iter(infile): for line in infile: if not line.strip(): continue yield data._make(line[:-1].split()[:take_columns]) for read, overlaps in itertools.groupby(iter(infile), key=sort_key): annotations = [x.name2 for x in overlaps] for anno in annotations: counts_per_alignment[anno] += 1 infile.close() for key, counts in counts_per_alignment.iteritems(): options.stdout.write("%s\t%i\n" % (key, counts)) if not options.keep_temp: os.unlink(tmpfilename) # write footer and output benchmark information. E.Stop()