def testNoOverlap(self):
"""test empty input."""
self.assertEqual(
Intervals.truncate([(0, 5), (10, 15)], [(5, 10)]), [(0, 5), (10, 15)])
self.assertEqual(
Intervals.truncate([(5, 10)], [(0, 5), (10, 15)]), [(5, 10)])
self.assertEqual(
Intervals.truncate([(0, 5), (5, 10)], [(10, 15)]), [(0, 5), (5, 10)])
def testNoOverlap(self):
"""test empty input."""
self.assertEqual(Intervals.truncate([(0, 5), (10, 15)], [(5, 10)]),
[(0, 5), (10, 15)])
self.assertEqual(Intervals.truncate([(5, 10)], [(0, 5), (10, 15)]),
[(5, 10)])
self.assertEqual(Intervals.truncate([(0, 5), (5, 10)], [(10, 15)]),
[(0, 5), (5, 10)])
def testMultiple(self):
"""test empty input."""
self.assertEqual(
Intervals.truncate([(0, 5), (10, 15)], [(0, 5)]), [(10, 15)])
self.assertEqual(
Intervals.truncate([(0, 5), (10, 15)], [(0, 10)]), [(10, 15)])
self.assertEqual(Intervals.truncate([(0, 5), (10, 15)], [(0, 15)]), [])
self.assertEqual(Intervals.truncate([(0, 5), (5, 10)], [(0, 10)]), [])
self.assertEqual(
Intervals.truncate([(0, 5), (5, 10)], []), [(0, 5), (5, 10)])
def testMultiple(self):
"""test empty input."""
self.assertEqual(Intervals.truncate([(0, 5), (10, 15)], [(0, 5)]),
[(10, 15)])
self.assertEqual(Intervals.truncate([(0, 5), (10, 15)], [(0, 10)]),
[(10, 15)])
self.assertEqual(Intervals.truncate([(0, 5), (10, 15)], [(0, 15)]), [])
self.assertEqual(Intervals.truncate([(0, 5), (5, 10)], [(0, 10)]), [])
self.assertEqual(Intervals.truncate([(0, 5), (5, 10)], []), [(0, 5),
(5, 10)])
def testSingle(self):
"""test empty input."""
self.assertEqual(Intervals.truncate([(0, 5)], [(0, 5)]), [])
self.assertEqual(Intervals.truncate([(0, 5)], [(0, 3)]), [(3, 5)])
self.assertEqual(Intervals.truncate([(0, 3)], [(0, 5)]), [])
self.assertEqual(Intervals.truncate([(0, 5)], [(3, 5)]), [(0, 3)])
self.assertEqual(Intervals.truncate([(3, 5)], [(0, 5)]), [])
self.assertEqual(Intervals.truncate([(5, 10)], [(5, 10)]), [])
self.assertEqual(Intervals.truncate([(5, 10)], [(5, 20)]), [])
self.assertEqual(Intervals.truncate([(5, 10)], [(0, 10)]), [])
self.assertEqual(Intervals.truncate([(5, 10)], [(0, 10)]), [])
self.assertEqual(Intervals.truncate([(5, 10)], [(0, 20)]), [])
def testSingle(self):
"""test empty input."""
self.assertEqual(Intervals.truncate([(0, 5)], [(0, 5)]), [])
self.assertEqual(Intervals.truncate([(0, 5)], [(0, 3)]), [(3, 5)])
self.assertEqual(Intervals.truncate([(0, 3)], [(0, 5)]), [])
self.assertEqual(Intervals.truncate([(0, 5)], [(3, 5)]), [(0, 3)])
self.assertEqual(Intervals.truncate([(3, 5)], [(0, 5)]), [])
self.assertEqual(Intervals.truncate([(5, 10)], [(5, 10)]), [])
self.assertEqual(Intervals.truncate([(5, 10)], [(5, 20)]), [])
self.assertEqual(Intervals.truncate([(5, 10)], [(0, 10)]), [])
self.assertEqual(Intervals.truncate([(5, 10)], [(0, 10)]), [])
self.assertEqual(Intervals.truncate([(5, 10)], [(0, 20)]), [])
def UTR5(transcript):
exons = GTF.asRanges(transcript, "exon")
cds = GTF.asRanges(transcript, "CDS")
utrs = Intervals.truncate(exons, cds)
if len(cds) == 0:
return list()
if transcript[0].strand == "-":
utr3 = [exon for exon in utrs if exon[0] >= cds[-1][1]]
else:
utr3 = [exon for exon in utrs if exon[-1] <= cds[0][0]]
for e in transcript:
if e.feature == "exon":
template_exon = e
break
returned_exons = []
for e in utr3:
gtf = GTF.Entry().fromGTF(template_exon)
gtf.start = e[0]
gtf.end = e[1]
returned_exons.append(gtf)
return returned_exons
def UTR3(transcript):
exons = GTF.asRanges(transcript, "exon")
cds = GTF.asRanges(transcript, "CDS")
if len(cds) == 0:
return list()
utrs = Intervals.truncate(exons, cds)
if transcript[0].strand == "+":
utr3 = [exon for exon in utrs
if exon[0] >= cds[-1][1]]
else:
utr3 = [exon for exon in utrs
if exon[-1] <= cds[0][0]]
for e in transcript:
if e.feature == "exon":
template_exon = e
break
returned_exons = []
for e in utr3:
gtf = GTF.Entry().fromGTF(template_exon)
gtf.start = e[0]
gtf.end = e[1]
returned_exons.append(gtf)
return returned_exons
def main(argv=None):
if not argv:
argv = sys.argv
parser = E.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
parser.add_option("-m", "--merge-exons",
dest="merge_exons",
action="store_true",
help="merge overlapping exons of all transcripts "
"within a gene. "
"The merged exons will be output. "
"Input needs to sorted by gene [default=%default].")
parser.add_option("-t", "--merge-transcripts",
dest="merge_transcripts",
action="store_true",
help="merge all transcripts within a gene. "
"The entry will span the whole gene "
"(exons and introns). "
"The transcript does not include the UTR unless "
"--with-utr is set. [default=%default].")
parser.add_option("--merge-genes", dest="merge_genes", action="store_true",
help="merge overlapping genes if their exons overlap. "
"A gene with a single transcript containing all exons "
"of the overlapping transcripts will be output. "
"This operation ignores strand information "
"The input needs te sorted by transcript "
"[default=%default].")
parser.add_option("--merge-exons-distance",
dest="merge_exons_distance",
type="int",
help="distance in nucleotides between "
"exons to be merged [default=%default].")
parser.add_option("-j", "--join-exons",
dest="join_exons",
action="store_true",
help="join all exons per transcript. "
"A new transcript will be "
"output that spans a whole transcript. "
"Input needs to be sorted by transcript "
"[default=%default].")
parser.add_option("--unset-genes", dest="unset_genes", type="string",
help="unset gene identifiers, keeping "
"transcripts intact. "
"New gene identifiers are set to the "
"pattern given. For example, "
"'--unset-genes=%06i' [default=%default].")
parser.add_option("--sort",
dest="sort",
type="choice",
choices=("gene",
"gene+transcript",
"transcript",
"position",
"contig+gene",
"position+gene",
"gene+position"),
help="sort input data [default=%default].")
parser.add_option("-u", "--with-utr",
dest="with_utr",
action="store_true",
help="include utr in merged transcripts "
"[default=%default].")
parser.add_option("--intersect-transcripts",
dest="intersect_transcripts",
action="store_true",
help="intersect all transcripts within a gene. "
"The entry will only span those bases "
"that are covered by all transcrips."
"The transcript does not include the UTR unless "
"--with-utr is set. This method "
"will remove all other features (stop_codon, etc.) "
"The input needs to be sorted by gene. "
"[default=%default].")
parser.add_option("-i", "--merge-introns",
dest="merge_introns",
action="store_true",
help="merge and output all introns within a "
"gene. The output will contain "
"all intronic regions within a gene. Single exon genes "
"are skipped. "
"The input needs to be sorted by gene. "
"[default=%default].")
parser.add_option("-g", "--set-transcript-to-gene",
"--set-transcript2gene",
dest="set_transcript2gene",
action="store_true",
help="set the transcript_id to the "
"gene_id [default=%default].")
parser.add_option("--set-protein-to-transcript",
dest="set_protein2transcript",
action="store_true",
help="set the protein_id to the "
"transcript_id [default=%default].")
parser.add_option("--add-protein-id",
dest="add_protein_id",
type="string",
help="add a protein_id for each transcript_id. "
"The argument is a filename containing a mapping "
"between "
"transcript_id to protein_id [default=%default].")
parser.add_option("-G", "--set-gene-to-transcript",
"--set-gene2transcript",
dest="set_gene2transcript",
action="store_true",
help="set the gene_id to the "
"transcript_id [default=%default].")
parser.add_option("-d",
"--set-score2distance",
dest="set_score2distance",
action="store_true",
help="set the score field for each feature to the "
"distance to "
"transcription start site [default=%default].")
parser.add_option("--exons2introns",
dest="exons2introns",
action="store_true",
help="for each gene build an 'intronic' transcript "
"containing the union of all intronic regions "
"of all transcripts in a gene."
"The features are labeled as 'intron'."
"The input needs to be sorted by gene. "
"[default=%default].")
parser.add_option("-f", "--filter", dest="filter",
type="choice",
choices=("gene", "transcript", "longest-gene",
"longest-transcript",
"representative-transcript"),
help="apply a filter to the input file. Available "
"filters are: "
"'gene': filter by gene_id, "
"'transcript': filter by transcript_id, "
"'longest-gene': output the longest gene for "
"overlapping genes ,"
"'longest-transcript': output the longest "
"transcript per gene,"
"'representative-transcript': output the "
"representative transcript per gene. "
"The representative transcript is the transcript "
"that shares most exons with "
"the other transcripts in a gene. "
"The input needs to be sorted by gene. "
"[default=%default].")
parser.add_option("-r", "--rename",
dest="rename",
type="choice",
choices=("gene", "transcript"),
help="rename genes or transcripts with a map "
"given by the option `--apply`. "
"Those that can not be renamed are removed "
"[default=%default].")
parser.add_option("--renumber-genes", dest="renumber_genes", type="string",
help="renumber genes according to the given pattern. "
"[default=%default].")
parser.add_option("--renumber-transcripts",
dest="renumber_transcripts",
type="string",
help="renumber transcripts according to the "
"given pattern. "
"[default=%default].")
parser.add_option("-a", "--apply", dest="filename_filter", type="string",
metavar="tsv",
help="filename of ids to map/filter [default=%default].")
parser.add_option("--invert-filter",
dest="invert_filter",
action="store_true",
help="when using --filter, invert selection "
"(like grep -v). "
"[default=%default].")
parser.add_option("--sample-size", dest="sample_size", type="int",
help="extract a random sample of size # if the option "
"'--filter' is set[default=%default].")
parser.add_option("--intron-min-length",
dest="intron_min_length", type="int",
help="minimum length for introns (for --exons2introns) "
"[default=%default].")
parser.add_option("--min-exons-length",
dest="min_exons_length",
type="int",
help="minimum length for gene (sum of exons) "
"(--sample-size) [default=%default].")
parser.add_option("--intron-border",
dest="intron_border",
type="int",
help="number of residues to exclude at intron at either end "
"(--exons2introns) [default=%default].")
parser.add_option("--transcripts2genes",
dest="transcripts2genes",
action="store_true",
help="cluster overlapping transcripts into genes.")
parser.add_option("--reset-strand",
dest="reset_strand",
action="store_true",
help="remove strandedness of features (set to '.') when "
"using --transcripts2genes"
"[default=%default].")
parser.add_option("--remove-overlapping", dest="remove_overlapping",
type="string",
metavar="gff",
help="remove all transcripts that overlap intervals "
"in a gff-formatted file."
"The comparison ignores strand "
"[default=%default].")
parser.add_option("--permit-duplicates", dest="strict",
action="store_false",
help="permit duplicate genes. "
"[default=%default]")
parser.add_option("--remove-duplicates", dest="remove_duplicates",
type="choice",
choices=("gene", "transcript", "ucsc", "coordinates"),
help="remove duplicates by gene/transcript. "
"If ``ucsc`` is chosen, transcripts ending on _dup# are "
"removed. This is necessary to remove duplicate entries "
"that are next to each other in the sort order "
"[%default]")
parser.add_option("--rename-duplicates", dest="rename_duplicates",
action="store_true",
help="rename duplicate gene_ids and transcript_ids by "
"addition of a numerical suffix")
parser.set_defaults(
sort=None,
merge_exons=False,
join_exons=False,
merge_exons_distance=0,
merge_transcripts=False,
set_score2distance=False,
set_gene2transcript=False,
set_transcript2gene=False,
set_protein2transcript=False,
add_protein_id=None,
filename_filter=None,
filter=None,
exons2introns=None,
merge_genes=False,
intron_border=None,
intron_min_length=None,
sample_size=0,
min_exons_length=0,
transripts2genes=False,
reset_strand=False,
with_utr=False,
invert_filter=False,
remove_duplicates=None,
remove_overlapping=None,
renumber_genes=None,
unset_genes=None,
renumber_transcripts=None,
strict=True,
intersect_transcripts=False,
rename_duplicates=False,
)
(options, args) = E.Start(parser, argv=argv)
ninput, noutput, nfeatures, ndiscarded = 0, 0, 0, 0
if options.set_transcript2gene:
for gff in GTF.iterator(options.stdin):
ninput += 1
gff.setAttribute("transcript_id", gff.gene_id)
options.stdout.write("%s\n" % str(gff))
noutput += 1
nfeatures += 1
elif options.remove_duplicates:
counts = collections.defaultdict(int)
if options.remove_duplicates == "ucsc":
store = []
remove = set()
f = lambda x: x[0].transcript_id
gffs = GTF.transcript_iterator(
GTF.iterator(options.stdin), strict=False)
outf = lambda x: "\n".join([str(y) for y in x])
for entry in gffs:
ninput += 1
store.append(entry)
id = f(entry)
if "_dup" in id:
remove.add(re.sub("_dup\d+", "", id))
remove.add(id)
for entry in store:
id = f(entry)
if id not in remove:
options.stdout.write(outf(entry) + "\n")
noutput += 1
else:
ndiscarded += 1
E.info("discarded duplicates for %s" % (id))
else:
if options.remove_duplicates == "gene":
gffs = GTF.gene_iterator(
GTF.iterator(options.stdin), strict=False)
f = lambda x: x[0][0].gene_id
outf = lambda x: "\n".join(
["\n".join([str(y) for y in xx]) for xx in x])
elif options.remove_duplicates == "transcript":
gffs = GTF.transcript_iterator(
GTF.iterator(options.stdin), strict=False)
f = lambda x: x[0].transcript_id
outf = lambda x: "\n".join([str(y) for y in x])
elif options.remove_duplicates == "coordinates":
gffs = GTF.chunk_iterator(GTF.iterator(options.stdin))
f = lambda x: x[0].contig + "_" + \
str(x[0].start) + "-" + str(x[0].end)
outf = lambda x: "\n".join([str(y) for y in x])
store = []
for entry in gffs:
ninput += 1
store.append(entry)
id = f(entry)
counts[id] += 1
# Assumes GTF file sorted by contig then start
last_id = ""
if options.remove_duplicates == "coordinates":
for entry in store:
id = f(entry)
if id == last_id:
ndiscarded += 1
E.info("discarded duplicates for %s: %i" %
(id, counts[id]))
else:
options.stdout.write(outf(entry) + "\n")
noutput += 1
last_id = id
else:
for entry in store:
id = f(entry)
if counts[id] == 1:
options.stdout.write(outf(entry) + "\n")
noutput += 1
else:
ndiscarded += 1
E.info("discarded duplicates for %s: %i" %
(id, counts[id]))
elif options.sort:
for gff in GTF.iterator_sorted(GTF.iterator(options.stdin),
sort_order=options.sort):
ninput += 1
options.stdout.write("%s\n" % str(gff))
noutput += 1
nfeatures += 1
elif options.set_gene2transcript:
for gff in GTF.iterator(options.stdin):
ninput += 1
gff.setAttribute("gene_id", gff.transcript_id)
options.stdout.write("%s\n" % str(gff))
noutput += 1
nfeatures += 1
elif options.set_protein2transcript:
for gff in GTF.iterator(options.stdin):
ninput += 1
gff.setAttribute("protein_id", gff.transcript_id)
options.stdout.write("%s\n" % str(gff))
noutput += 1
nfeatures += 1
elif options.add_protein_id:
transcript2protein = IOTools.readMap(open(options.add_protein_id, "r"))
missing = set()
for gff in GTF.iterator(options.stdin):
ninput += 1
if gff.transcript_id not in transcript2protein:
if gff.transcript_id not in missing:
E.debug(
("removing transcript '%s' due to "
"missing protein id") % gff.transcript_id)
missing.add(gff.transcript_id)
ndiscarded += 1
continue
gff.setAttribute(
"protein_id", transcript2protein[gff.transcript_id])
options.stdout.write("%s\n" % str(gff))
noutput += 1
nfeatures += 1
E.info("transcripts removed due to missing protein ids: %i" %
len(missing))
elif options.join_exons:
for exons in GTF.transcript_iterator(GTF.iterator(options.stdin)):
ninput += 1
strand = Genomics.convertStrand(exons[0].strand)
contig = exons[0].contig
transid = exons[0].transcript_id
geneid = exons[0].gene_id
biotype = exons[0].source
all_start, all_end = min([x.start for x in exons]), max(
[x.end for x in exons])
y = GTF.Entry()
y.contig = contig
y.source = biotype
y.feature = "transcript"
y.start = all_start
y.end = all_end
y.strand = strand
y.transcript_id = transid
y.gene_id = geneid
options.stdout.write("%s\n" % str(y))
elif options.merge_genes:
# merges overlapping genes
#
gffs = GTF.iterator_sorted_chunks(
GTF.flat_gene_iterator(GTF.iterator(options.stdin)),
sort_by="contig-strand-start")
def iterate_chunks(gff_chunks):
last = gff_chunks.next()
to_join = [last]
for gffs in gff_chunks:
d = gffs[0].start - last[-1].end
if gffs[0].contig == last[0].contig and \
gffs[0].strand == last[0].strand:
assert gffs[0].start >= last[0].start, \
("input file should be sorted by contig, strand "
"and position: d=%i:\nlast=\n%s\nthis=\n%s\n") % \
(d,
"\n".join([str(x) for x in last]),
"\n".join([str(x) for x in gffs]))
if gffs[0].contig != last[0].contig or \
gffs[0].strand != last[0].strand or \
d > 0:
yield to_join
to_join = []
last = gffs
to_join.append(gffs)
yield to_join
raise StopIteration
for chunks in iterate_chunks(gffs):
ninput += 1
if len(chunks) > 1:
gene_id = "merged_%s" % chunks[0][0].gene_id
transcript_id = "merged_%s" % chunks[0][0].transcript_id
info = ",".join([x[0].gene_id for x in chunks])
else:
gene_id = chunks[0][0].gene_id
transcript_id = chunks[0][0].transcript_id
info = None
intervals = []
for c in chunks:
intervals += [(x.start, x.end) for x in c]
intervals = Intervals.combine(intervals)
# take single strand
strand = chunks[0][0].strand
for start, end in intervals:
y = GTF.Entry()
y.fromGTF(chunks[0][0], gene_id, transcript_id)
y.start = start
y.end = end
y.strand = strand
if info:
y.addAttribute("merged", info)
options.stdout.write("%s\n" % str(y))
nfeatures += 1
noutput += 1
elif options.renumber_genes:
map_old2new = {}
for gtf in GTF.iterator(options.stdin):
ninput += 1
if gtf.gene_id not in map_old2new:
map_old2new[gtf.gene_id] = options.renumber_genes % (
len(map_old2new) + 1)
gtf.setAttribute("gene_id", map_old2new[gtf.gene_id])
options.stdout.write("%s\n" % str(gtf))
noutput += 1
elif options.unset_genes:
map_old2new = {}
for gtf in GTF.iterator(options.stdin):
ninput += 1
key = gtf.transcript_id
if key not in map_old2new:
map_old2new[key] = options.unset_genes % (len(map_old2new) + 1)
gtf.setAttribute("gene_id", map_old2new[key])
options.stdout.write("%s\n" % str(gtf))
noutput += 1
elif options.renumber_transcripts:
map_old2new = {}
for gtf in GTF.iterator(options.stdin):
ninput += 1
key = (gtf.gene_id, gtf.transcript_id)
if key not in map_old2new:
map_old2new[key] = options.renumber_transcripts % (
len(map_old2new) + 1)
gtf.setAttribute("transcript_id", map_old2new[key])
options.stdout.write("%s\n" % str(gtf))
noutput += 1
elif options.transcripts2genes:
transcripts = set()
genes = set()
reset_strand = options.reset_strand
for gtfs in GTF.iterator_transcripts2genes(
GTF.iterator(options.stdin)):
ninput += 1
for gtf in gtfs:
if reset_strand:
gtf.strand = "."
options.stdout.write("%s\n" % str(gtf))
transcripts.add(gtf.transcript_id)
genes.add(gtf.gene_id)
nfeatures += 1
noutput += 1
E.info("transcripts2genes: transcripts=%i, genes=%i" %
(len(transcripts), len(genes)))
elif options.rename:
map_old2new = IOTools.readMap(open(options.filename_filter, "r"))
if options.rename == "transcript":
is_gene_id = False
elif options.rename == "gene":
is_gene_id = True
for gff in GTF.iterator(options.stdin):
ninput += 1
if is_gene_id:
if gff.gene_id in map_old2new:
gff.setAttribute("gene_id", map_old2new[gff.gene_id])
else:
E.debug("removing missing gene_id %s" % gff.gene_id)
ndiscarded += 1
continue
else:
if gff.transcript_id in map_old2new:
gff.setAttribute(
"transcript_id", map_old2new[gff.transcript_id])
else:
E.debug("removing missing transcript_id %s" %
gff.transcript_id)
ndiscarded += 1
continue
noutput += 1
options.stdout.write("%s\n" % str(gff))
elif options.filter:
keep_genes = set()
if options.filter == "longest-gene":
iterator = GTF.flat_gene_iterator(GTF.iterator(options.stdin))
coords = []
gffs = []
for gff in iterator:
gff.sort(key=lambda x: x.start)
coords.append((gff[0].contig,
min([x.start for x in gff]),
max([x.end for x in gff]),
gff[0].gene_id))
gffs.append(gff)
coords.sort()
last_contig = None
max_end = 0
longest_gene_id = None
longest_length = None
for contig, start, end, gene_id in coords:
ninput += 1
if contig != last_contig or start >= max_end:
if longest_gene_id:
keep_genes.add(longest_gene_id)
longest_gene_id = gene_id
longest_length = end - start
max_end = end
else:
if end - start > longest_length:
longest_length, longest_gene_id = end - start, gene_id
last_contig = contig
max_end = max(max_end, end)
keep_genes.add(longest_gene_id)
invert = options.invert_filter
for gff in gffs:
keep = gff[0].gene_id in keep_genes
if (keep and not invert) or (not keep and invert):
noutput += 1
for g in gff:
nfeatures += 1
options.stdout.write("%s\n" % g)
else:
ndiscarded += 1
elif options.filter in ("longest-transcript", "representative-transcript"):
iterator = GTF.gene_iterator(GTF.iterator(options.stdin))
def selectLongestTranscript(gene):
r = []
for transcript in gene:
transcript.sort(key=lambda x: x.start)
length = transcript[-1].end - transcript[0].start
r.append((length, transcript))
r.sort()
return r[-1][1]
def selectRepresentativeTranscript(gene):
'''select a representative transcript.
The representative transcript represent the largest number
of exons over all transcripts.
'''
all_exons = []
for transcript in gene:
all_exons.extend([(x.start, x.end)
for x in transcript if x.feature == "exon"])
exon_counts = {}
for key, exons in itertools.groupby(all_exons):
exon_counts[key] = len(list(exons))
transcript_counts = []
for transcript in gene:
count = sum([exon_counts[(x.start, x.end)]
for x in transcript if x.feature == "exon"])
transcript_counts.append((count, transcript))
transcript_counts.sort()
return transcript_counts[-1][1]
if options.filter == "longest-transcript":
_select = selectLongestTranscript
elif options.filter == "representative-transcript":
_select = selectRepresentativeTranscript
for gene in iterator:
ninput += 1
transcript = _select(gene)
noutput += 1
for g in transcript:
nfeatures += 1
options.stdout.write("%s\n" % g)
elif options.filter in ("gene", "transcript"):
if options.filename_filter:
ids, nerrors = IOTools.ReadList(
open(options.filename_filter, "r"))
E.info("read %i ids" % len(ids))
ids = set(ids)
by_gene = options.filter == "gene"
by_transcript = options.filter == "transcript"
invert = options.invert_filter
reset_strand = options.reset_strand
for gff in GTF.iterator(options.stdin):
ninput += 1
keep = False
if by_gene:
keep = gff.gene_id in ids
if by_transcript:
keep = gff.transcript_id in ids
if (invert and keep) or (not invert and not keep):
continue
if reset_strand:
gff.strand = "."
options.stdout.write("%s\n" % str(gff))
nfeatures += 1
noutput += 1
elif options.sample_size:
if options.filter == "gene":
iterator = GTF.flat_gene_iterator(
GTF.iterator(options.stdin))
elif options.filter == "transcript":
iterator = GTF.transcript_iterator(
GTF.iterator(options.stdin))
if options.min_exons_length:
iterator = GTF.iterator_min_feature_length(
iterator,
min_length=options.min_exons_length,
feature="exon")
data = [x for x in iterator]
ninput = len(data)
if len(data) > options.sample_size:
data = random.sample(data, options.sample_size)
for d in data:
noutput += 1
for dd in d:
nfeatures += 1
options.stdout.write(str(dd) + "\n")
else:
assert False, "please supply either a filename "
"with ids to filter with (--apply) or a sample-size."
elif options.exons2introns:
for gffs in GTF.flat_gene_iterator(GTF.iterator(options.stdin)):
ninput += 1
cds_ranges = GTF.asRanges(gffs, "CDS")
exon_ranges = GTF.asRanges(gffs, "exon")
input_ranges = Intervals.combine(cds_ranges + exon_ranges)
if len(input_ranges) > 1:
last = input_ranges[0][1]
output_ranges = []
for start, end in input_ranges[1:]:
output_ranges.append((last, start))
last = end
if options.intron_border:
b = options.intron_border
output_ranges = [(x[0] + b, x[1] - b)
for x in output_ranges]
if options.intron_min_length:
l = options.intron_min_length
output_ranges = [
x for x in output_ranges if x[1] - x[0] > l]
for start, end in output_ranges:
entry = GTF.Entry()
entry.copy(gffs[0])
entry.clearAttributes()
entry.transcript_id = "merged"
entry.feature = "intron"
entry.start = start
entry.end = end
options.stdout.write("%s\n" % str(entry))
nfeatures += 1
noutput += 1
else:
ndiscarded += 1
elif options.set_score2distance:
for gffs in GTF.transcript_iterator(GTF.iterator(options.stdin)):
ninput += 1
strand = Genomics.convertStrand(gffs[0].strand)
all_start, all_end = min([x.start for x in gffs]), max(
[x.end for x in gffs])
if strand != ".":
t = 0
if strand == "-":
gffs.reverse()
for gff in gffs:
gff.score = t
t += gff.end - gff.start
if strand == "-":
gffs.reverse()
for gff in gffs:
options.stdout.write("%s\n" % str(gff))
nfeatures += 1
noutput += 1
elif options.remove_overlapping:
index = GTF.readAndIndex(
GTF.iterator(IOTools.openFile(options.remove_overlapping, "r")))
for gffs in GTF.transcript_iterator(GTF.iterator(options.stdin)):
ninput += 1
found = False
for e in gffs:
if index.contains(e.contig, e.start, e.end):
found = True
break
if found:
ndiscarded += 1
else:
noutput += 1
for e in gffs:
nfeatures += 1
options.stdout.write("%s\n" % str(e))
elif options.intersect_transcripts:
for gffs in GTF.gene_iterator(GTF.iterator(options.stdin),
strict=options.strict):
ninput += 1
r = []
for g in gffs:
if options.with_utr:
ranges = GTF.asRanges(g, "exon")
else:
ranges = GTF.asRanges(g, "CDS")
r.append(ranges)
result = r[0]
for x in r[1:]:
result = Intervals.intersect(result, x)
entry = GTF.Entry()
entry.copy(gffs[0][0])
entry.clearAttributes()
entry.transcript_id = "merged"
entry.feature = "exon"
for start, end in result:
entry.start = start
entry.end = end
options.stdout.write("%s\n" % str(entry))
nfeatures += 1
noutput += 1
elif options.rename_duplicates:
gene_ids = list()
transcript_ids = list()
gtfs = list()
for gtf in GTF.iterator(options.stdin):
gtfs.append(gtf)
if gtf.feature == "CDS":
gene_ids.append(gtf.gene_id)
transcript_ids.append(gtf.transcript_id)
dup_gene = [item for item in set(gene_ids) if gene_ids.count(item) > 1]
dup_transcript = [item for item in set(transcript_ids)
if transcript_ids.count(item) > 1]
E.info("Number of duplicated gene_ids: %i" % len(dup_gene))
E.info("Number of duplicated transcript_ids: %i" % len(dup_transcript))
gene_dict = dict(zip(dup_gene, ([0] * len(dup_gene))))
transcript_dict = dict(zip(dup_transcript,
([0] * len(dup_transcript))))
for gtf in gtfs:
if gtf.feature == "CDS":
if gtf.gene_id in dup_gene:
gene_dict[gtf.gene_id] = gene_dict[gtf.gene_id] + 1
gtf.setAttribute('gene_id',
gtf.gene_id + "." +
str(gene_dict[gtf.gene_id]))
if gtf.transcript_id in dup_transcript:
transcript_dict[gtf.transcript_id] = \
transcript_dict[gtf.transcript_id] + 1
gtf.setAttribute('transcript_id',
gtf.transcript_id + "." +
str(transcript_dict[gtf.transcript_id]))
options.stdout.write("%s\n" % gtf)
else:
for gffs in GTF.flat_gene_iterator(
GTF.iterator(options.stdin),
strict=options.strict):
ninput += 1
cds_ranges = GTF.asRanges(gffs, "CDS")
exon_ranges = GTF.asRanges(gffs, "exon")
# sanity checks
strands = set([x.strand for x in gffs])
contigs = set([x.contig for x in gffs])
if len(strands) > 1:
raise ValueError("can not merge gene '%s' on multiple strands: %s" % (
gffs[0].gene_id, str(strands)))
if len(contigs) > 1:
raise ValueError("can not merge gene '%s' on multiple contigs: %s" % (
gffs[0].gene_id, str(contigs)))
strand = Genomics.convertStrand(gffs[0].strand)
if cds_ranges and options.with_utr:
cds_start, cds_end = cds_ranges[0][0], cds_ranges[-1][1]
midpoint = (cds_end - cds_start) / 2 + cds_start
utr_ranges = []
for start, end in Intervals.truncate(exon_ranges, cds_ranges):
if end - start > 3:
if strand == ".":
feature = "UTR"
elif strand == "+":
if start < midpoint:
feature = "UTR5"
else:
feature = "UTR3"
elif strand == "-":
if start < midpoint:
feature = "UTR3"
else:
feature = "UTR5"
utr_ranges.append((feature, start, end))
output_feature = "CDS"
output_ranges = cds_ranges
else:
output_feature = "exon"
output_ranges = exon_ranges
utr_ranges = []
result = []
if options.merge_exons:
# need to combine per feature - skip
# utr_ranges = Intervals.combineAtDistance(
# utr_ranges,
# options.merge_exons_distance)
output_ranges = Intervals.combineAtDistance(
output_ranges, options.merge_exons_distance)
for feature, start, end in utr_ranges:
entry = GTF.Entry()
entry.copy(gffs[0])
entry.clearAttributes()
entry.feature = feature
entry.transcript_id = "merged"
entry.start = start
entry.end = end
result.append(entry)
for start, end in output_ranges:
entry = GTF.Entry()
entry.copy(gffs[0])
entry.clearAttributes()
entry.transcript_id = "merged"
entry.feature = output_feature
entry.start = start
entry.end = end
result.append(entry)
elif options.merge_transcripts:
entry = GTF.Entry()
entry.copy(gffs[0])
entry.clearAttributes()
entry.transcript_id = entry.gene_id
entry.start = output_ranges[0][0]
entry.end = output_ranges[-1][1]
result.append(entry)
elif options.merge_introns:
if len(output_ranges) >= 2:
entry = GTF.Entry()
entry.copy(gffs[0])
entry.clearAttributes()
entry.transcript_id = entry.gene_id
entry.start = output_ranges[0][1]
entry.end = output_ranges[-1][0]
result.append(entry)
else:
ndiscarded += 1
continue
result.sort(key=lambda x: x.start)
for x in result:
options.stdout.write("%s\n" % str(x))
nfeatures += 1
noutput += 1
E.info("ninput=%i, noutput=%i, nfeatures=%i, ndiscarded=%i" %
(ninput, noutput, nfeatures, ndiscarded))
E.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = E.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
parser.add_option("-g", "--gtf-file", dest="gtf", type="string",
help="GTF containing gene annotations")
parser.add_option("-s", "--sort", dest="sort", type="choice",
default="length",
choices=sort_choices,
help="Property to sort rows by. Choices are %s"
% ", ".join(sort_choices))
parser.add_option("-b", "--bin-size", dest="bin_size", type="int",
default=25,
help="Size of window over which to sum reads")
parser.add_option("-u", "--upstream-window", dest="us_win", type="int",
default=500,
help="Amount of sequence upstream of alignment point (less introns)")
parser.add_option("-d", "--downstream-window", dest="ds_win", type="int",
default=None,
help="Amount of sequence downstream of alignment point (default longest segment)")
parser.add_option("-a", "--align-at", dest="align_at", type="choice",
default="start",
choices=align_choices,
help="Where to align genes/transcripts at. Choices are %s"
% ", ".join(align_choices))
parser.add_option("-H", "--height", dest="height", type="int",
default=None,
help="Number of rows in output matrix/heigh of plot in px")
parser.add_option("-w", "--width", dest="width", type="int",
default=None,
help="Number of columns in output/width of plot in px"
"default based on bin size")
parser.add_option("-n", "--normalize", dest="normalize", type="choice",
default="none",
choices=norm_choices,
help="Row normalization to apply. Choices are: %s"
% ", ".join(norm_choices))
parser.add_option("-r", "--renormalize", dest="renormalize", type="choice",
default="none",
choices=norm_choices,
help="Row normalization to apply after row/column compression")
parser.add_option("--no-plot", dest="plot", action="store_false",
default=True,
help="Do not output plot - compute matrix only")
parser.add_option("--use-matrix", dest="use_matrix", type="string",
default=None,
help="Use existing matrix")
parser.add_option("--annotations", dest="annotations", type="choice",
action="append",
choices=annotation_choices,
help="Add annotations to the output plot")
parser.add_option("--reverse-strand", dest="rstrand", action="store_true",
default=False,
help="Find reads on reverse strand")
parser.add_option("-f", "--feature", dest="feature", type="choice",
choices=["gene", "transcript"],
default="gene",
help="use genes or transcripts")
parser.add_option("--quantile", dest="quantile", type="float",
default=0.99,
help="Quantile to use in quantile normalization")
parser.add_option("-o", "--outfile-prefix", dest="outfile_pattern", type="string",
default=None,
help="base of names for output files")
parser.add_option("-c", "--crop", dest="crop", type="string",
default=None,
help="crop view to a certain range on the xaxis. Specify like"
"-500:1000")
parser.add_option("--format", dest="format", type="string",
default="png",
help="Output format, use valid R graphics device")
parser.add_option("--plus-wig", dest="plus_wig", type="string",
help="Use this wig for plus strand info rather than bam file")
parser.add_option("--minus-wig", dest="minus_wig", type="string",
help="Use this wig for minus strand info rather than bam file")
parser.add_option("--bed", dest="bed", type="string",
help="Use this bed for signal(must be indexed)")
parser.add_option("--norm-mat", dest="norm_mat", type="string",
help="Use this matrix for normalizing (e.g. RNA data")
parser.add_option("--sort-order-file", dest="sort_file", type="string",
default=None,
help="Two column file containing gene names in the first
column and a numeric value to sort on in the second")
# add common options (-h/--help, ...) and parse command line
(options, args) = E.Start(parser, argv=argv)
if options.plot and (options.height is None):
options.height = 100
if options.gtf:
f = IOTools.openFile(options.gtf)
if options.feature == "gene":
gtf_iterator = GTF.flat_gene_iterator(GTF.iterator(f))
else:
gtf_iterator = GTF.transcript_iterator(GTF.iterator(f))
lengths = dict()
utr3_lengths = dict()
utr5_lengths = dict()
first_exon_lengths = dict()
for transcript in gtf_iterator:
lengths[transcript[0].transcript_id] = sum(
[e[1] - e[0] for e in GTF.asRanges(transcript, "exon")])
exons = GTF.asRanges(transcript, "exon")
utrs = GTF.asRanges(transcript, "UTR")
coding = Intervals.truncate(exons, utrs)
coding.sort()
utr5 = [utr for utr in utrs if utr[1] <= coding[0][0]]
utr3 = [utr for utr in utrs if utr[0] >= coding[-1][-1]]
if transcript[0].strand == "-":
utr3, utr5 = utr5, utr3
if transcript[0].strand == "+" or len(exons) == 1:
first_exon_lengths[transcript[0].transcript_id] = \
exons[0][1] - exons[0][0]
else:
first_exon_lengths[transcript[0].transcript_id] = \
exons[-1][1] - exons[-1][0]
utr3_lengths[transcript[0].transcript_id] = sum(
[e[1] - e[0] for e in utr3])
utr5_lengths[transcript[0].transcript_id] = sum(
[e[1] - e[0] for e in utr5])
lengths = pandas.Series(lengths)
utr3_lengths = pandas.Series(utr3_lengths)
utr5_lengths = pandas.Series(utr5_lengths)
first_exon_lengths = pandas.Series(first_exon_lengths)
else:
options.sort = "none"
options.annotations = None
if options.plus_wig:
getter = iCLIP.make_getter(plus_wig=options.plus_wig,
minus_wig=options.minus_wig)
elif options.bed:
getter = iCLIP.make_getter(bedfile=options.bed)
else:
try:
getter = iCLIP.make_getter(bamfile=args[0])
except IOError:
E.error("Cannot open bamfile %s" % args[0])
return(1)
except IndexError:
getter = None
if options.use_matrix:
raw_matrix = pandas.read_csv(options.use_matrix,
sep="\t",
index_col=0)
raw_matrix.columns = raw_matrix.columns.astype("int")
else:
raw_matrix = get_matrix(getter, lengths, options)
if options.crop:
crop_from, crop_to = map(int, options.crop.split(":"))
raw_matrix = raw_matrix.loc[:, crop_from:crop_to]
if options.norm_mat:
norm_matrix = pandas.read_csv(options.norm_mat,
sep="\t",
index_col=0)
norm_matrix.columns = norm_matrix.columns.astype("int")
if options.crop:
norm_matrix = norm_matrix.loc[:, crop_from:crop_to]
if all(norm_matrix.columns == raw_matrix.columns) and \
all(raw_matrix.index.isin(norm_matrix.index.values)):
norm_matrix = norm_matrix.loc[raw_matrix.index]
norm_matrix = norm_matrix.replace(
0, norm_matrix[norm_matrix > 0].min().min())
raw_matrix = raw_matrix/norm_matrix
norm_matrix = None
else:
raise ValueError("Incompatible normalisation matrix")
normalized_matrix = normalize(raw_matrix, options.normalize,
quantile=options.quantile)
if options.sort == "length":
sorter = lengths
elif options.sort == "3utr":
sorter = utr3_lengths
elif options.sort == "5utr":
sorter = utr5_lengths
elif options.sort == "first-exon":
sorter = first_exon_lengths
elif options.sort == "manual":
sorter = pandas.read_csv(options.sort_file, sep="\t",
index_col=0, usecols=[0, 1])
sorter = sorter[sorter.columns[0]]
elif options.sort == "none":
sorter = pandas.Series(range(raw_matrix.shape[0]),
index=raw_matrix.index[::-1])
sorter = sorter[sorter.index.isin(normalized_matrix.index)]
sorter = sorter.sort_values(ascending=False)
sorted_matrix = normalized_matrix.loc[sorter.index.values]
compress_matrix = iCLIP.compress_matrix(sorted_matrix,
ncols=options.width,
nrows=options.height)
renormalized_matrix = normalize(compress_matrix, options.renormalize,
quantile=options.quantile)
if renormalized_matrix is raw_matrix and options.use_matrix is not None:
E.info("Input and output matrices are identical, no matrix output")
else:
if options.outfile_pattern:
mat_outfile = IOTools.openFile(
options.outfile_pattern + ".matrix.tsv.gz", "w")
else:
mat_outfile = options.stdout
renormalized_matrix.to_csv(mat_outfile, sep="\t")
if options.plot:
try:
from rpy2.robjects import r as R
from rpy2 import robjects as ro
except:
E.info("No rpy2. Not plotting image")
return(0)
from rpy2.robjects.numpy2ri import numpy2ri
ro.conversion.py2ri = numpy2ri
ro.numpy2ri.activate()
if options.outfile_pattern:
plot_outfile = options.outfile_pattern + ".png"
else:
plot_outfile = "bam2heatmap_out.png"
c = R["c"]
R[options.format](plot_outfile,
width=renormalized_matrix.shape[1] + 72,
height=renormalized_matrix.shape[0] + 72,
unit="px",
res=72)
R.par(mai=c(1, 0.5, 0, 0.5))
cols = R["colorRampPalette"](c("white", "blue"))(50)
bases = renormalized_matrix.columns.values.astype("int")
groups = renormalized_matrix.index.values.astype("int")
mat = renormalized_matrix.as_matrix()
mat[mat >= 1] = 1
R.image(bases, groups, R.t(mat),
zlim=c(0, 1),
raster=True,
col=cols,
xlab="Base",
yaxt="n")
def _sort_and_compress_annotation(anno):
sorted_anno = anno.loc[sorter.index]
comp_anno = iCLIP.compress_matrix(
sorted_anno, renormalized_matrix.shape[0])
return comp_anno
if options.annotations:
ends = _sort_and_compress_annotation(lengths)
starts = pandas.Series(0, index=renormalized_matrix.index)
if options.align_at == "end":
starts, ends = -1 * ends, starts
if "start" in options.annotations:
R.lines(starts.values, starts.index.values, col="black", pch=".")
if "end" in options.annotations:
R.lines(ends.values, ends.index.values,
pch=".", col="black")
if "5utr" in options.annotations:
utr5s = _sort_and_compress_annotation(utr5_lengths)
utr5s = starts + utr5s
R.lines(utr5s.values, utr5s.index.values, col="orange", pch=".")
if "3utr" in options.annotations:
utr3s = _sort_and_compress_annotation(utr3_lengths)
utr3s = ends - utr3s
R.lines(utr3s.values, utr3s.index.values, col="orange", pch=".")
R["dev.off"]()
# write footer and output benchmark information.
E.Stop()
def testHalfEmpty(self):
"""test empty input."""
self.assertEqual(Intervals.truncate([], [(0, 5)]), [])
self.assertEqual(Intervals.truncate([(0, 5)], []), [(0, 5)])
def testEmpty(self):
"""test empty input."""
self.assertEqual(Intervals.truncate([], []), [])
def testHalfEmpty(self):
"""test empty input."""
self.assertEqual(Intervals.truncate([], [(0, 5)]), [])
self.assertEqual(Intervals.truncate([(0, 5)], []), [(0, 5)])
def annotateGenome(iterator, fasta, options, default_code=DEFAULT_CODE):
"""annotate a genome given by the indexed *fasta* file and
an iterator over gtf annotations.
"""
annotations = {}
contig_sizes = fasta.getContigSizes(with_synonyms=False)
E.info("allocating memory for %i contigs and %i bytes" %
(len(contig_sizes), sum(contig_sizes.values()) * array.array("c").itemsize))
# AString.AString( "a").itemsize ))
for contig, size in contig_sizes.items():
E.debug("allocating %s: %i bases" % (contig, size))
# annotations[contig] = AString.AString( default_code * size )
annotations[contig] = array.array("c", default_code * size)
E.info("allocated memory for %i contigs" % len(fasta))
counter = E.Counter()
# output splice junctions
outfile_junctions = E.openOutputFile("junctions")
outfile_junctions.write(
"contig\tstrand\tpos1\tpos2\tframe\tgene_id\ttranscript_id\n")
for gtfs in iterator:
counter.input += 1
if counter.input % options.report_step == 0:
E.info("iteration %i" % counter.input)
try:
contig = fasta.getToken(gtfs[0].contig)
except KeyError, msg:
E.warn("contig %s not found - annotation ignored" % gtfs[0].contig)
counter.skipped_contig += 1
continue
lcontig = fasta.getLength(contig)
# make sure that exons are sorted by coordinate
gtfs.sort(key=lambda x: x.start)
is_positive = Genomics.IsPositiveStrand(gtfs[0].strand)
source = gtfs[0].source
# process non-coding data
if source in MAP_ENSEMBL:
code = MAP_ENSEMBL[source]
intervals = [(x.start, x.end) for x in gtfs]
addSegments(annotations[contig],
intervals,
is_positive,
code)
elif source == "protein_coding":
# collect exons for utr
exons = [(x.start, x.end) for x in gtfs if x.feature == "exon"]
cds = [(x.start, x.end) for x in gtfs if x.feature == "CDS"]
if len(cds) == 0:
counter.skipped_transcripts += 1
E.warn("protein-coding transcript %s without CDS - skipped" %
gtfs[0].transcript_id)
continue
exons = Intervals.truncate(exons, cds)
start, end = cds[0][0], cds[-1][1]
UTR5 = [x for x in exons if x[1] < start]
UTR3 = [x for x in exons if x[0] >= end]
if not is_positive:
UTR5, UTR3 = UTR3, UTR5
splice_code = "S"
else:
splice_code = "s"
addSegments(annotations[contig],
UTR5,
is_positive,
"u")
addIntrons(annotations[contig],
UTR5,
is_positive,
options.max_frameshift_length)
addSegments(annotations[contig],
UTR3,
is_positive,
"v")
addIntrons(annotations[contig],
UTR3,
is_positive,
options.max_frameshift_length)
# output CDS according to frame
addCDS(annotations[contig],
[x for x in gtfs if x.feature == "CDS"],
is_positive)
# add introns between CDS
addIntrons(annotations[contig],
cds,
is_positive,
options.max_frameshift_length)
# output splice junctions
cds = [x for x in gtfs if x.feature == "CDS"]
# apply corrections for 1-past end coordinates
# to point between residues within CDS
if is_positive:
ender = lambda x: x.end - 1
starter = lambda x: x.start
out_positive = "+"
else:
ender = lambda x: lcontig - x.start - 1
starter = lambda x: lcontig - x.end
out_positive = "-"
cds.reverse()
end = ender(cds[0])
for c in cds[1:]:
start = starter(c)
outfile_junctions.write("%s\t%s\t%i\t%i\t%s\t%s\t%s\n" %
(contig,
out_positive,
end,
start,
c.frame,
c.gene_id,
c.transcript_id,
))
end = ender(c)
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
parser = E.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
parser.add_option("--is-gtf",
dest="is_gtf",
action="store_true",
help="input is gtf instead of gff.")
parser.add_option("-g",
"--genome-file",
dest="genome_file",
type="string",
help="filename with genome [default=%default].")
parser.add_option("-m",
"--merge-adjacent",
dest="merge",
action="store_true",
help="merge adjacent intervals with the same attributes."
" [default=%default]")
parser.add_option("-e",
"--feature",
dest="feature",
type="string",
help="filter by a feature, for example 'exon', 'CDS'."
" If set to the empty string, all entries are output "
"[%default].")
parser.add_option("-f",
"--maskregions-bed-file",
dest="filename_masks",
type="string",
metavar="gff",
help="mask sequences with regions given in gff file "
"[%default].")
parser.add_option("--remove-masked-regions",
dest="remove_masked_regions",
action="store_true",
help="remove regions instead of masking [%default].")
parser.add_option("--min-interval-length",
dest="min_length",
type="int",
help="set minimum length for sequences output "
"[%default]")
parser.add_option("--max-length",
dest="max_length",
type="int",
help="set maximum length for sequences output "
"[%default]")
parser.add_option("--extend-at",
dest="extend_at",
type="choice",
choices=("none", "3", "5", "both", "3only", "5only"),
help="extend at no end, 3', 5' or both ends. If "
"3only or 5only are set, only the added sequence "
"is returned [default=%default]")
parser.add_option("--extend-by",
dest="extend_by",
type="int",
help="extend by # bases [default=%default]")
parser.add_option("--extend-with",
dest="extend_with",
type="string",
help="extend using base [default=%default]")
parser.add_option("--masker",
dest="masker",
type="choice",
choices=("dust", "dustmasker", "softmask", "none"),
help="apply masker [%default].")
parser.add_option("--fold-at",
dest="fold_at",
type="int",
help="fold sequence every n bases[%default].")
parser.add_option(
"--fasta-name-attribute",
dest="naming_attribute",
type="string",
help="use attribute to name fasta entry. Currently only compatable"
" with gff format [%default].")
parser.set_defaults(is_gtf=False,
genome_file=None,
merge=False,
feature=None,
filename_masks=None,
remove_masked_regions=False,
min_length=0,
max_length=0,
extend_at=None,
extend_by=100,
extend_with=None,
masker=None,
fold_at=None,
naming_attribute=False)
(options, args) = E.Start(parser)
if options.genome_file:
fasta = IndexedFasta.IndexedFasta(options.genome_file)
contigs = fasta.getContigSizes()
if options.is_gtf:
iterator = GTF.transcript_iterator(GTF.iterator(options.stdin))
else:
gffs = GTF.iterator(options.stdin)
if options.merge:
iterator = GTF.joined_iterator(gffs)
else:
iterator = GTF.chunk_iterator(gffs)
masks = None
if options.filename_masks:
masks = {}
with IOTools.openFile(options.filename_masks, "r") as infile:
e = GTF.readAsIntervals(GTF.iterator(infile))
# convert intervals to intersectors
for contig in list(e.keys()):
intersector = bx.intervals.intersection.Intersecter()
for start, end in e[contig]:
intersector.add_interval(bx.intervals.Interval(start, end))
masks[contig] = intersector
ninput, noutput, nmasked, nskipped_masked = 0, 0, 0, 0
nskipped_length = 0
nskipped_noexons = 0
feature = options.feature
# iterator is a list containing groups (lists) of features.
# Each group of features have in common the same transcript ID, in case of
# GTF files.
for ichunk in iterator:
ninput += 1
if feature:
chunk = [x for x in ichunk if x.feature == feature]
else:
chunk = ichunk
if len(chunk) == 0:
nskipped_noexons += 1
E.info("no features in entry from "
"%s:%i..%i - %s" % (ichunk[0].contig, ichunk[0].start,
ichunk[0].end, str(ichunk[0])))
continue
contig, strand = chunk[0].contig, chunk[0].strand
if options.is_gtf:
name = chunk[0].transcript_id
else:
if options.naming_attribute:
attr_dict = {
x.split("=")[0]: x.split("=")[1]
for x in chunk[0].attributes.split(";")
}
name = attr_dict[options.naming_attribute]
else:
name = str(chunk[0].attributes)
lcontig = contigs[contig]
positive = Genomics.IsPositiveStrand(strand)
intervals = [(x.start, x.end) for x in chunk]
intervals.sort()
if masks:
if contig in masks:
masked_regions = []
for start, end in intervals:
masked_regions += [(x.start, x.end)
for x in masks[contig].find(start, end)]
masked_regions = Intervals.combine(masked_regions)
if len(masked_regions):
nmasked += 1
if options.remove_masked_regions:
intervals = Intervals.truncate(intervals, masked_regions)
else:
raise NotImplementedError("unimplemented")
if len(intervals) == 0:
nskipped_masked += 1
if options.loglevel >= 1:
options.stdlog.write(
"# skipped because fully masked: "
"%s: regions=%s masks=%s\n" %
(name, str([(x.start, x.end)
for x in chunk]), masked_regions))
continue
out = intervals
if options.extend_at and not options.extend_with:
if options.extend_at == "5only":
intervals = [(max(0, intervals[0][0] - options.extend_by),
intervals[0][0])]
elif options.extend_at == "3only":
intervals = [(intervals[-1][1],
min(lcontig,
intervals[-1][1] + options.extend_by))]
else:
if options.extend_at in ("5", "both"):
intervals[0] = (max(0,
intervals[0][0] - options.extend_by),
intervals[0][1])
if options.extend_at in ("3", "both"):
intervals[-1] = (intervals[-1][0],
min(lcontig,
intervals[-1][1] + options.extend_by))
if not positive:
intervals = [(lcontig - x[1], lcontig - x[0])
for x in intervals[::-1]]
out.reverse()
s = [
fasta.getSequence(contig, strand, start, end)
for start, end in intervals
]
# IMS: allow for masking of sequences
s = Masker.maskSequences(s, options.masker)
l = sum([len(x) for x in s])
if (l < options.min_length
or (options.max_length and l > options.max_length)):
nskipped_length += 1
if options.loglevel >= 1:
options.stdlog.write("# skipped because length out of bounds "
"%s: regions=%s len=%i\n" %
(name, str(intervals), l))
continue
if options.extend_at and options.extend_with:
extension = "".join((options.extend_with, ) * options.extend_by)
if options.extend_at in ("5", "both"):
s[1] = extension + s[1]
if options.extend_at in ("3", "both"):
s[-1] = s[-1] + extension
if options.fold_at:
n = options.fold_at
s = "".join(s)
seq = "\n".join([s[i:i + n] for i in range(0, len(s), n)])
else:
seq = "\n".join(s)
options.stdout.write(
">%s %s:%s:%s\n%s\n" %
(name, contig, strand, ";".join(["%i-%i" % x for x in out]), seq))
noutput += 1
E.info("ninput=%i, noutput=%i, nmasked=%i, nskipped_noexons=%i, "
"nskipped_masked=%i, nskipped_length=%i" %
(ninput, noutput, nmasked, nskipped_noexons, nskipped_masked,
nskipped_length))
E.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
parser = E.OptionParser(
version="%prog version: $Id$",
usage=globals()["__doc__"])
parser.add_option("--is-gtf", dest="is_gtf", action="store_true",
help="input is gtf instead of gff.")
parser.add_option("-g", "--genome-file", dest="genome_file", type="string",
help="filename with genome [default=%default].")
parser.add_option(
"-m", "--merge-adjacent", dest="merge", action="store_true",
help="merge adjacent intervals with the same attributes."
" [default=%default]")
parser.add_option(
"-e", "--feature", dest="feature", type="string",
help="filter by a feature, for example 'exon', 'CDS'."
" If set to the empty string, all entries are output "
"[%default].")
parser.add_option(
"-f", "--maskregions-bed-file", dest="filename_masks",
type="string", metavar="gff",
help="mask sequences with regions given in gff file "
"[%default].")
parser.add_option(
"--remove-masked-regions", dest="remove_masked_regions",
action="store_true",
help="remove regions instead of masking [%default].")
parser.add_option(
"--min-interval-length", dest="min_length", type="int",
help="set minimum length for sequences output "
"[%default]")
parser.add_option(
"--max-length", dest="max_length", type="int",
help="set maximum length for sequences output "
"[%default]")
parser.add_option(
"--extend-at", dest="extend_at", type="choice",
choices=("none", "3", "5", "both", "3only", "5only"),
help="extend at no end, 3', 5' or both ends. If "
"3only or 5only are set, only the added sequence "
"is returned [default=%default]")
parser.add_option(
"--extend-by", dest="extend_by", type="int",
help="extend by # bases [default=%default]")
parser.add_option(
"--extend-with", dest="extend_with", type="string",
help="extend using base [default=%default]")
parser.add_option(
"--masker", dest="masker", type="choice",
choices=("dust", "dustmasker", "softmask", "none"),
help="apply masker [%default].")
parser.add_option(
"--fold-at", dest="fold_at", type="int",
help="fold sequence every n bases[%default].")
parser.add_option(
"--fasta-name-attribute", dest="naming_attribute", type="string",
help="use attribute to name fasta entry. Currently only compatable"
" with gff format [%default].")
parser.set_defaults(
is_gtf=False,
genome_file=None,
merge=False,
feature=None,
filename_masks=None,
remove_masked_regions=False,
min_length=0,
max_length=0,
extend_at=None,
extend_by=100,
extend_with=None,
masker=None,
fold_at=None,
naming_attribute=False
)
(options, args) = E.Start(parser)
if options.genome_file:
fasta = IndexedFasta.IndexedFasta(options.genome_file)
contigs = fasta.getContigSizes()
if options.is_gtf:
iterator = GTF.transcript_iterator(GTF.iterator(options.stdin))
else:
gffs = GTF.iterator(options.stdin)
if options.merge:
iterator = GTF.joined_iterator(gffs)
else:
iterator = GTF.chunk_iterator(gffs)
masks = None
if options.filename_masks:
masks = {}
with open(options.filename_masks, "r") as infile:
e = GTF.readAsIntervals(GTF.iterator(infile))
# convert intervals to intersectors
for contig in e.keys():
intersector = bx.intervals.intersection.Intersecter()
for start, end in e[contig]:
intersector.add_interval(bx.intervals.Interval(start, end))
masks[contig] = intersector
ninput, noutput, nmasked, nskipped_masked = 0, 0, 0, 0
nskipped_length = 0
nskipped_noexons = 0
feature = options.feature
# for item in iterator:
# print len(item) # 3, 2
# for i in item:
# print len(i) # 9, 9, 9, 9, 9
# print i.contig
# print i.strand
# print i.transcript_id
# iterator is a list containing groups (lists) of features.
# Each group of features have in common the same transcript ID, in case of
# GTF files.
for ichunk in iterator:
ninput += 1
if feature:
chunk = filter(lambda x: x.feature == feature, ichunk)
else:
chunk = ichunk
if len(chunk) == 0:
nskipped_noexons += 1
E.info("no features in entry from "
"%s:%i..%i - %s" % (ichunk[0].contig,
ichunk[0].start,
ichunk[0].end,
str(ichunk[0])))
continue
contig, strand = chunk[0].contig, chunk[0].strand
if options.is_gtf:
name = chunk[0].transcript_id
else:
if options.naming_attribute:
attr_dict = {x.split("=")[0]: x.split("=")[1]
for x in chunk[0].attributes.split(";")}
name = attr_dict[options.naming_attribute]
else:
name = str(chunk[0].attributes)
lcontig = contigs[contig]
positive = Genomics.IsPositiveStrand(strand)
intervals = [(x.start, x.end) for x in chunk]
intervals.sort()
if masks:
if contig in masks:
masked_regions = []
for start, end in intervals:
masked_regions += [(x.start, x.end)
for x in masks[contig].find(start, end)]
masked_regions = Intervals.combine(masked_regions)
if len(masked_regions):
nmasked += 1
if options.remove_masked_regions:
intervals = Intervals.truncate(intervals, masked_regions)
else:
raise "unimplemented"
if len(intervals) == 0:
nskipped_masked += 1
if options.loglevel >= 1:
options.stdlog.write("# skipped because fully masked: "
"%s: regions=%s masks=%s\n" %
(name,
str([(x.start,
x.end) for x in chunk]),
masked_regions))
continue
out = intervals
if options.extend_at and not options.extend_with:
if options.extend_at == "5only":
intervals = [(max(0, intervals[0][0] - options.extend_by),
intervals[0][0])]
elif options.extend_at == "3only":
intervals = [(intervals[-1][1],
min(lcontig,
intervals[-1][1] + options.extend_by))]
else:
if options.extend_at in ("5", "both"):
intervals[0] = (max(0,
intervals[0][0] - options.extend_by),
intervals[0][1])
if options.extend_at in ("3", "both"):
intervals[-1] = (intervals[-1][0],
min(lcontig,
intervals[-1][1] + options.extend_by))
if not positive:
intervals = [(lcontig - x[1], lcontig - x[0])
for x in intervals[::-1]]
out.reverse()
s = [fasta.getSequence(contig, strand, start, end)
for start, end in intervals]
# IMS: allow for masking of sequences
s = Masker.maskSequences(s, options.masker)
l = sum([len(x) for x in s])
if (l < options.min_length or
(options.max_length and l > options.max_length)):
nskipped_length += 1
if options.loglevel >= 1:
options.stdlog.write("# skipped because length out of bounds "
"%s: regions=%s len=%i\n" %
(name, str(intervals), l))
continue
if options.extend_at and options.extend_with:
extension = "".join((options.extend_with,) * options.extend_by)
if options.extend_at in ("5", "both"):
s[1] = extension + s[1]
if options.extend_at in ("3", "both"):
s[-1] = s[-1] + extension
if options.fold_at:
n = options.fold_at
s = "".join(s)
seq = "\n".join([s[i:i+n] for i in range(0, len(s), n)])
else:
seq = "\n".join(s)
options.stdout.write(">%s %s:%s:%s\n%s\n" % (name,
contig,
strand,
";".join(
["%i-%i" %
x for x in out]),
seq))
noutput += 1
E.info("ninput=%i, noutput=%i, nmasked=%i, nskipped_noexons=%i, "
"nskipped_masked=%i, nskipped_length=%i" %
(ninput, noutput, nmasked, nskipped_noexons,
nskipped_masked, nskipped_length))
E.Stop()
def main(argv=None):
if not argv:
argv = sys.argv
parser = E.OptionParser(version="%prog version: $Id$", usage=globals()["__doc__"])
parser.add_option(
"--merge-exons-distance",
dest="merge_exons_distance",
type="int",
help="distance in nucleotides between " "exons to be merged [%default].",
)
parser.add_option(
"--pattern-identifier",
dest="pattern",
type="string",
help="pattern to use for renaming genes/transcripts. "
"The pattern should contain a %i, for example "
"--pattern-identifier=ENSG%010i [%default].",
)
parser.add_option(
"--sort-order",
dest="sort_order",
type="choice",
choices=("gene", "gene+transcript", "transcript", "position", "contig+gene", "position+gene", "gene+position"),
help="sort input data [%default].",
)
parser.add_option(
"-u",
"--with-utr",
dest="with_utr",
action="store_true",
help="include utr in merged transcripts " "[%default].",
)
parser.add_option(
"--filter-method",
dest="filter_method",
type="choice",
choices=(
"gene",
"transcript",
"longest-gene",
"longest-transcript",
"representative-transcript",
"proteincoding",
"lincrna",
),
help="Filter method to apply. Available filters are: "
"'gene': filter by gene_id given in ``--map-tsv-file``, "
"'transcript': filter by transcript_id given in ``--map-tsv-file``, "
"'longest-gene': output the longest gene for overlapping genes ,"
"'longest-transcript': output the longest transcript per gene,"
"'representative-transcript': output the representative transcript "
"per gene. The representative transcript is the transcript "
"that shares most exons with other transcripts in a gene. "
"The input needs to be sorted by gene. "
"'proteincoding': only output protein coding features. "
"'lincrna': only output lincRNA features. "
"[%default].",
)
parser.add_option(
"-a",
"--map-tsv-file",
dest="filename_filter",
type="string",
metavar="tsv",
help="filename of ids to map/filter [%default].",
)
parser.add_option(
"--gff-file",
dest="filename_gff",
type="string",
metavar="GFF",
help="second filename of features (see --remove-overlapping) " "[%default]",
)
parser.add_option(
"--invert-filter",
dest="invert_filter",
action="store_true",
help="when using --filter, invert selection " "(like grep -v). " "[%default].",
)
parser.add_option(
"--sample-size",
dest="sample_size",
type="int",
help="extract a random sample of size # if the option "
"'--method=filter --filter-method' is set "
"[%default].",
)
parser.add_option(
"--intron-min-length",
dest="intron_min_length",
type="int",
help="minimum length for introns (for --exons-file2introns) " "[%default].",
)
parser.add_option(
"--min-exons-length",
dest="min_exons_length",
type="int",
help="minimum length for gene (sum of exons) " "(--sam-fileple-size) [%default].",
)
parser.add_option(
"--intron-border",
dest="intron_border",
type="int",
help="number of residues to exclude at intron at either end " "(--exons-file2introns) [%default].",
)
parser.add_option(
"--ignore-strand",
dest="ignore_strand",
action="store_true",
help="remove strandedness of features (set to '.') when "
"using ``transcripts2genes`` or ``filter``"
"[%default].",
)
parser.add_option(
"--permit-duplicates", dest="strict", action="store_false", help="permit duplicate genes. " "[%default]"
)
parser.add_option(
"--duplicate-feature",
dest="duplicate_feature",
type="choice",
choices=("gene", "transcript", "both", "ucsc", "coordinates"),
help="remove duplicates by gene/transcript. "
"If ``ucsc`` is chosen, transcripts ending on _dup# are "
"removed. This is necessary to remove duplicate entries "
"that are next to each other in the sort order "
"[%default]",
)
parser.add_option(
"-m",
"--method",
dest="method",
type="choice",
action="append",
choices=(
"add-protein-id",
"exons2introns",
"filter",
"find-retained-introns",
"genes-to-unique-chunks",
"intersect-transcripts",
"join-exons",
"merge-exons",
"merge-transcripts",
"merge-genes",
"merge-introns",
"remove-overlapping",
"remove-duplicates",
"rename-genes",
"rename-transcripts",
"rename-duplicates",
"renumber-genes",
"renumber-transcripts",
"set-transcript-to-gene",
"set-gene-to-transcript",
"set-protein-to-transcript",
"set-score-to-distance",
"set-gene_biotype-to-source",
"sort",
"transcript2genes",
"unset-genes",
),
help="Method to apply [%default]." "Please only select one.",
)
parser.set_defaults(
sort_order="gene",
filter_method="gene",
pattern="%i",
merge_exons_distance=0,
filename_filter=None,
intron_border=None,
intron_min_length=None,
sample_size=0,
min_exons_length=0,
ignore_strand=False,
with_utr=False,
invert_filter=False,
duplicate_feature=None,
strict=True,
method=None,
)
(options, args) = E.Start(parser, argv=argv)
ninput, noutput, nfeatures, ndiscarded = 0, 0, 0, 0
if options.method is None:
raise ValueError("please specify a --method")
if len(options.method) > 1:
raise ValueError("multiple --method arguements specified")
else:
options.method = options.method[0]
if options.method == "set-transcript-to-gene":
for gff in GTF.iterator(options.stdin):
ninput += 1
gff.setAttribute("transcript_id", gff.gene_id)
options.stdout.write("%s\n" % str(gff))
noutput += 1
nfeatures += 1
elif options.method == "set-gene_biotype-to-source":
for gff in GTF.iterator(options.stdin):
ninput += 1
if "gene_biotype" not in gff:
gff.setAttribute("gene_biotype", gff.source)
options.stdout.write("%s\n" % str(gff))
noutput += 1
nfeatures += 1
elif options.method == "remove-duplicates":
counts = collections.defaultdict(int)
if options.duplicate_feature == "ucsc":
store = []
remove = set()
f = lambda x: x[0].transcript_id
gffs = GTF.transcript_iterator(GTF.iterator(options.stdin), strict=False)
outf = lambda x: "\n".join([str(y) for y in x])
for entry in gffs:
ninput += 1
store.append(entry)
id = f(entry)
if "_dup" in id:
remove.add(re.sub("_dup\d+", "", id))
remove.add(id)
for entry in store:
id = f(entry)
if id not in remove:
options.stdout.write(outf(entry) + "\n")
noutput += 1
else:
ndiscarded += 1
E.info("discarded duplicates for %s" % (id))
else:
if options.duplicate_feature == "gene":
gffs = GTF.gene_iterator(GTF.iterator(options.stdin), strict=False)
f = lambda x: x[0][0].gene_id
outf = lambda x: "\n".join(["\n".join([str(y) for y in xx]) for xx in x])
elif options.duplicate_feature == "transcript":
gffs = GTF.transcript_iterator(GTF.iterator(options.stdin), strict=False)
f = lambda x: x[0].transcript_id
outf = lambda x: "\n".join([str(y) for y in x])
elif options.duplicate_feature == "coordinates":
gffs = GTF.chunk_iterator(GTF.iterator(options.stdin))
f = lambda x: x[0].contig + "_" + str(x[0].start) + "-" + str(x[0].end)
outf = lambda x: "\n".join([str(y) for y in x])
store = []
for entry in gffs:
ninput += 1
store.append(entry)
id = f(entry)
counts[id] += 1
# Assumes GTF file sorted by contig then start
last_id = ""
if options.duplicate_feature == "coordinates":
for entry in store:
id = f(entry)
if id == last_id:
ndiscarded += 1
E.info("discarded duplicates for %s: %i" % (id, counts[id]))
else:
options.stdout.write(outf(entry) + "\n")
noutput += 1
last_id = id
else:
for entry in store:
id = f(entry)
if counts[id] == 1:
options.stdout.write(outf(entry) + "\n")
noutput += 1
else:
ndiscarded += 1
E.info("discarded duplicates for %s: %i" % (id, counts[id]))
elif "sort" == options.method:
for gff in GTF.iterator_sorted(GTF.iterator(options.stdin), sort_order=options.sort_order):
ninput += 1
options.stdout.write("%s\n" % str(gff))
noutput += 1
nfeatures += 1
elif "set-gene-to-transcript" == options.method:
for gff in GTF.iterator(options.stdin):
ninput += 1
gff.setAttribute("gene_id", gff.transcript_id)
options.stdout.write("%s\n" % str(gff))
noutput += 1
nfeatures += 1
elif "set-protein-to-transcript" == options.method:
for gff in GTF.iterator(options.stdin):
ninput += 1
gff.setAttribute("protein_id", gff.transcript_id)
options.stdout.write("%s\n" % str(gff))
noutput += 1
nfeatures += 1
elif "add-protein-id" == options.method:
transcript2protein = IOTools.readMap(IOTools.openFile(options.filename_filter, "r"))
missing = set()
for gff in GTF.iterator(options.stdin):
ninput += 1
if gff.transcript_id not in transcript2protein:
if gff.transcript_id not in missing:
E.debug(("removing transcript '%s' due to " "missing protein id") % gff.transcript_id)
missing.add(gff.transcript_id)
ndiscarded += 1
continue
gff.setAttribute("protein_id", transcript2protein[gff.transcript_id])
options.stdout.write("%s\n" % str(gff))
noutput += 1
nfeatures += 1
E.info("transcripts removed due to missing protein ids: %i" % len(missing))
elif "join-exons" == options.method:
for exons in GTF.transcript_iterator(GTF.iterator(options.stdin)):
ninput += 1
strand = Genomics.convertStrand(exons[0].strand)
contig = exons[0].contig
transid = exons[0].transcript_id
geneid = exons[0].gene_id
biotype = exons[0].source
all_start, all_end = min([x.start for x in exons]), max([x.end for x in exons])
y = GTF.Entry()
y.contig = contig
y.source = biotype
y.feature = "transcript"
y.start = all_start
y.end = all_end
y.strand = strand
y.transcript_id = transid
y.gene_id = geneid
options.stdout.write("%s\n" % str(y))
elif "merge-genes" == options.method:
# merges overlapping genes
#
gffs = GTF.iterator_sorted_chunks(
GTF.flat_gene_iterator(GTF.iterator(options.stdin)), sort_by="contig-strand-start"
)
def iterate_chunks(gff_chunks):
last = gff_chunks.next()
to_join = [last]
for gffs in gff_chunks:
d = gffs[0].start - last[-1].end
if gffs[0].contig == last[0].contig and gffs[0].strand == last[0].strand:
assert gffs[0].start >= last[0].start, (
"input file should be sorted by contig, strand " "and position: d=%i:\nlast=\n%s\nthis=\n%s\n"
) % (d, "\n".join([str(x) for x in last]), "\n".join([str(x) for x in gffs]))
if gffs[0].contig != last[0].contig or gffs[0].strand != last[0].strand or d > 0:
yield to_join
to_join = []
last = gffs
to_join.append(gffs)
yield to_join
raise StopIteration
for chunks in iterate_chunks(gffs):
ninput += 1
if len(chunks) > 1:
gene_id = "merged_%s" % chunks[0][0].gene_id
transcript_id = "merged_%s" % chunks[0][0].transcript_id
info = ",".join([x[0].gene_id for x in chunks])
else:
gene_id = chunks[0][0].gene_id
transcript_id = chunks[0][0].transcript_id
info = None
intervals = []
for c in chunks:
intervals += [(x.start, x.end) for x in c]
intervals = Intervals.combine(intervals)
# take single strand
strand = chunks[0][0].strand
for start, end in intervals:
y = GTF.Entry()
y.fromGTF(chunks[0][0], gene_id, transcript_id)
y.start = start
y.end = end
y.strand = strand
if info:
y.addAttribute("merged", info)
options.stdout.write("%s\n" % str(y))
nfeatures += 1
noutput += 1
elif options.method == "renumber-genes":
map_old2new = {}
for gtf in GTF.iterator(options.stdin):
ninput += 1
if gtf.gene_id not in map_old2new:
map_old2new[gtf.gene_id] = options.pattern % (len(map_old2new) + 1)
gtf.setAttribute("gene_id", map_old2new[gtf.gene_id])
options.stdout.write("%s\n" % str(gtf))
noutput += 1
elif options.method == "unset-genes":
map_old2new = {}
for gtf in GTF.iterator(options.stdin):
ninput += 1
key = gtf.transcript_id
if key not in map_old2new:
map_old2new[key] = options.pattern % (len(map_old2new) + 1)
gtf.setAttribute("gene_id", map_old2new[key])
options.stdout.write("%s\n" % str(gtf))
noutput += 1
elif options.method == "renumber-transcripts":
map_old2new = {}
for gtf in GTF.iterator(options.stdin):
ninput += 1
key = (gtf.gene_id, gtf.transcript_id)
if key not in map_old2new:
map_old2new[key] = options.pattern % (len(map_old2new) + 1)
gtf.setAttribute("transcript_id", map_old2new[key])
options.stdout.write("%s\n" % str(gtf))
noutput += 1
elif options.method == "transcripts2genes":
transcripts = set()
genes = set()
ignore_strand = options.ignore_strand
for gtfs in GTF.iterator_transcripts2genes(GTF.iterator(options.stdin)):
ninput += 1
for gtf in gtfs:
if ignore_strand:
gtf.strand = "."
options.stdout.write("%s\n" % str(gtf))
transcripts.add(gtf.transcript_id)
genes.add(gtf.gene_id)
nfeatures += 1
noutput += 1
E.info("transcripts2genes: transcripts=%i, genes=%i" % (len(transcripts), len(genes)))
elif options.method in ("rename-genes", "rename-transcripts"):
map_old2new = IOTools.readMap(IOTools.openFile(options.filename_filter, "r"))
if options.method == "rename-transcripts":
is_gene_id = False
elif options.method == "rename-genes":
is_gene_id = True
for gff in GTF.iterator(options.stdin):
ninput += 1
if is_gene_id:
if gff.gene_id in map_old2new:
gff.setAttribute("gene_id", map_old2new[gff.gene_id])
else:
E.debug("removing missing gene_id %s" % gff.gene_id)
ndiscarded += 1
continue
else:
if gff.transcript_id in map_old2new:
gff.setAttribute("transcript_id", map_old2new[gff.transcript_id])
else:
E.debug("removing missing transcript_id %s" % gff.transcript_id)
ndiscarded += 1
continue
noutput += 1
options.stdout.write("%s\n" % str(gff))
elif options.method == "filter":
keep_genes = set()
if options.filter_method == "longest-gene":
iterator = GTF.flat_gene_iterator(GTF.iterator(options.stdin))
coords = []
gffs = []
for gff in iterator:
gff.sort(key=lambda x: x.start)
coords.append((gff[0].contig, min([x.start for x in gff]), max([x.end for x in gff]), gff[0].gene_id))
gffs.append(gff)
coords.sort()
last_contig = None
max_end = 0
longest_gene_id = None
longest_length = None
for contig, start, end, gene_id in coords:
ninput += 1
if contig != last_contig or start >= max_end:
if longest_gene_id:
keep_genes.add(longest_gene_id)
longest_gene_id = gene_id
longest_length = end - start
max_end = end
else:
if end - start > longest_length:
longest_length, longest_gene_id = end - start, gene_id
last_contig = contig
max_end = max(max_end, end)
keep_genes.add(longest_gene_id)
invert = options.invert_filter
for gff in gffs:
keep = gff[0].gene_id in keep_genes
if (keep and not invert) or (not keep and invert):
noutput += 1
for g in gff:
nfeatures += 1
options.stdout.write("%s\n" % g)
else:
ndiscarded += 1
elif options.filter_method in ("longest-transcript", "representative-transcript"):
iterator = GTF.gene_iterator(GTF.iterator(options.stdin))
def selectLongestTranscript(gene):
r = []
for transcript in gene:
transcript.sort(key=lambda x: x.start)
length = transcript[-1].end - transcript[0].start
r.append((length, transcript))
r.sort()
return r[-1][1]
def selectRepresentativeTranscript(gene):
"""select a representative transcript.
The representative transcript represent the largest number
of exons over all transcripts.
"""
all_exons = []
for transcript in gene:
all_exons.extend([(x.start, x.end) for x in transcript if x.feature == "exon"])
exon_counts = {}
for key, exons in itertools.groupby(all_exons):
exon_counts[key] = len(list(exons))
transcript_counts = []
for transcript in gene:
count = sum([exon_counts[(x.start, x.end)] for x in transcript if x.feature == "exon"])
# add transcript id to sort to provide a stable
# segmentation.
transcript_counts.append((count, transcript[0].transcript_id, transcript))
transcript_counts.sort()
return transcript_counts[-1][-1]
if options.filter_method == "longest-transcript":
_select = selectLongestTranscript
elif options.filter_method == "representative-transcript":
_select = selectRepresentativeTranscript
for gene in iterator:
ninput += 1
# sort in order to make reproducible which
# gene is chosen.
transcript = _select(sorted(gene))
noutput += 1
for g in transcript:
nfeatures += 1
options.stdout.write("%s\n" % g)
elif options.filter_method in ("gene", "transcript"):
if options.filename_filter:
ids, nerrors = IOTools.ReadList(IOTools.openFile(options.filename_filter, "r"))
E.info("read %i ids" % len(ids))
ids = set(ids)
by_gene = options.filter_method == "gene"
by_transcript = options.filter_method == "transcript"
invert = options.invert_filter
ignore_strand = options.ignore_strand
for gff in GTF.iterator(options.stdin):
ninput += 1
keep = False
if by_gene:
keep = gff.gene_id in ids
if by_transcript:
keep = gff.transcript_id in ids
if (invert and keep) or (not invert and not keep):
continue
if ignore_strand:
gff.strand = "."
options.stdout.write("%s\n" % str(gff))
nfeatures += 1
noutput += 1
elif options.sample_size:
if options.filter_method == "gene":
iterator = GTF.flat_gene_iterator(GTF.iterator(options.stdin))
elif options.filter_method == "transcript":
iterator = GTF.transcript_iterator(GTF.iterator(options.stdin))
if options.min_exons_length:
iterator = GTF.iterator_min_feature_length(
iterator, min_length=options.min_exons_length, feature="exon"
)
data = [x for x in iterator]
ninput = len(data)
if len(data) > options.sample_size:
data = random.sample(data, options.sample_size)
for d in data:
noutput += 1
for dd in d:
nfeatures += 1
options.stdout.write(str(dd) + "\n")
else:
assert False, "please supply either a filename "
"with ids to filter with (--map-tsv-file) or a sample-size."
elif options.filter_method in ("proteincoding", "lincrna", "processed-pseudogene"):
# extract entries by transcript/gene biotype.
# This filter uses a test on the source field (ENSEMBL pre v78)
# a regular expression on the attributes (ENSEMBL >= v78).
tag = {
"proteincoding": "protein_coding",
"processed-pseudogene": "processed_pseudogene",
"lincrna": "lincRNA",
}[options.filter_method]
rx = re.compile('"%s"' % tag)
if not options.invert_filter:
f = lambda x: x.source == tag or rx.search(x.attributes)
else:
f = lambda x: x.source != tag and not rx.search(x.attributes)
for gff in GTF.iterator(options.stdin):
ninput += 1
if f(gff):
options.stdout.write(str(gff) + "\n")
noutput += 1
else:
ndiscarded += 1
elif options.method == "exons2introns":
for gffs in GTF.flat_gene_iterator(GTF.iterator(options.stdin)):
ninput += 1
cds_ranges = GTF.asRanges(gffs, "CDS")
exon_ranges = GTF.asRanges(gffs, "exon")
input_ranges = Intervals.combine(cds_ranges + exon_ranges)
if len(input_ranges) > 1:
last = input_ranges[0][1]
output_ranges = []
for start, end in input_ranges[1:]:
output_ranges.append((last, start))
last = end
if options.intron_border:
b = options.intron_border
output_ranges = [(x[0] + b, x[1] - b) for x in output_ranges]
if options.intron_min_length:
l = options.intron_min_length
output_ranges = [x for x in output_ranges if x[1] - x[0] > l]
for start, end in output_ranges:
entry = GTF.Entry()
entry.copy(gffs[0])
entry.clearAttributes()
entry.transcript_id = "merged"
entry.feature = "intron"
entry.start = start
entry.end = end
options.stdout.write("%s\n" % str(entry))
nfeatures += 1
noutput += 1
else:
ndiscarded += 1
elif options.method == "set-score-to-distance":
for gffs in GTF.transcript_iterator(GTF.iterator(options.stdin)):
ninput += 1
strand = Genomics.convertStrand(gffs[0].strand)
all_start, all_end = min([x.start for x in gffs]), max([x.end for x in gffs])
if strand != ".":
t = 0
if strand == "-":
gffs.reverse()
for gff in gffs:
gff.score = t
t += gff.end - gff.start
if strand == "-":
gffs.reverse()
for gff in gffs:
options.stdout.write("%s\n" % str(gff))
nfeatures += 1
noutput += 1
elif options.method == "remove-overlapping":
index = GTF.readAndIndex(GTF.iterator(IOTools.openFile(options.filename_gff, "r")))
for gffs in GTF.transcript_iterator(GTF.iterator(options.stdin)):
ninput += 1
found = False
for e in gffs:
if index.contains(e.contig, e.start, e.end):
found = True
break
if found:
ndiscarded += 1
else:
noutput += 1
for e in gffs:
nfeatures += 1
options.stdout.write("%s\n" % str(e))
elif options.method == "intersect-transcripts":
for gffs in GTF.gene_iterator(GTF.iterator(options.stdin), strict=options.strict):
ninput += 1
r = []
for g in gffs:
if options.with_utr:
ranges = GTF.asRanges(g, "exon")
else:
ranges = GTF.asRanges(g, "CDS")
r.append(ranges)
result = r[0]
for x in r[1:]:
result = Intervals.intersect(result, x)
entry = GTF.Entry()
entry.copy(gffs[0][0])
entry.clearAttributes()
entry.transcript_id = "merged"
entry.feature = "exon"
for start, end in result:
entry.start = start
entry.end = end
options.stdout.write("%s\n" % str(entry))
nfeatures += 1
noutput += 1
elif "rename-duplicates" == options.method:
# note: this will only rename entries with "CDS" in feature column
assert options.duplicate_feature in ["gene", "transcript", "both"], (
"for renaming duplicates, --duplicate-feature must be set to one " "of 'gene', transcript' or 'both'"
)
gene_ids = list()
transcript_ids = list()
gtfs = list()
for gtf in GTF.iterator(options.stdin):
gtfs.append(gtf)
if gtf.feature == "CDS":
gene_ids.append(gtf.gene_id)
transcript_ids.append(gtf.transcript_id)
dup_gene = [item for item in set(gene_ids) if gene_ids.count(item) > 1]
dup_transcript = [item for item in set(transcript_ids) if transcript_ids.count(item) > 1]
E.info("Number of duplicated gene_ids: %i" % len(dup_gene))
E.info("Number of duplicated transcript_ids: %i" % len(dup_transcript))
gene_dict = dict(zip(dup_gene, ([0] * len(dup_gene))))
transcript_dict = dict(zip(dup_transcript, ([0] * len(dup_transcript))))
for gtf in gtfs:
if gtf.feature == "CDS":
if options.duplicate_feature in ["both", "gene"]:
if gtf.gene_id in dup_gene:
gene_dict[gtf.gene_id] = gene_dict[gtf.gene_id] + 1
gtf.setAttribute("gene_id", gtf.gene_id + "." + str(gene_dict[gtf.gene_id]))
if options.duplicate_feature in ["both", "transcript"]:
if gtf.transcript_id in dup_transcript:
transcript_dict[gtf.transcript_id] = transcript_dict[gtf.transcript_id] + 1
gtf.setAttribute(
"transcript_id", gtf.transcript_id + "." + str(transcript_dict[gtf.transcript_id])
)
options.stdout.write("%s\n" % gtf)
elif options.method in ("merge-exons", "merge-introns", "merge-transcripts"):
for gffs in GTF.flat_gene_iterator(GTF.iterator(options.stdin), strict=options.strict):
ninput += 1
cds_ranges = GTF.asRanges(gffs, "CDS")
exon_ranges = GTF.asRanges(gffs, "exon")
# sanity checks
strands = set([x.strand for x in gffs])
contigs = set([x.contig for x in gffs])
if len(strands) > 1:
raise ValueError("can not merge gene '%s' on multiple strands: %s" % (gffs[0].gene_id, str(strands)))
if len(contigs) > 1:
raise ValueError("can not merge gene '%s' on multiple contigs: %s" % (gffs[0].gene_id, str(contigs)))
strand = Genomics.convertStrand(gffs[0].strand)
if cds_ranges and options.with_utr:
cds_start, cds_end = cds_ranges[0][0], cds_ranges[-1][1]
midpoint = (cds_end - cds_start) / 2 + cds_start
utr_ranges = []
for start, end in Intervals.truncate(exon_ranges, cds_ranges):
if end - start > 3:
if strand == ".":
feature = "UTR"
elif strand == "+":
if start < midpoint:
feature = "UTR5"
else:
feature = "UTR3"
elif strand == "-":
if start < midpoint:
feature = "UTR3"
else:
feature = "UTR5"
utr_ranges.append((feature, start, end))
output_feature = "CDS"
output_ranges = cds_ranges
else:
output_feature = "exon"
output_ranges = exon_ranges
utr_ranges = []
result = []
try:
biotypes = [x["gene_biotype"] for x in gffs]
biotype = ":".join(set(biotypes))
except (KeyError, AttributeError):
biotype = None
if options.method == "merge-exons":
# need to combine per feature - skip
# utr_ranges = Intervals.combineAtDistance(
# utr_ranges,
# options.merge_exons_distance)
output_ranges = Intervals.combineAtDistance(output_ranges, options.merge_exons_distance)
for feature, start, end in utr_ranges:
entry = GTF.Entry()
entry.copy(gffs[0])
entry.clearAttributes()
entry.feature = feature
entry.transcript_id = "merged"
if biotype:
entry.addAttribute("gene_biotype", biotype)
entry.start = start
entry.end = end
result.append(entry)
for start, end in output_ranges:
entry = GTF.Entry()
entry.copy(gffs[0])
entry.clearAttributes()
entry.transcript_id = "merged"
if biotype:
entry.addAttribute("gene_biotype", biotype)
entry.feature = output_feature
entry.start = start
entry.end = end
result.append(entry)
elif options.method == "merge-transcripts":
entry = GTF.Entry()
entry.copy(gffs[0])
entry.clearAttributes()
entry.transcript_id = entry.gene_id
if biotype:
entry.addAttribute("gene_biotype", biotype)
entry.start = output_ranges[0][0]
entry.end = output_ranges[-1][1]
result.append(entry)
elif options.method == "merge-introns":
if len(output_ranges) >= 2:
entry = GTF.Entry()
entry.copy(gffs[0])
entry.clearAttributes()
entry.transcript_id = entry.gene_id
if biotype:
entry.addAttribute("gene_biotype", biotype)
entry.start = output_ranges[0][1]
entry.end = output_ranges[-1][0]
result.append(entry)
else:
ndiscarded += 1
continue
result.sort(key=lambda x: x.start)
for x in result:
options.stdout.write("%s\n" % str(x))
nfeatures += 1
noutput += 1
elif options.method == "find-retained-introns":
for gene in GTF.gene_iterator(GTF.iterator(options.stdin)):
ninput += 1
found_any = False
for intron in find_retained_introns(gene):
found_any = True
options.stdout.write("%s\n" % str(intron))
nfeatures += 1
if found_any:
noutput += 1
elif options.method == "genes-to-unique-chunks":
for gene in GTF.flat_gene_iterator(GTF.iterator(options.stdin)):
ninput += 1
for exon in gene_to_blocks(gene):
options.stdout.write("%s\n" % str(exon))
nfeatures += 1
noutput += 1
else:
raise ValueError("unknown method '%s'" % options.method)
E.info("ninput=%i, noutput=%i, nfeatures=%i, ndiscarded=%i" % (ninput, noutput, nfeatures, ndiscarded))
E.Stop()
def testEmpty(self):
"""test empty input."""
self.assertEqual(Intervals.truncate([], []), [])
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv == None: argv = sys.argv
parser = E.OptionParser(
version=
"%prog version: $Id: gff2fasta.py 2861 2010-02-23 17:36:32Z andreas $",
usage=globals()["__doc__"])
parser.add_option("--is-gtf",
dest="is_gtf",
action="store_true",
help="input is gtf instead of gff.")
parser.add_option("-g",
"--genome-file",
dest="genome_file",
type="string",
help="filename with genome [default=%default].")
parser.add_option(
"-m",
"--merge",
dest="merge",
action="store_true",
help="merge adjacent intervals with the same attributes. "
"[default=%default]")
parser.add_option(
"-e",
"--feature",
dest="feature",
type="string",
help="filter by a feature, for example 'exon', 'CDS'. If "
"set to the empty string, all entries are output [%default].")
parser.add_option(
"-f",
"--filename-masks",
dest="filename_masks",
type="string",
metavar="gff",
help="mask sequences with regions given in gff file [%default].")
parser.add_option("--remove-masked-regions",
dest="remove_masked_regions",
action="store_true",
help="remove regions instead of masking [%default].")
parser.add_option(
"--min-length",
dest="min_length",
type="int",
help="set minimum length for sequences output [%default]")
parser.add_option(
"--max-length",
dest="max_length",
type="int",
help="set maximum length for sequences output [%default]")
parser.add_option("--extend-at",
dest="extend_at",
type="choice",
choices=("none", "3", "5", "both", "3only", "5only"),
help="extend at no end, 3', 5' or both ends. If "
"3only or 5only are set, only the added sequence "
"is returned [default=%default]")
parser.add_option("--extend-by",
dest="extend_by",
type="int",
help="extend by # bases [default=%default]")
parser.add_option("--masker",
dest="masker",
type="choice",
choices=("dust", "dustmasker", "softmask", "none"),
help="apply masker [%default].")
parser.set_defaults(is_gtf=False,
genome_file=None,
merge=False,
feature=None,
filename_masks=None,
remove_masked_regions=False,
min_length=0,
max_length=0,
extend_at=None,
extend_by=100,
masker=None)
(options, args) = E.Start(parser)
if options.genome_file:
fasta = IndexedFasta.IndexedFasta(options.genome_file)
contigs = fasta.getContigSizes()
if options.is_gtf:
iterator = GTF.transcript_iterator(GTF.iterator(sys.stdin))
else:
gffs = GTF.iterator(sys.stdin)
if options.merge:
iterator = GTF.joined_iterator(gffs)
else:
iterator = GTF.chunk_iterator(gffs)
masks = None
if options.filename_masks:
masks = {}
with open(options.filename_masks, "r") as infile:
e = GTF.readAsIntervals(GFF.iterator(infile))
# convert intervals to intersectors
for contig in e.keys():
intersector = bx.intervals.intersection.Intersecter()
for start, end in e[contig]:
intersector.add_interval(bx.intervals.Interval(start, end))
masks[contig] = intersector
ninput, noutput, nmasked, nskipped_masked = 0, 0, 0, 0
nskipped_length = 0
nskipped_noexons = 0
feature = options.feature
# for item in iterator:
# print len(item) # 3, 2
# for i in item:
# print len(i) # 9, 9, 9, 9, 9
# print i.contig
# print i.strand
# print i.transcript_id
# iterator is a list containing groups (lists) of features.
# Each group of features have in common the same transcript ID, in case of GTF files.
for ichunk in iterator:
ninput += 1
if feature:
chunk = filter(lambda x: x.feature == feature, ichunk)
else:
chunk = ichunk
if len(chunk) == 0:
nskipped_noexons += 1
E.info("no features in entry from %s:%i..%i - %s" %
(ichunk[0].contig, ichunk[0].start, ichunk[0].end,
str(ichunk[0])))
continue
contig, strand = chunk[0].contig, chunk[0].strand
if options.is_gtf:
name = chunk[0].transcript_id
else:
name = str(chunk[0].attributes)
lcontig = contigs[contig]
positive = Genomics.IsPositiveStrand(strand)
intervals = [(x.start, x.end) for x in chunk]
intervals.sort()
if masks:
if contig in masks:
masked_regions = []
for start, end in intervals:
masked_regions += [(x.start, x.end)
for x in masks[contig].find(start, end)]
masked_regions = Intervals.combine(masked_regions)
if len(masked_regions): nmasked += 1
if options.remove_masked_regions:
intervals = Intervals.truncate(intervals, masked_regions)
else:
raise "unimplemented"
if len(intervals) == 0:
nskipped_masked += 1
if options.loglevel >= 1:
options.stdlog.write( "# skipped because fully masked: %s: regions=%s masks=%s\n" %\
(name, str([ (x.start, x.end) for x in chunk ]), masked_regions) )
continue
out = intervals
if options.extend_at:
if options.extend_at == "5only":
intervals = [(max(0, intervals[0][0] - options.extend_by),
intervals[0][0])]
elif options.extend_at == "3only":
intervals = [(intervals[-1][1],
min(lcontig,
intervals[-1][1] + options.extend_by))]
else:
if options.extend_at in ("5", "both"):
intervals[0] = (max(0,
intervals[0][0] - options.extend_by),
intervals[0][1])
if options.extend_at in ("3", "both"):
intervals[-1] = (intervals[-1][0],
min(lcontig,
intervals[-1][1] + options.extend_by))
if not positive:
intervals = [(lcontig - x[1], lcontig - x[0])
for x in intervals[::-1]]
out.reverse()
s = [
fasta.getSequence(contig, strand, start, end)
for start, end in intervals
]
#IMS: allow for masking of sequences
s = Masker.maskSequences(s, options.masker)
l = sum([len(x) for x in s])
if l < options.min_length or (options.max_length
and l > options.max_length):
nskipped_length += 1
if options.loglevel >= 1:
options.stdlog.write( "# skipped because length out of bounds %s: regions=%s len=%i\n" %\
(name, str(intervals), l) )
continue
options.stdout.write(
">%s %s:%s:%s\n%s\n" %
(name, contig, strand, ";".join(["%i-%i" % x
for x in out]), "\n".join(s)))
noutput += 1
E.info( "ninput=%i, noutput=%i, nmasked=%i, nskipped_noexons=%i, nskipped_masked=%i, nskipped_length=%i" %\
(ninput, noutput, nmasked, nskipped_noexons, nskipped_masked, nskipped_length ) )
E.Stop()
def main(argv=None):
if not argv:
argv = sys.argv
parser = E.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
parser.add_option("-m",
"--merge-exons",
dest="merge_exons",
action="store_true",
help="merge overlapping exons of all transcripts "
"within a gene. "
"The merged exons will be output. "
"Input needs to sorted by gene [default=%default].")
parser.add_option("-t",
"--merge-transcripts",
dest="merge_transcripts",
action="store_true",
help="merge all transcripts within a gene. "
"The entry will span the whole gene "
"(exons and introns). "
"The transcript does not include the UTR unless "
"--with-utr is set. [default=%default].")
parser.add_option("--merge-genes",
dest="merge_genes",
action="store_true",
help="merge overlapping genes if their exons overlap. "
"A gene with a single transcript containing all exons "
"of the overlapping transcripts will be output. "
"This operation ignores strand information "
"The input needs te sorted by transcript "
"[default=%default].")
parser.add_option("--merge-exons-distance",
dest="merge_exons_distance",
type="int",
help="distance in nucleotides between "
"exons to be merged [default=%default].")
parser.add_option("-j",
"--join-exons",
dest="join_exons",
action="store_true",
help="join all exons per transcript. "
"A new transcript will be "
"output that spans a whole transcript. "
"Input needs to be sorted by transcript "
"[default=%default].")
parser.add_option("--unset-genes",
dest="unset_genes",
type="string",
help="unset gene identifiers, keeping "
"transcripts intact. "
"New gene identifiers are set to the "
"pattern given. For example, "
"'--unset-genes=%06i' [default=%default].")
parser.add_option("--sort",
dest="sort",
type="choice",
choices=("gene", "gene+transcript", "transcript",
"position", "contig+gene", "position+gene",
"gene+position"),
help="sort input data [default=%default].")
parser.add_option("-u",
"--with-utr",
dest="with_utr",
action="store_true",
help="include utr in merged transcripts "
"[default=%default].")
parser.add_option("--intersect-transcripts",
dest="intersect_transcripts",
action="store_true",
help="intersect all transcripts within a gene. "
"The entry will only span those bases "
"that are covered by all transcrips."
"The transcript does not include the UTR unless "
"--with-utr is set. This method "
"will remove all other features (stop_codon, etc.) "
"The input needs to be sorted by gene. "
"[default=%default].")
parser.add_option("-i",
"--merge-introns",
dest="merge_introns",
action="store_true",
help="merge and output all introns within a "
"gene. The output will contain "
"all intronic regions within a gene. Single exon genes "
"are skipped. "
"The input needs to be sorted by gene. "
"[default=%default].")
parser.add_option("-g",
"--set-transcript-to-gene",
"--set-transcript2gene",
dest="set_transcript2gene",
action="store_true",
help="set the transcript_id to the "
"gene_id [default=%default].")
parser.add_option("--set-protein-to-transcript",
dest="set_protein2transcript",
action="store_true",
help="set the protein_id to the "
"transcript_id [default=%default].")
parser.add_option("--add-protein-id",
dest="add_protein_id",
type="string",
help="add a protein_id for each transcript_id. "
"The argument is a filename containing a mapping "
"between "
"transcript_id to protein_id [default=%default].")
parser.add_option("-G",
"--set-gene-to-transcript",
"--set-gene2transcript",
dest="set_gene2transcript",
action="store_true",
help="set the gene_id to the "
"transcript_id [default=%default].")
parser.add_option("-d",
"--set-score2distance",
dest="set_score2distance",
action="store_true",
help="set the score field for each feature to the "
"distance to "
"transcription start site [default=%default].")
parser.add_option("--exons2introns",
dest="exons2introns",
action="store_true",
help="for each gene build an 'intronic' transcript "
"containing the union of all intronic regions "
"of all transcripts in a gene."
"The features are labeled as 'intron'."
"The input needs to be sorted by gene. "
"[default=%default].")
parser.add_option("-f",
"--filter",
dest="filter",
type="choice",
choices=("gene", "transcript", "longest-gene",
"longest-transcript",
"representative-transcript"),
help="apply a filter to the input file. Available "
"filters are: "
"'gene': filter by gene_id, "
"'transcript': filter by transcript_id, "
"'longest-gene': output the longest gene for "
"overlapping genes ,"
"'longest-transcript': output the longest "
"transcript per gene,"
"'representative-transcript': output the "
"representative transcript per gene. "
"The representative transcript is the transcript "
"that shares most exons with "
"the other transcripts in a gene. "
"The input needs to be sorted by gene. "
"[default=%default].")
parser.add_option("-r",
"--rename",
dest="rename",
type="choice",
choices=("gene", "transcript"),
help="rename genes or transcripts with a map "
"given by the option `--apply`. "
"Those that can not be renamed are removed "
"[default=%default].")
parser.add_option("--renumber-genes",
dest="renumber_genes",
type="string",
help="renumber genes according to the given pattern. "
"[default=%default].")
parser.add_option("--renumber-transcripts",
dest="renumber_transcripts",
type="string",
help="renumber transcripts according to the "
"given pattern. "
"[default=%default].")
parser.add_option("-a",
"--apply",
dest="filename_filter",
type="string",
metavar="tsv",
help="filename of ids to map/filter [default=%default].")
parser.add_option("--invert-filter",
dest="invert_filter",
action="store_true",
help="when using --filter, invert selection "
"(like grep -v). "
"[default=%default].")
parser.add_option("--sample-size",
dest="sample_size",
type="int",
help="extract a random sample of size # if the option "
"'--filter' is set[default=%default].")
parser.add_option("--intron-min-length",
dest="intron_min_length",
type="int",
help="minimum length for introns (for --exons2introns) "
"[default=%default].")
parser.add_option("--min-exons-length",
dest="min_exons_length",
type="int",
help="minimum length for gene (sum of exons) "
"(--sample-size) [default=%default].")
parser.add_option(
"--intron-border",
dest="intron_border",
type="int",
help="number of residues to exclude at intron at either end "
"(--exons2introns) [default=%default].")
parser.add_option("--transcripts2genes",
dest="transcripts2genes",
action="store_true",
help="cluster overlapping transcripts into genes.")
parser.add_option("--reset-strand",
dest="reset_strand",
action="store_true",
help="remove strandedness of features (set to '.') when "
"using --transcripts2genes"
"[default=%default].")
parser.add_option("--remove-overlapping",
dest="remove_overlapping",
type="string",
metavar="gff",
help="remove all transcripts that overlap intervals "
"in a gff-formatted file."
"The comparison ignores strand "
"[default=%default].")
parser.add_option("--permit-duplicates",
dest="strict",
action="store_false",
help="permit duplicate genes. "
"[default=%default]")
parser.add_option("--remove-duplicates",
dest="remove_duplicates",
type="choice",
choices=("gene", "transcript", "ucsc", "coordinates"),
help="remove duplicates by gene/transcript. "
"If ``ucsc`` is chosen, transcripts ending on _dup# are "
"removed. This is necessary to remove duplicate entries "
"that are next to each other in the sort order "
"[%default]")
parser.add_option("--rename-duplicates",
dest="rename_duplicates",
action="store_true",
help="rename duplicate gene_ids and transcript_ids by "
"addition of a numerical suffix")
parser.set_defaults(
sort=None,
merge_exons=False,
join_exons=False,
merge_exons_distance=0,
merge_transcripts=False,
set_score2distance=False,
set_gene2transcript=False,
set_transcript2gene=False,
set_protein2transcript=False,
add_protein_id=None,
filename_filter=None,
filter=None,
exons2introns=None,
merge_genes=False,
intron_border=None,
intron_min_length=None,
sample_size=0,
min_exons_length=0,
transripts2genes=False,
reset_strand=False,
with_utr=False,
invert_filter=False,
remove_duplicates=None,
remove_overlapping=None,
renumber_genes=None,
unset_genes=None,
renumber_transcripts=None,
strict=True,
intersect_transcripts=False,
rename_duplicates=False,
)
(options, args) = E.Start(parser, argv=argv)
ninput, noutput, nfeatures, ndiscarded = 0, 0, 0, 0
if options.set_transcript2gene:
for gff in GTF.iterator(options.stdin):
ninput += 1
gff.setAttribute("transcript_id", gff.gene_id)
options.stdout.write("%s\n" % str(gff))
noutput += 1
nfeatures += 1
elif options.remove_duplicates:
counts = collections.defaultdict(int)
if options.remove_duplicates == "ucsc":
store = []
remove = set()
f = lambda x: x[0].transcript_id
gffs = GTF.transcript_iterator(GTF.iterator(options.stdin),
strict=False)
outf = lambda x: "\n".join([str(y) for y in x])
for entry in gffs:
ninput += 1
store.append(entry)
id = f(entry)
if "_dup" in id:
remove.add(re.sub("_dup\d+", "", id))
remove.add(id)
for entry in store:
id = f(entry)
if id not in remove:
options.stdout.write(outf(entry) + "\n")
noutput += 1
else:
ndiscarded += 1
E.info("discarded duplicates for %s" % (id))
else:
if options.remove_duplicates == "gene":
gffs = GTF.gene_iterator(GTF.iterator(options.stdin),
strict=False)
f = lambda x: x[0][0].gene_id
outf = lambda x: "\n".join(
["\n".join([str(y) for y in xx]) for xx in x])
elif options.remove_duplicates == "transcript":
gffs = GTF.transcript_iterator(GTF.iterator(options.stdin),
strict=False)
f = lambda x: x[0].transcript_id
outf = lambda x: "\n".join([str(y) for y in x])
elif options.remove_duplicates == "coordinates":
gffs = GTF.chunk_iterator(GTF.iterator(options.stdin))
f = lambda x: x[0].contig + "_" + \
str(x[0].start) + "-" + str(x[0].end)
outf = lambda x: "\n".join([str(y) for y in x])
store = []
for entry in gffs:
ninput += 1
store.append(entry)
id = f(entry)
counts[id] += 1
# Assumes GTF file sorted by contig then start
last_id = ""
if options.remove_duplicates == "coordinates":
for entry in store:
id = f(entry)
if id == last_id:
ndiscarded += 1
E.info("discarded duplicates for %s: %i" %
(id, counts[id]))
else:
options.stdout.write(outf(entry) + "\n")
noutput += 1
last_id = id
else:
for entry in store:
id = f(entry)
if counts[id] == 1:
options.stdout.write(outf(entry) + "\n")
noutput += 1
else:
ndiscarded += 1
E.info("discarded duplicates for %s: %i" %
(id, counts[id]))
elif options.sort:
for gff in GTF.iterator_sorted(GTF.iterator(options.stdin),
sort_order=options.sort):
ninput += 1
options.stdout.write("%s\n" % str(gff))
noutput += 1
nfeatures += 1
elif options.set_gene2transcript:
for gff in GTF.iterator(options.stdin):
ninput += 1
gff.setAttribute("gene_id", gff.transcript_id)
options.stdout.write("%s\n" % str(gff))
noutput += 1
nfeatures += 1
elif options.set_protein2transcript:
for gff in GTF.iterator(options.stdin):
ninput += 1
gff.setAttribute("protein_id", gff.transcript_id)
options.stdout.write("%s\n" % str(gff))
noutput += 1
nfeatures += 1
elif options.add_protein_id:
transcript2protein = IOTools.readMap(open(options.add_protein_id, "r"))
missing = set()
for gff in GTF.iterator(options.stdin):
ninput += 1
if gff.transcript_id not in transcript2protein:
if gff.transcript_id not in missing:
E.debug(("removing transcript '%s' due to "
"missing protein id") % gff.transcript_id)
missing.add(gff.transcript_id)
ndiscarded += 1
continue
gff.setAttribute("protein_id",
transcript2protein[gff.transcript_id])
options.stdout.write("%s\n" % str(gff))
noutput += 1
nfeatures += 1
E.info("transcripts removed due to missing protein ids: %i" %
len(missing))
elif options.join_exons:
for exons in GTF.transcript_iterator(GTF.iterator(options.stdin)):
ninput += 1
strand = Genomics.convertStrand(exons[0].strand)
contig = exons[0].contig
transid = exons[0].transcript_id
geneid = exons[0].gene_id
biotype = exons[0].source
all_start, all_end = min([x.start for x in exons
]), max([x.end for x in exons])
y = GTF.Entry()
y.contig = contig
y.source = biotype
y.feature = "transcript"
y.start = all_start
y.end = all_end
y.strand = strand
y.transcript_id = transid
y.gene_id = geneid
options.stdout.write("%s\n" % str(y))
elif options.merge_genes:
# merges overlapping genes
#
gffs = GTF.iterator_sorted_chunks(GTF.flat_gene_iterator(
GTF.iterator(options.stdin)),
sort_by="contig-strand-start")
def iterate_chunks(gff_chunks):
last = gff_chunks.next()
to_join = [last]
for gffs in gff_chunks:
d = gffs[0].start - last[-1].end
if gffs[0].contig == last[0].contig and \
gffs[0].strand == last[0].strand:
assert gffs[0].start >= last[0].start, \
("input file should be sorted by contig, strand "
"and position: d=%i:\nlast=\n%s\nthis=\n%s\n") % \
(d,
"\n".join([str(x) for x in last]),
"\n".join([str(x) for x in gffs]))
if gffs[0].contig != last[0].contig or \
gffs[0].strand != last[0].strand or \
d > 0:
yield to_join
to_join = []
last = gffs
to_join.append(gffs)
yield to_join
raise StopIteration
for chunks in iterate_chunks(gffs):
ninput += 1
if len(chunks) > 1:
gene_id = "merged_%s" % chunks[0][0].gene_id
transcript_id = "merged_%s" % chunks[0][0].transcript_id
info = ",".join([x[0].gene_id for x in chunks])
else:
gene_id = chunks[0][0].gene_id
transcript_id = chunks[0][0].transcript_id
info = None
intervals = []
for c in chunks:
intervals += [(x.start, x.end) for x in c]
intervals = Intervals.combine(intervals)
# take single strand
strand = chunks[0][0].strand
for start, end in intervals:
y = GTF.Entry()
y.fromGTF(chunks[0][0], gene_id, transcript_id)
y.start = start
y.end = end
y.strand = strand
if info:
y.addAttribute("merged", info)
options.stdout.write("%s\n" % str(y))
nfeatures += 1
noutput += 1
elif options.renumber_genes:
map_old2new = {}
for gtf in GTF.iterator(options.stdin):
ninput += 1
if gtf.gene_id not in map_old2new:
map_old2new[gtf.gene_id] = options.renumber_genes % (
len(map_old2new) + 1)
gtf.setAttribute("gene_id", map_old2new[gtf.gene_id])
options.stdout.write("%s\n" % str(gtf))
noutput += 1
elif options.unset_genes:
map_old2new = {}
for gtf in GTF.iterator(options.stdin):
ninput += 1
key = gtf.transcript_id
if key not in map_old2new:
map_old2new[key] = options.unset_genes % (len(map_old2new) + 1)
gtf.setAttribute("gene_id", map_old2new[key])
options.stdout.write("%s\n" % str(gtf))
noutput += 1
elif options.renumber_transcripts:
map_old2new = {}
for gtf in GTF.iterator(options.stdin):
ninput += 1
key = (gtf.gene_id, gtf.transcript_id)
if key not in map_old2new:
map_old2new[key] = options.renumber_transcripts % (
len(map_old2new) + 1)
gtf.setAttribute("transcript_id", map_old2new[key])
options.stdout.write("%s\n" % str(gtf))
noutput += 1
elif options.transcripts2genes:
transcripts = set()
genes = set()
reset_strand = options.reset_strand
for gtfs in GTF.iterator_transcripts2genes(GTF.iterator(
options.stdin)):
ninput += 1
for gtf in gtfs:
if reset_strand:
gtf.strand = "."
options.stdout.write("%s\n" % str(gtf))
transcripts.add(gtf.transcript_id)
genes.add(gtf.gene_id)
nfeatures += 1
noutput += 1
E.info("transcripts2genes: transcripts=%i, genes=%i" %
(len(transcripts), len(genes)))
elif options.rename:
map_old2new = IOTools.readMap(open(options.filename_filter, "r"))
if options.rename == "transcript":
is_gene_id = False
elif options.rename == "gene":
is_gene_id = True
for gff in GTF.iterator(options.stdin):
ninput += 1
if is_gene_id:
if gff.gene_id in map_old2new:
gff.setAttribute("gene_id", map_old2new[gff.gene_id])
else:
E.debug("removing missing gene_id %s" % gff.gene_id)
ndiscarded += 1
continue
else:
if gff.transcript_id in map_old2new:
gff.setAttribute("transcript_id",
map_old2new[gff.transcript_id])
else:
E.debug("removing missing transcript_id %s" %
gff.transcript_id)
ndiscarded += 1
continue
noutput += 1
options.stdout.write("%s\n" % str(gff))
elif options.filter:
keep_genes = set()
if options.filter == "longest-gene":
iterator = GTF.flat_gene_iterator(GTF.iterator(options.stdin))
coords = []
gffs = []
for gff in iterator:
gff.sort(key=lambda x: x.start)
coords.append((gff[0].contig, min([x.start for x in gff]),
max([x.end for x in gff]), gff[0].gene_id))
gffs.append(gff)
coords.sort()
last_contig = None
max_end = 0
longest_gene_id = None
longest_length = None
for contig, start, end, gene_id in coords:
ninput += 1
if contig != last_contig or start >= max_end:
if longest_gene_id:
keep_genes.add(longest_gene_id)
longest_gene_id = gene_id
longest_length = end - start
max_end = end
else:
if end - start > longest_length:
longest_length, longest_gene_id = end - start, gene_id
last_contig = contig
max_end = max(max_end, end)
keep_genes.add(longest_gene_id)
invert = options.invert_filter
for gff in gffs:
keep = gff[0].gene_id in keep_genes
if (keep and not invert) or (not keep and invert):
noutput += 1
for g in gff:
nfeatures += 1
options.stdout.write("%s\n" % g)
else:
ndiscarded += 1
elif options.filter in ("longest-transcript",
"representative-transcript"):
iterator = GTF.gene_iterator(GTF.iterator(options.stdin))
def selectLongestTranscript(gene):
r = []
for transcript in gene:
transcript.sort(key=lambda x: x.start)
length = transcript[-1].end - transcript[0].start
r.append((length, transcript))
r.sort()
return r[-1][1]
def selectRepresentativeTranscript(gene):
'''select a representative transcript.
The representative transcript represent the largest number
of exons over all transcripts.
'''
all_exons = []
for transcript in gene:
all_exons.extend([(x.start, x.end) for x in transcript
if x.feature == "exon"])
exon_counts = {}
for key, exons in itertools.groupby(all_exons):
exon_counts[key] = len(list(exons))
transcript_counts = []
for transcript in gene:
count = sum([
exon_counts[(x.start, x.end)] for x in transcript
if x.feature == "exon"
])
transcript_counts.append((count, transcript))
transcript_counts.sort()
return transcript_counts[-1][1]
if options.filter == "longest-transcript":
_select = selectLongestTranscript
elif options.filter == "representative-transcript":
_select = selectRepresentativeTranscript
for gene in iterator:
ninput += 1
# sort in order to make reproducible which
# gene is chosen.
transcript = _select(sorted(gene))
noutput += 1
for g in transcript:
nfeatures += 1
options.stdout.write("%s\n" % g)
elif options.filter in ("gene", "transcript"):
if options.filename_filter:
ids, nerrors = IOTools.ReadList(
open(options.filename_filter, "r"))
E.info("read %i ids" % len(ids))
ids = set(ids)
by_gene = options.filter == "gene"
by_transcript = options.filter == "transcript"
invert = options.invert_filter
reset_strand = options.reset_strand
for gff in GTF.iterator(options.stdin):
ninput += 1
keep = False
if by_gene:
keep = gff.gene_id in ids
if by_transcript:
keep = gff.transcript_id in ids
if (invert and keep) or (not invert and not keep):
continue
if reset_strand:
gff.strand = "."
options.stdout.write("%s\n" % str(gff))
nfeatures += 1
noutput += 1
elif options.sample_size:
if options.filter == "gene":
iterator = GTF.flat_gene_iterator(
GTF.iterator(options.stdin))
elif options.filter == "transcript":
iterator = GTF.transcript_iterator(
GTF.iterator(options.stdin))
if options.min_exons_length:
iterator = GTF.iterator_min_feature_length(
iterator,
min_length=options.min_exons_length,
feature="exon")
data = [x for x in iterator]
ninput = len(data)
if len(data) > options.sample_size:
data = random.sample(data, options.sample_size)
for d in data:
noutput += 1
for dd in d:
nfeatures += 1
options.stdout.write(str(dd) + "\n")
else:
assert False, "please supply either a filename "
"with ids to filter with (--apply) or a sample-size."
elif options.exons2introns:
for gffs in GTF.flat_gene_iterator(GTF.iterator(options.stdin)):
ninput += 1
cds_ranges = GTF.asRanges(gffs, "CDS")
exon_ranges = GTF.asRanges(gffs, "exon")
input_ranges = Intervals.combine(cds_ranges + exon_ranges)
if len(input_ranges) > 1:
last = input_ranges[0][1]
output_ranges = []
for start, end in input_ranges[1:]:
output_ranges.append((last, start))
last = end
if options.intron_border:
b = options.intron_border
output_ranges = [(x[0] + b, x[1] - b)
for x in output_ranges]
if options.intron_min_length:
l = options.intron_min_length
output_ranges = [
x for x in output_ranges if x[1] - x[0] > l
]
for start, end in output_ranges:
entry = GTF.Entry()
entry.copy(gffs[0])
entry.clearAttributes()
entry.transcript_id = "merged"
entry.feature = "intron"
entry.start = start
entry.end = end
options.stdout.write("%s\n" % str(entry))
nfeatures += 1
noutput += 1
else:
ndiscarded += 1
elif options.set_score2distance:
for gffs in GTF.transcript_iterator(GTF.iterator(options.stdin)):
ninput += 1
strand = Genomics.convertStrand(gffs[0].strand)
all_start, all_end = min([x.start for x in gffs
]), max([x.end for x in gffs])
if strand != ".":
t = 0
if strand == "-":
gffs.reverse()
for gff in gffs:
gff.score = t
t += gff.end - gff.start
if strand == "-":
gffs.reverse()
for gff in gffs:
options.stdout.write("%s\n" % str(gff))
nfeatures += 1
noutput += 1
elif options.remove_overlapping:
index = GTF.readAndIndex(
GTF.iterator(IOTools.openFile(options.remove_overlapping, "r")))
for gffs in GTF.transcript_iterator(GTF.iterator(options.stdin)):
ninput += 1
found = False
for e in gffs:
if index.contains(e.contig, e.start, e.end):
found = True
break
if found:
ndiscarded += 1
else:
noutput += 1
for e in gffs:
nfeatures += 1
options.stdout.write("%s\n" % str(e))
elif options.intersect_transcripts:
for gffs in GTF.gene_iterator(GTF.iterator(options.stdin),
strict=options.strict):
ninput += 1
r = []
for g in gffs:
if options.with_utr:
ranges = GTF.asRanges(g, "exon")
else:
ranges = GTF.asRanges(g, "CDS")
r.append(ranges)
result = r[0]
for x in r[1:]:
result = Intervals.intersect(result, x)
entry = GTF.Entry()
entry.copy(gffs[0][0])
entry.clearAttributes()
entry.transcript_id = "merged"
entry.feature = "exon"
for start, end in result:
entry.start = start
entry.end = end
options.stdout.write("%s\n" % str(entry))
nfeatures += 1
noutput += 1
elif options.rename_duplicates:
gene_ids = list()
transcript_ids = list()
gtfs = list()
for gtf in GTF.iterator(options.stdin):
gtfs.append(gtf)
if gtf.feature == "CDS":
gene_ids.append(gtf.gene_id)
transcript_ids.append(gtf.transcript_id)
dup_gene = [item for item in set(gene_ids) if gene_ids.count(item) > 1]
dup_transcript = [
item for item in set(transcript_ids)
if transcript_ids.count(item) > 1
]
E.info("Number of duplicated gene_ids: %i" % len(dup_gene))
E.info("Number of duplicated transcript_ids: %i" % len(dup_transcript))
gene_dict = dict(zip(dup_gene, ([0] * len(dup_gene))))
transcript_dict = dict(zip(dup_transcript,
([0] * len(dup_transcript))))
for gtf in gtfs:
if gtf.feature == "CDS":
if gtf.gene_id in dup_gene:
gene_dict[gtf.gene_id] = gene_dict[gtf.gene_id] + 1
gtf.setAttribute(
'gene_id',
gtf.gene_id + "." + str(gene_dict[gtf.gene_id]))
if gtf.transcript_id in dup_transcript:
transcript_dict[gtf.transcript_id] = \
transcript_dict[gtf.transcript_id] + 1
gtf.setAttribute(
'transcript_id', gtf.transcript_id + "." +
str(transcript_dict[gtf.transcript_id]))
options.stdout.write("%s\n" % gtf)
else:
for gffs in GTF.flat_gene_iterator(GTF.iterator(options.stdin),
strict=options.strict):
ninput += 1
cds_ranges = GTF.asRanges(gffs, "CDS")
exon_ranges = GTF.asRanges(gffs, "exon")
# sanity checks
strands = set([x.strand for x in gffs])
contigs = set([x.contig for x in gffs])
if len(strands) > 1:
raise ValueError(
"can not merge gene '%s' on multiple strands: %s" %
(gffs[0].gene_id, str(strands)))
if len(contigs) > 1:
raise ValueError(
"can not merge gene '%s' on multiple contigs: %s" %
(gffs[0].gene_id, str(contigs)))
strand = Genomics.convertStrand(gffs[0].strand)
if cds_ranges and options.with_utr:
cds_start, cds_end = cds_ranges[0][0], cds_ranges[-1][1]
midpoint = (cds_end - cds_start) / 2 + cds_start
utr_ranges = []
for start, end in Intervals.truncate(exon_ranges, cds_ranges):
if end - start > 3:
if strand == ".":
feature = "UTR"
elif strand == "+":
if start < midpoint:
feature = "UTR5"
else:
feature = "UTR3"
elif strand == "-":
if start < midpoint:
feature = "UTR3"
else:
feature = "UTR5"
utr_ranges.append((feature, start, end))
output_feature = "CDS"
output_ranges = cds_ranges
else:
output_feature = "exon"
output_ranges = exon_ranges
utr_ranges = []
result = []
if options.merge_exons:
# need to combine per feature - skip
# utr_ranges = Intervals.combineAtDistance(
# utr_ranges,
# options.merge_exons_distance)
output_ranges = Intervals.combineAtDistance(
output_ranges, options.merge_exons_distance)
for feature, start, end in utr_ranges:
entry = GTF.Entry()
entry.copy(gffs[0])
entry.clearAttributes()
entry.feature = feature
entry.transcript_id = "merged"
entry.start = start
entry.end = end
result.append(entry)
for start, end in output_ranges:
entry = GTF.Entry()
entry.copy(gffs[0])
entry.clearAttributes()
entry.transcript_id = "merged"
entry.feature = output_feature
entry.start = start
entry.end = end
result.append(entry)
elif options.merge_transcripts:
entry = GTF.Entry()
entry.copy(gffs[0])
entry.clearAttributes()
entry.transcript_id = entry.gene_id
entry.start = output_ranges[0][0]
entry.end = output_ranges[-1][1]
result.append(entry)
elif options.merge_introns:
if len(output_ranges) >= 2:
entry = GTF.Entry()
entry.copy(gffs[0])
entry.clearAttributes()
entry.transcript_id = entry.gene_id
entry.start = output_ranges[0][1]
entry.end = output_ranges[-1][0]
result.append(entry)
else:
ndiscarded += 1
continue
result.sort(key=lambda x: x.start)
for x in result:
options.stdout.write("%s\n" % str(x))
nfeatures += 1
noutput += 1
E.info("ninput=%i, noutput=%i, nfeatures=%i, ndiscarded=%i" %
(ninput, noutput, nfeatures, ndiscarded))
E.Stop()
