def intersectBedFiles(infiles, outfile):
'''merge :term:`bed` formatted *infiles* by intersection
and write to *outfile*.
Only intervals that overlap in all files are retained.
Interval coordinates are given by the first file in *infiles*.
Bed files are normalized (overlapping intervals within
a file are merged) before intersection.
Intervals are renumbered starting from 1.
'''
if len(infiles) == 1:
shutil.copyfile(infiles[0], outfile)
elif len(infiles) == 2:
if P.isEmpty(infiles[0]) or P.isEmpty(infiles[1]):
P.touch(outfile)
else:
statement = '''
intersectBed -u -a %s -b %s
| cut -f 1,2,3,4,5
| awk 'BEGIN { OFS="\\t"; } {$4=++a; print;}'
| bgzip > %%(outfile)s
''' % (infiles[0], infiles[1])
P.run()
else:
tmpfile = P.getTempFilename(".")
# need to merge incrementally
fn = infiles[0]
if P.isEmpty(infiles[0]):
P.touch(outfile)
return
statement = '''mergeBed -i %(fn)s > %(tmpfile)s'''
P.run()
for fn in infiles[1:]:
if P.isEmpty(infiles[0]):
P.touch(outfile)
os.unlink(tmpfile)
return
statement = '''mergeBed -i %(fn)s | intersectBed -u -a %(tmpfile)s -b stdin > %(tmpfile)s.tmp; mv %(tmpfile)s.tmp %(tmpfile)s'''
P.run()
statement = '''cat %(tmpfile)s
| cut -f 1,2,3,4,5
| awk 'BEGIN { OFS="\\t"; } {$4=++a; print;}'
| bgzip
> %(outfile)s '''
P.run()
os.unlink(tmpfile)
def subtractBedFiles(infile, subtractfile, outfile):
'''subtract intervals in *subtractfile* from *infile*
and store in *outfile*.
'''
if P.isEmpty(subtractfile):
shutil.copyfile(infile, outfile)
return
elif P.isEmpty(infile):
P.touch(outfile)
return
statement = '''
intersectBed -v -a %(infile)s -b %(subtractfile)s
| cut -f 1,2,3,4,5
| awk 'BEGIN { OFS="\\t"; } {$4=++a; print;}'
| bgzip > %(outfile)s ; tabix -p bed %(outfile)s
'''
P.run()
def BedFileVenn(infiles, outfile):
'''merge :term:`bed` formatted *infiles* by intersection
and write to *outfile*.
Only intervals that overlap in all files are retained.
Interval coordinates are given by the first file in *infiles*.
Bed files are normalized (overlapping intervals within
a file are merged) before intersection.
Intervals are renumbered starting from 1.
'''
bed1, bed2 = infiles
liver_name = P.snip(os.path.basename(liver), ".replicated.bed")
testes_name = P.snip(os.path.basename(testes), ".replicated.bed")
to_cluster = True
statement = '''cat %(liver)s %(testes)s | mergeBed -i stdin | awk 'OFS="\\t" {print $1,$2,$3,"CAPseq"NR}' > replicated_intervals/liver.testes.merge.bed;
echo "Total merged intervals" > %(outfile)s; cat replicated_intervals/liver.testes.merge.bed | wc -l >> %(outfile)s;
echo "Liver & testes" >> %(outfile)s; intersectBed -a replicated_intervals/liver.testes.merge.bed -b %(liver)s -u | intersectBed -a stdin -b %(testes)s -u > replicated_intervals/liver.testes.shared.bed; cat replicated_intervals/liver.testes.shared.bed | wc -l >> %(outfile)s;
echo "Testes only" >> %(outfile)s; intersectBed -a replicated_intervals/liver.testes.merge.bed -b %(liver)s -v > replicated_intervals/%(testes_name)s.liver.testes.unique.bed; cat replicated_intervals/%(testes_name)s.liver.testes.unique.bed | wc -l >> %(outfile)s;
echo "Liver only" >> %(outfile)s; intersectBed -a replicated_intervals/liver.testes.merge.bed -b %(testes)s -v > replicated_intervals/%(liver_name)s.liver.testes.unique.bed; cat replicated_intervals/%(liver_name)s.liver.testes.unique.bed | wc -l >> %(outfile)s;
sed -i '{N;s/\\n/\\t/g}' %(outfile)s; '''
if len(infiles) == 1:
shutil.copyfile(infiles[0], outfile)
elif len(infiles) == 2:
if P.isEmpty(infiles[0]) or P.isEmpty(infiles[1]):
P.touch(outfile)
else:
statement = '''
intersectBed -u -a %s -b %s
| cut -f 1,2,3,4,5
| awk 'BEGIN { OFS="\\t"; } {$4=++a; print;}'
> %%(outfile)s
''' % (infiles[0], infiles[1])
P.run()
else:
tmpfile = P.getTempFilename(".")
# need to merge incrementally
fn = infiles[0]
if P.isEmpty(infiles[0]):
P.touch(outfile)
return
statement = '''mergeBed -i %(fn)s > %(tmpfile)s'''
P.run()
for fn in infiles[1:]:
if P.isEmpty(infiles[0]):
P.touch(outfile)
os.unlink(tmpfile)
return
statement = '''mergeBed -i %(fn)s | intersectBed -u -a %(tmpfile)s -b stdin > %(tmpfile)s.tmp; mv %(tmpfile)s.tmp %(tmpfile)s'''
P.run()
statement = '''cat %(tmpfile)s
| cut -f 1,2,3,4,5
| awk 'BEGIN { OFS="\\t"; } {$4=++a; print;}'
> %(outfile)s '''
P.run()
os.unlink(tmpfile)
def buildPseudogenes(infiles, outfile, dbhandle):
'''build a set of pseudogenes.
*infiles* is an ENSEMBL gtf file and a list of associated
peptide sequences.
Transcripts are extracted from the GTF file and designated
as pseudogenes if:
* the gene_type or transcript_type contains the phrase
"pseudo". This taken is from the database.
* the feature is 'processed_transcript' and has similarity to
protein coding genes. Similarity is assessed by aligning the transcript
and peptide set against each other with exonerate.
Pseudogenic transcripts can overlap with protein coding
transcripts.
'''
infile_gtf, infile_peptides_fasta = infiles
# JJ - there are also 'nontranslated_CDS', but no explanation of these
if PARAMS["genome"].startswith("dm"):
E.warn("Ensembl dm genome annotations only contain source"
" 'pseudogenes' - skipping exonerate step")
statement = """zcat %(infile_gtf)s
|awk '$2 ~ /pseudogene/'
| gzip
> %(outfile)s"""
P.run()
return
tmpfile1 = P.getTempFilename(shared=True)
# collect processed transcripts and save as fasta sequences
statement = '''
zcat %(infile_gtf)s
| awk '$2 ~ /processed/'
| python %(scriptsdir)s/gff2fasta.py
--is-gtf
--genome-file=%(genome_dir)s/%(genome)s
--log=%(outfile)s.log
> %(tmpfile1)s
'''
P.run()
if P.isEmpty(tmpfile1):
E.warn("no pseudogenes found")
os.unlink(tmpfile1)
P.touch(outfile)
return
model = "protein2dna"
# map processed transcripts against peptide sequences
statement = '''
cat %(tmpfile1)s
| %(cmd-farm)s --split-at-regex=\"^>(\S+)\" --chunk-size=100
--log=%(outfile)s.log
"exonerate --target %%STDIN%%
--query %(infile_peptides_fasta)s
--model %(model)s
--bestn 1
--score 200
--ryo \\"%%qi\\\\t%%ti\\\\t%%s\\\\n\\"
--showalignment no --showsugar no --showcigar no --showvulgar no
"
| grep -v -e "exonerate" -e "Hostname"
| gzip > %(outfile)s.links.gz
'''
P.run()
os.unlink(tmpfile1)
inf = IOTools.openFile("%s.links.gz" % outfile)
best_matches = {}
for line in inf:
peptide_id, transcript_id, score = line[:-1].split("\t")
score = int(score)
if transcript_id in best_matches and \
best_matches[transcript_id][0] > score:
continue
best_matches[transcript_id] = (score, peptide_id)
inf.close()
E.info("found %i best links" % len(best_matches))
new_pseudos = set(best_matches.keys())
cc = dbhandle.cursor()
known_pseudos = set([x[0] for x in cc.execute(
"""SELECT DISTINCT transcript_id
FROM transcript_info
WHERE transcript_biotype like '%pseudo%' OR
gene_biotype like '%pseudo%' """)])
E.info("pseudogenes from: processed_transcripts=%i, known_pseudos=%i, "
"intersection=%i" % (
(len(new_pseudos),
len(known_pseudos),
len(new_pseudos.intersection(known_pseudos)))))
all_pseudos = new_pseudos.union(known_pseudos)
c = E.Counter()
outf = IOTools.openFile(outfile, "w")
inf = GTF.iterator(IOTools.openFile(infile_gtf))
for gtf in inf:
c.input += 1
if gtf.transcript_id not in all_pseudos:
continue
c.output += 1
outf.write("%s\n" % gtf)
outf.close()
E.info("exons: %s" % str(c))
def buildNUMTs(infile, outfile):
'''build annotation with nuclear mitochondrial sequences.
map mitochondrial chromosome against genome using
exonerate
'''
if not PARAMS["numts_mitochrom"]:
E.info("skipping numts creation")
P.touch(outfile)
return
fasta = IndexedFasta.IndexedFasta(
os.path.join(PARAMS["genome_dir"], PARAMS["genome"]))
if PARAMS["numts_mitochrom"] not in fasta:
E.warn("mitochondrial genome %s not found" % PARAMS["numts_mitochrom"])
P.touch(outfile)
return
tmpfile_mito = P.getTempFilename(".")
statement = '''
python %(scriptsdir)s/index_fasta.py
--extract=%(numts_mitochrom)s
--log=%(outfile)s.log
%(genome_dir)s/%(genome)s
> %(tmpfile_mito)s
'''
P.run()
if P.isEmpty(tmpfile_mito):
E.warn("mitochondrial genome empty.")
os.unlink(tmpfile_mito)
P.touch(outfile)
return
format = ("qi", "qS", "qab", "qae",
"ti", "tS", "tab", "tae",
"s",
"pi",
"C")
format = "\\\\t".join(["%%%s" % x for x in format])
# collect all results
min_score = 100
statement = '''
cat %(genome_dir)s/%(genome)s.fasta
| %(cmd-farm)s --split-at-regex=\"^>(\S+)\" --chunk-size=1
--log=%(outfile)s.log
"exonerate --target %%STDIN%%
--query %(tmpfile_mito)s
--model affine:local
--score %(min_score)i
--showalignment no --showsugar no --showcigar no
--showvulgar no
--ryo \\"%(format)s\\n\\"
"
| grep -v -e "exonerate" -e "Hostname"
| gzip > %(outfile)s.links.gz
'''
P.run()
# convert to gtf
inf = IOTools.openFile("%s.links.gz" % outfile)
outf = IOTools.openFile(outfile, "w")
min_score = PARAMS["numts_score"]
c = E.Counter()
for line in inf:
(query_contig, query_strand, query_start, query_end,
target_contig, target_strand, target_start, target_end,
score, pid, alignment) = line[:-1].split("\t")
c.input += 1
score = int(score)
if score < min_score:
c.skipped += 1
continue
if target_strand == "-":
target_start, target_end = target_end, target_start
gff = GTF.Entry()
gff.contig = target_contig
gff.start, gff.end = int(target_start), int(target_end)
assert gff.start < gff.end
gff.strand = target_strand
gff.score = int(score)
gff.feature = "numts"
gff.gene_id = "%s:%s-%s" % (query_contig, query_start, query_end)
gff.transcript_id = "%s:%s-%s" % (query_contig, query_start, query_end)
outf.write("%s\n" % str(gff))
c.output += 1
inf.close()
outf.close()
E.info("filtering numts: %s" % str(c))
os.unlink(tmpfile_mito)
def loadZinba(infile, outfile, bamfile,
tablename=None,
controlfile=None):
'''load Zinba results in *tablename*
This method loads only positive peaks. It filters peaks by p-value,
q-value and fold change and loads the diagnostic data and
re-calculates peakcenter, peakval, ... using the supplied bamfile.
If *tablename* is not given, it will be :file:`<track>_intervals`
where track is derived from ``infile`` and assumed to end
in :file:`.zinba`.
If no peaks were predicted, an empty table is created.
This method creates :file:`<outfile>.tsv.gz` with the results
of the filtering.
This method uses the refined peak locations.
Zinba peaks can be overlapping. This method does not merge
overlapping intervals.
Zinba calls peaks in regions where there are many reads inside
the control. Thus this method applies a filtering step
removing all intervals in which there is a peak of
more than readlength / 2 height in the control.
.. note:
Zinba calls peaks that are overlapping.
'''
track = P.snip(os.path.basename(infile), ".zinba")
folder = os.path.dirname(infile)
infilename = infile + ".peaks"
outtemp = P.getTempFile(".")
tmpfilename = outtemp.name
outtemp.write("\t".join((
"interval_id",
"contig", "start", "end",
"npeaks", "peakcenter",
"length",
"avgval",
"peakval",
"nprobes",
"pvalue", "fold", "qvalue",
"macs_summit", "macs_nprobes",
)) + "\n")
counter = E.Counter()
if not os.path.exists(infilename):
E.warn("could not find %s" % infilename)
elif P.isEmpty(infile):
E.warn("no data in %s" % filename)
else:
# filter peaks
shift = getPeakShiftFromZinba(infile)
assert shift is not None, "could not determine peak shift from Zinba file %s" % infile
E.info("%s: found peak shift of %i" % (track, shift))
samfiles = [pysam.Samfile(bamfile, "rb")]
offsets = [shift / 2]
if controlfile:
controlfiles = [pysam.Samfile(controlfile, "rb")]
readlength = PipelineMapping.getReadLengthFromBamfile(controlfile)
control_max_peakval = readlength // 2
E.info("removing intervals in which control has peak higher than %i reads" %
control_max_peakval)
else:
controlfiles = None
id = 0
# get thresholds
max_qvalue = float(PARAMS["zinba_fdr_threshold"])
with IOTools.openFile(infilename, "r") as ins:
for peak in WrapperZinba.iteratePeaks(ins):
# filter by qvalue
if peak.fdr > max_qvalue:
counter.removed_qvalue += 1
continue
assert peak.refined_start < peak.refined_end
# filter by control
if controlfiles:
npeaks, peakcenter, length, avgval, peakval, nreads = countPeaks(peak.contig,
peak.refined_start,
peak.refined_end,
controlfiles,
offsets)
if peakval > control_max_peakval:
counter.removed_control += 1
continue
# output peak
npeaks, peakcenter, length, avgval, peakval, nreads = countPeaks(peak.contig,
peak.refined_start,
peak.refined_end,
samfiles,
offsets)
outtemp.write("\t".join(map(str, (
id, peak.contig, peak.refined_start, peak.refined_end,
npeaks, peakcenter, length, avgval, peakval, nreads,
1.0 - peak.posterior, 1.0, peak.fdr,
peak.refined_start + peak.summit - 1,
peak.height))) + "\n")
id += 1
counter.output += 1
outtemp.close()
# output filtering summary
outf = IOTools.openFile("%s.tsv.gz" % outfile, "w")
outf.write("category\tcounts\n")
outf.write("%s\n" % counter.asTable())
outf.close()
E.info("%s filtering: %s" % (track, str(counter)))
if counter.output == 0:
E.warn("%s: no peaks found" % track)
# load data into table
if tablename is None:
tablename = "%s_intervals" % track
statement = '''
python %(scriptsdir)s/csv2db.py %(csv2db_options)s
--allow-empty-file
--add-index=interval_id
--add-index=contig,start
--table=%(tablename)s
< %(tmpfilename)s
> %(outfile)s
'''
P.run()
os.unlink(tmpfilename)
