def runTsdFinder(faPath, inBedPath, outBedPath, args):
""" call tsdFinder and either overwrite or append output. also call
removeBedOverlaps on final output to make sure it is clean """
# convert input to bed if necessary
tempBed = None
if os.path.splitext(inBedPath)[1].lower() == ".bb":
tempBed = getLocalTempPath("Temp_addTsdTrack", ".bed")
runShellCommand("bigBedToBed %s %s" % (inFile, tempBed))
inBedPath = tempBed
# run tsdfinder on input
tempOut = getLocalTempPath("Temp_addTsdTrack", ".bed")
runShellCommand("tsdFinder.py %s %s %s %s" % (faPath, inBedPath,
tempOut,
args.tsdFinderOptions))
if tempBed is not None:
runShellCommand("rm %s" % tempBed)
# merge with existing track
if os.path.isfile(outBedPath) and args.append is True:
runShellCommand("cat %s >> %s" % (outBedPath, tempOut))
# remove overlaps into final output
runShellCommand("removeBedOverlaps.py %s > %s" % (tempOut, outBedPath))
runShellCommand("rm %s" % tempOut)
def runCleaning(args, tempTracksInfo):
""" run scripts for cleaning chaux, ltr_finder, and termini"""
trackList = TrackList(args.tracksInfo)
for track in trackList:
if track.getPreprocess() is None:
continue
# convert bigbed/wig
inFile = track.getPath()
tempBed1 = None
if inFile[-3:] == ".bb" or inFile[-3:] == ".bw":
tempBed1 = getLocalTempPath("Temp_%s" % track.getName(), ".bed")
if inFile[-3:] == ".bb":
runShellCommand("bigBedToBed %s %s" % (inFile, tempBed1))
else:
runShellCommand("bigWigToBedGraph %s %s" % (inFile, tempBed1))
inFile = tempBed1
# run cleanRM.py on all tracks with rm or rmu preprocessor
if track.getPreprocess() == "rm" or track.getPreprocess() == "rmu":
flag = ""
if track.getPreprocess() == "rmu":
flag == "--keepUnderscore"
inFile = track.getPath()
outFile = cleanPath(args, track)
tempBed = getLocalTempPath("Temp_%s" % track.getName(), ".bed")
runShellCommand("cleanRM.py %s %s > %s" % (inFile, flag, tempBed))
runShellCommand("removeBedOverlaps.py --rm %s > %s" %
(tempBed, outFile))
runShellCommand("rm -f %s" % tempBed)
track.setPath(outFile)
# run cleanTermini.py
elif track.getPreprocess() == "termini":
outFile = cleanPath(args, track)
inFile = track.getPath()
runShellCommand("cleanTermini.py %s %s" % (inFile, outFile))
track.setPath(outFile)
# run removeBedOverlaps
elif track.getPreprocess() == "overlap":
outFile = cleanPath(args, track)
inFile = track.getPath()
runShellCommand("removeBedOverlaps.py %s > %s" % (inFile, outFile))
track.setPath(outFile)
# run cleanLtrFinder.py
elif track.getPreprocess() == "ltr_finder":
inFile = track.getPath()
outFile = cleanPath(args, track)
# note: overlaps now removed in cleanLtrFinderID script
runShellCommand("cleanLtrFinderID.py %s %s" % (inFile, outFile))
track.setPath(outFile)
if tempBed1 is not None:
runShellCommand("rm -f %s" % tempBed1)
# save a temporary xml
trackList.saveXML(tempTracksInfo)
def runParallel(args, bedIntervals):
""" Quick hack to rerun parallel jobs on different interval subsets. """
nameSet = None
if args.names is not None:
nameSet = set(args.names.split(","))
# chunk up BED input
numIntervals = 0
for interval in bedIntervals:
name = None
if len(interval) > 3:
name = interval[3]
if nameSet is None or name in nameSet:
numIntervals += 1
jobSize = 1 + (numIntervals / args.numProc)
logger.info("Dviding %d intervals into %d processes (%d intervals per)" %
(numIntervals, args.numProc, jobSize))
tempBeds = []
curSize = sys.maxint
curFile = None
for interval in bedIntervals:
name = None
if len(interval) > 3:
name = interval[3]
if nameSet is None or name in nameSet:
if curSize >= jobSize:
if curFile is not None:
curFile.close()
tempBed = getLocalTempPath("TempTsdFinderIn", ".bed")
tempBeds.append(tempBed)
curFile = open(tempBed, "w")
curSize = 0
curFile.write("\t".join([str(s) for s in interval]))
curFile.write("\n")
curSize += 1
if curFile is not None:
curFile.close()
# map jobs
assert len(tempBeds) <= args.numProc
tempOuts = []
jobCmds = []
for tempBed in tempBeds:
cmdLine = " ".join(sys.argv)
cmdLine = cmdLine.replace("--numProc %d" % args.numProc, "--numProc 1")
cmdLine = cmdLine.replace(args.inBed, tempBed)
tempOut = getLocalTempPath("TempTsdFinderOut", ".bed")
cmdLine = cmdLine.replace(args.outBed, tempOut)
tempOuts.append(tempOut)
jobCmds.append(cmdLine)
runParallelShellCommands(jobCmds, args.numProc)
# reduce
for i, tempOut in enumerate(tempOuts):
if i == 0:
runShellCommand("mv %s %s" % (tempOut, args.outBed))
else:
runShellCommand("cat %s >> %s" % (tempOut, args.outBed))
runShellCommand("rm -f %s" % (tempOut))
def cutOutMaskIntervals(inBed, minLength, maxLength, tracksInfoPath):
""" Filter out intervals of mask tracks from inBed with lengths
outside given range. Idea is that it makes less sense to simply ignore,
say, giant stretches of N's (like centromeres), as we would by masking
them normally, than it does to remove them entirely, splitting the
genome into multiple chunks. Can also be used during comparision to
get rid of all masked intervals """
outPath = getLocalTempPath("Tempcut", ".bed")
trackList = TrackList(tracksInfoPath)
maskPaths = [t.getPath() for t in trackList.getMaskTracks()]
if len(maskPaths) == 0:
return None
tempPath1 = getLocalTempPath("Tempcut1", ".bed")
tempPath2 = getLocalTempPath("Tempcut2", ".bed")
runShellCommand("cp %s %s" % (inBed, outPath))
for maskPath in maskPaths:
runShellCommand("cat %s | awk \'{print $1\"\t\"$2\"\t\"$3}\' >> %s" %
(maskPath, tempPath1))
if os.path.getsize(tempPath1) > 0:
runShellCommand("sortBed -i %s > %s ; mergeBed -i %s > %s" %
(tempPath1, tempPath2, tempPath2, tempPath1))
runShellCommand("filterBedLengths.py %s %d %d > %s" %
(tempPath1, minLength + 1, maxLength - 1, tempPath2))
runShellCommand("subtractBed -a %s -b %s | sortBed > %s" %
(outPath, tempPath2, tempPath1))
runShellCommand("mv %s %s" % (tempPath1, outPath))
runShellCommand("rm -f %s %s" % (tempPath1, tempPath2))
if os.path.getsize(outPath) == 0:
raise RuntimeError(
"cutOutMaskIntervals removed everything. Can't continue."
" probably best to rerun calling script on bigger region?")
return outPath
def filterCutTrack(genomePath, fragmentFilterLen, trackListPath, cutTrackName,
cutTrackLenFilter):
""" return path of length filtered cut track"""
tracks = TrackList(trackListPath)
track = tracks.getTrackByName(cutTrackName)
assert track is not None
cutTrackOriginalPath = track.getPath()
cutTrackPath = getOutPath(cutTrackOriginalPath, outDir,
"filter%d" % cutTrackLenFilter)
runShellCommand("filterBedLengths.py %s %s > %s" %
(cutTrackOriginalPath, cutTrackLenFilter, cutTrackPath))
tempPath1 = getLocalTempPath("Temp", ".bed")
runShellCommand("subtractBed -a %s -b %s | sortBed > %s" %
(genomePath, cutTrackPath, tempPath1))
tempPath2 = getLocalTempPath("Temp", ".bed")
S = string.ascii_uppercase + string.digits
tag = ''.join(random.choice(S) for x in range(200))
runShellCommand(
"filterBedLengths.py %s %d --rename %s |grep %s | sortBed> %s" %
(tempPath1, fragmentFilterLen, tag, tag, tempPath2))
runShellCommand(
"cat %s | setBedCol.py 3 N | setBedCol.py 4 0 | setBedCol.py 5 . > %s"
% (tempPath2, tempPath1))
runShellCommand(
"cat %s | setBedCol.py 3 N | setBedCol.py 4 0 | setBedCol.py 5 . >> %s"
% (cutTrackPath, tempPath1))
runShellCommand("sortBed -i %s > %s" % (tempPath1, tempPath2))
runShellCommand("mergeBed -i %s > %s" % (tempPath2, cutTrackPath))
runShellCommand("rm -f %s %s" % (tempPath1, tempPath2))
return cutTrackPath
def cutOutMaskIntervals(inBed, minLength, maxLength, tracksInfoPath):
""" Filter out intervals of mask tracks from inBed with lengths
outside given range. Idea is that it makes less sense to simply ignore,
say, giant stretches of N's (like centromeres), as we would by masking
them normally, than it does to remove them entirely, splitting the
genome into multiple chunks. Can also be used during comparision to
get rid of all masked intervals """
outPath = getLocalTempPath("Tempcut", ".bed")
trackList = TrackList(tracksInfoPath)
maskPaths = [t.getPath() for t in trackList.getMaskTracks()]
if len(maskPaths) == 0:
return None
tempPath1 = getLocalTempPath("Tempcut1", ".bed")
tempPath2 = getLocalTempPath("Tempcut2", ".bed")
runShellCommand("cp %s %s" % (inBed, outPath))
for maskPath in maskPaths:
runShellCommand("cat %s | awk \'{print $1\"\t\"$2\"\t\"$3}\' >> %s" % (
maskPath, tempPath1))
if os.path.getsize(tempPath1) > 0:
runShellCommand("sortBed -i %s > %s ; mergeBed -i %s > %s" % (
tempPath1, tempPath2, tempPath2, tempPath1))
runShellCommand("filterBedLengths.py %s %d %d > %s" % (
tempPath1, minLength+1, maxLength-1, tempPath2))
runShellCommand("subtractBed -a %s -b %s | sortBed > %s" % (
outPath, tempPath2, tempPath1))
runShellCommand("mv %s %s" % (tempPath1, outPath))
runShellCommand("rm -f %s %s" % (tempPath1, tempPath2))
if os.path.getsize(outPath) == 0:
raise RuntimeError("cutOutMaskIntervals removed everything. Can't continue."
" probably best to rerun calling script on bigger region?")
return outPath
def filterCutTrack(genomePath, fragmentFilterLen, trackListPath, cutTrackName,
cutTrackLenFilter):
""" return path of length filtered cut track"""
tracks = TrackList(trackListPath)
track = tracks.getTrackByName(cutTrackName)
assert track is not None
cutTrackOriginalPath = track.getPath()
cutTrackPath = getOutPath(cutTrackOriginalPath, outDir,
"filter%d" % cutTrackLenFilter)
runShellCommand("filterBedLengths.py %s %s > %s" % (cutTrackOriginalPath,
cutTrackLenFilter,
cutTrackPath))
tempPath1 = getLocalTempPath("Temp", ".bed")
runShellCommand("subtractBed -a %s -b %s | sortBed > %s" % (genomePath,
cutTrackPath,
tempPath1))
tempPath2 = getLocalTempPath("Temp", ".bed")
S = string.ascii_uppercase + string.digits
tag = ''.join(random.choice(S) for x in range(200))
runShellCommand("filterBedLengths.py %s %d --rename %s |grep %s | sortBed> %s" % (
tempPath1, fragmentFilterLen, tag, tag, tempPath2))
runShellCommand("cat %s | setBedCol.py 3 N | setBedCol.py 4 0 | setBedCol.py 5 . > %s" % (tempPath2, tempPath1))
runShellCommand("cat %s | setBedCol.py 3 N | setBedCol.py 4 0 | setBedCol.py 5 . >> %s" % (cutTrackPath, tempPath1))
runShellCommand("sortBed -i %s > %s" % (tempPath1, tempPath2))
runShellCommand("mergeBed -i %s > %s" %(tempPath2, cutTrackPath))
runShellCommand("rm -f %s %s" % (tempPath1, tempPath2))
return cutTrackPath
def runParallel(args, bedIntervals):
""" Quick hack to rerun parallel jobs on different interval subsets. """
nameSet = None
if args.names is not None:
nameSet = set(args.names.split(","))
# chunk up BED input
numIntervals = 0
for interval in bedIntervals:
name = None
if len(interval) > 3:
name = interval[3]
if nameSet is None or name in nameSet:
numIntervals += 1
jobSize = 1 + (numIntervals / args.numProc)
logger.info("Dviding %d intervals into %d processes (%d intervals per)" % (
numIntervals, args.numProc, jobSize))
tempBeds = []
curSize = sys.maxint
curFile = None
for interval in bedIntervals:
name = None
if len(interval) > 3:
name = interval[3]
if nameSet is None or name in nameSet:
if curSize >= jobSize:
if curFile is not None:
curFile.close()
tempBed = getLocalTempPath("TempTsdFinderIn", ".bed")
tempBeds.append(tempBed)
curFile = open(tempBed, "w")
curSize = 0
curFile.write("\t".join([str(s) for s in interval]))
curFile.write("\n")
curSize += 1
if curFile is not None:
curFile.close()
# map jobs
assert len(tempBeds) <= args.numProc
tempOuts = []
jobCmds = []
for tempBed in tempBeds:
cmdLine = " ".join(sys.argv)
cmdLine = cmdLine.replace("--numProc %d" % args.numProc,"--numProc 1")
cmdLine = cmdLine.replace(args.inBed, tempBed)
tempOut = getLocalTempPath("TempTsdFinderOut", ".bed")
cmdLine = cmdLine.replace(args.outBed, tempOut)
tempOuts.append(tempOut)
jobCmds.append(cmdLine)
runParallelShellCommands(jobCmds, args.numProc)
# reduce
for i, tempOut in enumerate(tempOuts):
if i == 0:
runShellCommand("mv %s %s" % (tempOut, args.outBed))
else:
runShellCommand("cat %s >> %s" % (tempOut, args.outBed))
runShellCommand("rm -f %s" % (tempOut))
def runTsdFinder(faPath, inBedPath, outBedPath, args):
""" call tsdFinder and either overwrite or append output. also call
removeBedOverlaps on final output to make sure it is clean """
# convert input to bed if necessary
tempBed = None
if os.path.splitext(inBedPath)[1].lower() == ".bb":
tempBed = getLocalTempPath("Temp_addTsdTrack", ".bed")
runShellCommand("bigBedToBed %s %s" % (inFile, tempBed))
inBedPath = tempBed
# run tsdfinder on input
tempOut = getLocalTempPath("Temp_addTsdTrack", ".bed")
runShellCommand("tsdFinder.py %s %s %s %s" %
(faPath, inBedPath, tempOut, args.tsdFinderOptions))
if tempBed is not None:
runShellCommand("rm %s" % tempBed)
# merge with existing track
if os.path.isfile(outBedPath) and args.append is True:
runShellCommand("cat %s >> %s" % (outBedPath, tempOut))
# remove overlaps into final output
runShellCommand("removeBedOverlaps.py %s > %s" % (tempOut, outBedPath))
runShellCommand("rm %s" % tempOut)
def runCleaning(args, tempTracksInfo):
""" run scripts for cleaning chaux, ltr_finder, and termini"""
trackList = TrackList(args.tracksInfo)
for track in trackList:
if track.getPreprocess() is None:
continue
# convert bigbed/wig
inFile = track.getPath()
tempBed1 = None
if inFile[-3:] == ".bb" or inFile[-3:] == ".bw":
tempBed1 = getLocalTempPath("Temp_%s" % track.getName(), ".bed")
if inFile[-3:] == ".bb":
runShellCommand("bigBedToBed %s %s" % (inFile, tempBed1))
else:
runShellCommand("bigWigToBedGraph %s %s" % (inFile, tempBed1))
inFile = tempBed1
# run cleanRM.py on all tracks with rm or rmu preprocessor
if track.getPreprocess() == "rm" or track.getPreprocess() == "rmu":
flag = ""
if track.getPreprocess() == "rmu":
flag == "--keepUnderscore"
inFile = track.getPath()
outFile = cleanPath(args, track)
tempBed = getLocalTempPath("Temp_%s" % track.getName(), ".bed")
runShellCommand("cleanRM.py %s %s > %s" % (inFile, flag, tempBed))
runShellCommand("removeBedOverlaps.py --rm %s > %s" % (tempBed, outFile))
runShellCommand("rm -f %s" % tempBed)
track.setPath(outFile)
# run cleanTermini.py
elif track.getPreprocess() == "termini":
outFile = cleanPath(args, track)
inFile = track.getPath()
runShellCommand("cleanTermini.py %s %s" % (inFile, outFile))
track.setPath(outFile)
# run removeBedOverlaps
elif track.getPreprocess() == "overlap":
outFile = cleanPath(args, track)
inFile = track.getPath()
runShellCommand("removeBedOverlaps.py %s > %s" % (inFile, outFile))
track.setPath(outFile)
# run cleanLtrFinder.py
elif track.getPreprocess() == "ltr_finder":
inFile = track.getPath()
outFile = cleanPath(args, track)
# note: overlaps now removed in cleanLtrFinderID script
runShellCommand("cleanLtrFinderID.py %s %s" % (inFile, outFile))
track.setPath(outFile)
if tempBed1 is not None:
runShellCommand("rm -f %s" % tempBed1)
# save a temporary xml
trackList.saveXML(tempTracksInfo)
def cutBedRegion(bedInterval, cutTrackPath, inBed, outBed):
""" intersect with a given interval """
tempPath = getLocalTempPath("Temp_cut", ".bed")
tempPath2 = getLocalTempPath("Temp_cut", ".bed")
runShellCommand("rm -f %s" % outBed)
runShellCommand(
"echo \"%s\t%s\t%s\n\" > %s" %
(bedInterval[0], bedInterval[1], bedInterval[2], tempPath2))
runShellCommand("intersectBed -a %s -b %s | sortBed > %s" %
(inBed, tempPath2, tempPath))
runShellCommand("subtractBed -a %s -b %s | sortBed > %s" %
(tempPath, cutTrackPath, outBed))
runShellCommand("rm -f %s %s" % (tempPath, tempPath2))
def combineTrack(track, outPath, tempRegionPath, iter, args):
""" merge track with outPath """
# make sure track is of form chrom start end state
tempColPath = getLocalTempPath("Temp", "_col.bed")
tempColFile = open(tempColPath, "w")
vc = track.getValCol() + 1
if track.getDist() == "binary":
assert track.getName() != args.outside
vc = 3
bedIntervals = readBedIntervals(track.getPath(), vc,
sort = True)
for bedInterval in bedIntervals:
outStr = "\t".join([str(x) for x in bedInterval])
if track.getDist() == "binary":
# state name = track name for binary track
outStr += "\t%s" % track.getName()
outStr += "\n"
tempColFile.write(outStr)
tempColFile.close()
# intersect the target region
tempIntersectPath = getLocalTempPath("Temp", "_int.bed")
runShellCommand("intersectBed -a %s -b %s > %s" % (
tempColPath, tempRegionPath, tempIntersectPath))
# add the outside states
tempGappedPath = getLocalTempPath("Temp", "_gap.bed")
runShellCommand("addBedGaps.py --state %s %s %s %s" % (
args.outside, tempRegionPath, tempIntersectPath, tempGappedPath))
# fit the names with previous interations' result
tempFitPath = getLocalTempPath("Temp", "_fit.bed")
if iter == 0:
runShellCommand("cp %s %s" % (tempGappedPath, tempFitPath))
else:
runShellCommand("fitStateNames.py %s %s %s --qualThresh %f --ignoreTgt %s" % (
outPath, tempGappedPath, tempFitPath, args.fitThresh, args.outside))
# now merge into outPath
runShellCommand("cat %s >> %s" % (tempFitPath, outPath))
runShellCommand("removeBedOverlaps.py %s > %s" % (outPath, tempColPath))
runShellCommand("mv %s %s" % (tempColPath, outPath))
# clean crap (note tempCol should already be gone)
runShellCommand("rm -f %s" % tempColPath)
runShellCommand("rm -f %s" % tempIntersectPath)
runShellCommand("rm -f %s" % tempGappedPath)
runShellCommand("rm -f %s" % tempFitPath)
def cutBedRegion(bedInterval, cutTrackPath, inBed, outBed):
""" intersect with a given interval """
tempPath = getLocalTempPath("Temp_cut", ".bed")
tempPath2 = getLocalTempPath("Temp_cut", ".bed")
runShellCommand("rm -f %s" % outBed)
runShellCommand("echo \"%s\t%s\t%s\n\" > %s" % (bedInterval[0],
bedInterval[1],
bedInterval[2],
tempPath2))
runShellCommand("intersectBed -a %s -b %s | sortBed > %s" % (inBed,
tempPath2,
tempPath))
runShellCommand("subtractBed -a %s -b %s | sortBed > %s" % (tempPath,
cutTrackPath,
outBed))
runShellCommand("rm -f %s %s" % (tempPath, tempPath2))
def checkExactOverlap(bed1, bed2):
""" make sure two bed files cover same region exactly: a requirement for all
code based on the comparisons in this module."""
errorMessage = (
"Bed files %s and %s cannot be compared. xxx. "
" Input files must be both sorted, cover the exact same region,"
" and contain no self-overlaps.") % (bed1, bed2)
# empty file may break downstream comparisons
size1 = os.path.getsize(bed1)
size2 = os.path.getsize(bed2)
if size1 == 0 or size2 == 0:
raise RuntimeError(
errorMessage.replace("xxx", "one or both inputs empty"))
# test self-overlap and sorting
intervals1 = readBedIntervals(bed1, sort=False)
for i in xrange(1, len(intervals1)):
if intersectSize(intervals1[i - 1], intervals1[i]) != 0:
raise RuntimeError(
errorMessage.replace(
"xxx", "Overlapping intervals %s and %s found in input1" %
(intervals1[i - 1], intervals1[i])))
if intervals1[i - 1] > intervals1[i]:
raise RuntimeError(
errorMessage.replace(
"xxx", "Out of order intervals %s and %s found in input1" %
(intervals1[i - 1], intervals1[i])))
# test self-overlap and sorting
intervals2 = readBedIntervals(bed1, sort=False)
for i in xrange(1, len(intervals2)):
if intersectSize(intervals2[i - 1], intervals2[i]) != 0:
raise RuntimeError(
errorMessage.replace(
"xxx", "Overlapping intervals %s and %s found in input2" %
(intervals2[i - 1], intervals2[i])))
if intervals2[i - 1] > intervals2[i]:
raise RuntimeError(
errorMessage.replace(
"xxx", "Out of order intervals %s and %s found in input2" %
(intervals2[i - 1], intervals2[i])))
# test intersection size
tempFile = getLocalTempPath("Temp_test", ".bed")
runShellCommand("subtractBed -a %s -b %s > %s" % (bed1, bed2, tempFile))
if os.path.getsize(tempFile) != 0:
runShellCommand("rm -f %s" % tempFile)
raise RuntimeError(
errorMessage.replace("xxx",
"Input1 covers regions outside input2"))
runShellCommand("subtractBed -a %s -b %s > %s" % (bed2, bed1, tempFile))
if os.path.getsize(tempFile) != 0:
runShellCommand("rm -f %s" % tempFile)
raise RuntimeError(
errorMessage.replace("xxx",
"Input2 covers regions outside input1"))
runShellCommand("rm -f %s" % tempFile)
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Set the score column of each bed interval in input to "
"(MODE, BINNED) average value of the intersection region in another track). "
"Can be used, for instance, to assign a copy number of each RepeatModeler "
"prediction...")
parser.add_argument("tracksInfo",
help="Path of Tracks Info file "
"containing paths to genome annotation tracks")
parser.add_argument("inBed", help="BED file to annotate")
parser.add_argument("track", help="Track to use for annotation")
parser.add_argument("outBed", help="Path for output, annotated BED file")
parser.add_argument("--name",
help="Set ID field (column 4 instead of 5)",
action="store_true",
default=False)
addLoggingOptions(parser)
args = parser.parse_args()
setLoggingFromOptions(args)
tempBedToolPath = initBedTool()
# read the tracks list
trackList = TrackList(args.tracksInfo)
track = trackList.getTrackByName(args.track)
if track is None:
raise RuntimeError("Can't find track %s" % args.track)
# make temporary tracks list with just our track so we can keep using
# tracks list interface but not read unecessary crap.
singleListPath = getLocalTempPath("Temp_secScore", ".bed")
trackList.trackList = [track]
trackList.saveXML(singleListPath)
obFile = open(args.outBed, "w")
# trackData interface not so great at cherry picking intervals.
# need to merge them up and use segmentation interface
filledIntervals, mergedIntervals = fillGaps(args.inBed)
# read track into trackData
trackData = TrackData()
logger.info("loading track %s" % singleListPath)
trackData.loadTrackData(singleListPath,
mergedIntervals,
segmentIntervals=filledIntervals,
applyMasking=False)
# finally, write the annotation
writeAnnotatedIntervals(trackData, filledIntervals, mergedIntervals,
obFile, args)
runShellCommand("rm -f %s" % singleListPath)
obFile.close()
cleanBedTool(tempBedToolPath)
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Set the score column of each bed interval in input to "
"(MODE, BINNED) average value of the intersection region in another track). "
"Can be used, for instance, to assign a copy number of each RepeatModeler "
"prediction...")
parser.add_argument("tracksInfo", help="Path of Tracks Info file "
"containing paths to genome annotation tracks")
parser.add_argument("inBed", help="BED file to annotate")
parser.add_argument("track", help="Track to use for annotation")
parser.add_argument("outBed", help="Path for output, annotated BED file")
parser.add_argument("--name", help="Set ID field (column 4 instead of 5)",
action="store_true", default=False)
addLoggingOptions(parser)
args = parser.parse_args()
setLoggingFromOptions(args)
tempBedToolPath = initBedTool()
# read the tracks list
trackList = TrackList(args.tracksInfo)
track = trackList.getTrackByName(args.track)
if track is None:
raise RuntimeError("Can't find track %s" % args.track)
# make temporary tracks list with just our track so we can keep using
# tracks list interface but not read unecessary crap.
singleListPath = getLocalTempPath("Temp_secScore", ".bed")
trackList.trackList = [track]
trackList.saveXML(singleListPath)
obFile = open(args.outBed, "w")
# trackData interface not so great at cherry picking intervals.
# need to merge them up and use segmentation interface
filledIntervals, mergedIntervals = fillGaps(args.inBed)
# read track into trackData
trackData = TrackData()
logger.info("loading track %s" % singleListPath)
trackData.loadTrackData(singleListPath, mergedIntervals,
segmentIntervals=filledIntervals,
applyMasking=False)
# finally, write the annotation
writeAnnotatedIntervals(trackData, filledIntervals, mergedIntervals, obFile,
args)
runShellCommand("rm -f %s" % singleListPath)
obFile.close()
cleanBedTool(tempBedToolPath)
def checkExactOverlap(bed1, bed2):
""" make sure two bed files cover same region exactly: a requirement for all
code based on the comparisons in this module."""
errorMessage = ("Bed files %s and %s cannot be compared. xxx. "
" Input files must be both sorted, cover the exact same region,"
" and contain no self-overlaps.") % (bed1, bed2)
# empty file may break downstream comparisons
size1 = os.path.getsize(bed1)
size2 = os.path.getsize(bed2)
if size1 == 0 or size2 == 0:
raise RuntimeError(errorMessage.replace("xxx", "one or both inputs empty"))
# test self-overlap and sorting
intervals1 = readBedIntervals(bed1, sort=False)
for i in xrange(1, len(intervals1)):
if intersectSize(intervals1[i-1], intervals1[i]) != 0:
raise RuntimeError(errorMessage.replace(
"xxx", "Overlapping intervals %s and %s found in input1" % (
intervals1[i-1], intervals1[i])))
if intervals1[i-1] > intervals1[i]:
raise RuntimeError(errorMessage.replace(
"xxx", "Out of order intervals %s and %s found in input1" % (
intervals1[i-1], intervals1[i])))
# test self-overlap and sorting
intervals2 = readBedIntervals(bed1, sort=False)
for i in xrange(1, len(intervals2)):
if intersectSize(intervals2[i-1], intervals2[i]) != 0:
raise RuntimeError(errorMessage.replace(
"xxx", "Overlapping intervals %s and %s found in input2" % (
intervals2[i-1], intervals2[i])))
if intervals2[i-1] > intervals2[i]:
raise RuntimeError(errorMessage.replace(
"xxx", "Out of order intervals %s and %s found in input2" % (
intervals2[i-1], intervals2[i])))
# test intersection size
tempFile = getLocalTempPath("Temp_test", ".bed")
runShellCommand("subtractBed -a %s -b %s > %s" % (bed1, bed2, tempFile))
if os.path.getsize(tempFile) != 0:
runShellCommand("rm -f %s" % tempFile)
raise RuntimeError(errorMessage.replace(
"xxx", "Input1 covers regions outside input2"))
runShellCommand("subtractBed -a %s -b %s > %s" % (bed2, bed1, tempFile))
if os.path.getsize(tempFile) != 0:
runShellCommand("rm -f %s" % tempFile)
raise RuntimeError(errorMessage.replace(
"xxx", "Input2 covers regions outside input1"))
runShellCommand("rm -f %s" % tempFile)
def filterEmptyRegions(genomePath, regions, outDir, cutTrackPath):
""" to a trial cut on each region. return a list of those that
aren't empty after cut """
filteredRegions = []
for i, region in enumerate(regions):
regionName = getRegionName(region, i)
tempPath1 = getLocalTempPath("Temp", ".bed")
cutBedRegion(region, cutTrackPath, genomePath, tempPath1)
intervals = bedRead(tempPath1)
runShellCommand("rm -f %s" % tempPath1)
if len(intervals) > 0:
filteredRegions.append(region)
return filteredRegions
def filterEmptyRegions(genomePath, regions, outDir, cutTrackPath):
""" to a trial cut on each region. return a list of those that
aren't empty after cut """
filteredRegions = []
for i, region in enumerate(regions):
regionName = getRegionName(region, i)
tempPath1 = getLocalTempPath("Temp", ".bed")
cutBedRegion(region, cutTrackPath, genomePath, tempPath1)
intervals = bedRead(tempPath1)
runShellCommand("rm -f %s" % tempPath1)
if len(intervals) > 0:
filteredRegions.append(region)
return filteredRegions
def runVennMaker(args0):
# venn_maker seems designed to run on intervals (and looks pretty broken doing this).
# try converting to base intervals.
todie = []
for i, f in enumerate(args.inputFiles):
tempFile = getLocalTempPath("Temp_%d" % i, ".bed")
todie.append(tempFile)
baserize(f, tempFile)
args.inputFiles[i] = tempFile
venn_maker(args.inputFiles, args.names, args.outTiff, "venn.R",
additional_args=None, run=True)
for f in todie:
runShellCommand("rm -f %s" % f)
def runVennMaker(args0):
# venn_maker seems designed to run on intervals (and looks pretty broken doing this).
# try converting to base intervals.
todie = []
for i, f in enumerate(args.inputFiles):
tempFile = getLocalTempPath("Temp_%d" % i, ".bed")
todie.append(tempFile)
baserize(f, tempFile)
args.inputFiles[i] = tempFile
venn_maker(args.inputFiles,
args.names,
args.outTiff,
"venn.R",
additional_args=None,
run=True)
for f in todie:
runShellCommand("rm -f %s" % f)
cmd += " --iter %d" % iter
cmd += " --segment %s" % trainSegPath
runShellCommand(cmd)
# eval ############
evalPath = "eval.bed"
if startPoint <=3:
cmd = "teHmmEval.py %s %s %s --bed %s --segment %s" % (trainTracksPath, modelPath, evalSegPath, evalPath, logOpts)
runShellCommand(cmd)
# fit ############
fitPath = "fit.bed"
fitFdrPath = "fitFdr.bed"
labelPath = "label.bed"
if startPoint <=4:
tempPath = getLocalTempPath("Tempmask", ".bed")
runShellCommand("mergeBed -i %s | sortBed > %s" % (evalSegPath, tempPath))
runShellCommand("intersectBed -a %s -b %s | sortBed > %s" % (modelerPath, tempPath, labelPath))
runShellCommand("rm -f %s" % tempPath)
fitCmd = "fitStateNames.py %s %s %s %s" % (labelPath, evalPath, fitPath, fitFlags)
fitFdrCmd = "fitStateNames.py %s %s %s %s" % (labelPath, evalPath, fitFdrPath, fitFlagsFdr)
runParallelShellCommands([fitCmd, fitFdrCmd], 2)
# compare ############
compDir = "comp"
if not os.path.exists(compDir):
runShellCommand("mkdir %s" % compDir)
def getTruthPath(idx):
return os.path.join(compDir, truthNames[idx] + ".bed")
fitPathMI = "fitMI.bed"
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Fill in masked intervals of an hmm prediction "
"(from teHmmEval.py) with state corresponding to surrounding"
" intervals.")
parser.add_argument("tracksXML", help="XML track list (used to id masking"
" tracks")
parser.add_argument("allBed", help="Target scope. Masked intervals outside"
" of these regions will not be included")
parser.add_argument("inBed", help="TE prediction BED file. State labels"
" should probably be mapped (ie with fitStateNames.py)")
parser.add_argument("outBed", help="Output BED. Will be equivalent to"
" the input bed except all gaps corresponding to "
"masked intervals will be filled")
parser.add_argument("--maxLen", help="Maximum length of a masked interval"
" to fill (inclusive). Use --delMask option with same value"
"if running compareBedStates.py after.",
type=int, default=sys.maxint)
parser.add_argument("--default", help="Default label to give to masked "
"region if no label can be determined", default="0")
parser.add_argument("--tgts", help="Only relabel gaps that "
"are flanked on both sides by the same state, and this state"
" is in this comma- separated list. --default used for other"
" gaps. If not targetst specified then all states checked.",
default=None)
parser.add_argument("--oneSidedTgts", help="Only relabel gaps that "
"are flanked on at least one side by a state in this comma-"
"separated list --default used for other gaps",
default=None)
parser.add_argument("--onlyDefault", help="Add the default state (--default) no"
" no all masked gaps no matter what. ie ignoring all other "
"logic", action="store_true", default=False)
parser.add_argument("--cut", help="Cut out gaps for masked tracks from the input."
" By default, the input is expected to come from the HMM "
"with mask intervals already absent, and will crash on with"
" an assertion error if an overlap is detected.",
action="store_true", default=False)
addLoggingOptions(parser)
args = parser.parse_args()
setLoggingFromOptions(args)
tempBedToolPath = initBedTool()
# make sets
tgtSet = set()
if args.tgts is not None:
tgtSet = set(args.tgts.split(","))
oneSidedTgtSet = set()
if args.oneSidedTgts is not None:
oneSidedTgtSet = set(args.oneSidedTgts.split(","))
assert len(tgtSet.intersection(oneSidedTgtSet)) == 0
# read the track list
trackList = TrackList(args.tracksXML)
maskTracks = trackList.getMaskTracks()
# read the input bed
inBed = args.inBed
if args.cut is True:
inBed = cutOutMaskIntervals(inBed, -1, args.maxLen + 1, args.tracksXML)
inputIntervals = readBedIntervals(inBed, ncol = 4, sort = True)
if args.cut is True:
runShellCommand("rm -f %s" % inBed)
if len(maskTracks) == 0 or len(inputIntervals) == 0:
runShellCommand("cp %s %s" % (args.inBed, args.outBed))
logger.warning("No mask tracks located in %s or"
" %s empty" % (args.tracksXML, args.inBed))
return 0
# make a temporary, combined, merged masking bed file
tempMaskBed = getLocalTempPath("Temp_mb", ".bed")
for maskTrack in maskTracks:
assert os.path.isfile(maskTrack.getPath())
runShellCommand("cat %s | setBedCol.py 3 mask | awk \'{print $1\"\t\"$2\"\t\"$3}\'>> %s" % (
maskTrack.getPath(), tempMaskBed))
maskedIntervals = getMergedBedIntervals(tempMaskBed, sort = True)
resolvedMasks = 0
if len(inputIntervals) == 0:
logger.warning("No mask tracks located in %s" % args.tracksXML)
return
inputIdx = 0
rightFlank = inputIntervals[inputIdx]
tempOutMask = getLocalTempPath("Temp_om", ".bed")
tempOutMaskFile = open(tempOutMask, "w")
for maskIdx, maskInterval in enumerate(maskedIntervals):
if maskInterval[2] - maskInterval[1] > args.maxLen:
continue
# find candidate right flank
while rightFlank < maskInterval:
if inputIdx == len(inputIntervals) - 1:
rightFlank = None
break
else:
inputIdx += 1
rightFlank = inputIntervals[inputIdx]
# candidate left flank
leftFlank = None
if inputIdx > 0:
leftFlank = inputIntervals[inputIdx - 1]
# identify flanking states if the intervals perfectly abut
leftState = None
if leftFlank is not None:
if leftFlank[0] == maskInterval[0] and leftFlank[2] == maskInterval[1]:
leftState = str(leftFlank[3])
else:
assert intersectSize(leftFlank, maskInterval) == 0
rightState = None
if rightFlank is not None:
if rightFlank[0] == maskInterval[0] and rightFlank[1] == maskInterval[2]:
rightState = str(rightFlank[3])
else:
assert intersectSize(rightFlank, maskInterval) == 0
# choose a state for the mask interval
maskState = str(args.default)
if args.onlyDefault is True:
pass
elif leftState is not None and leftState == rightState:
if len(tgtSet) == 0 or leftState in tgtSet:
maskState = leftState
elif leftState in oneSidedTgtSet:
maskState = leftState
elif rightState in oneSidedTgtSet:
maskState = rightState
# write our mask interval
tempOutMaskFile.write("%s\t%d\t%d\t%s\n" % (maskInterval[0], maskInterval[1],
maskInterval[2], maskState))
tempOutMaskFile.close()
tempMergePath1 = getLocalTempPath("Temp_mp", ".bed")
tempMergePath2 = getLocalTempPath("Temp_mp", ".bed")
runShellCommand("cp %s %s ; cat %s >> %s" % (args.inBed, tempMergePath1,
tempOutMask, tempMergePath1))
runShellCommand("cat %s | sortBed > %s" % (tempMergePath1, tempMergePath2))
tempScopePath = getLocalTempPath("temp_all", ".bed")
runShellCommand("mergeBed -i %s |sortBed > %s" % (args.allBed, tempScopePath))
runShellCommand("intersectBed -a %s -b %s > %s" % (tempMergePath2, tempScopePath,
args.outBed))
runShellCommand("rm -f %s" % " ".join([tempMaskBed, tempOutMask, tempMergePath1,
tempMergePath2, tempScopePath]))
cleanBedTool(tempBedToolPath)
def runPositionalComparison(argv, args):
""" hack to recursively exectute compareBedStates.py on a sliding window of the two
inputs and report accuracy in a BED file """
try:
windowToks = args.window.split(",")
assert len(windowToks) == 5
windowSize = int(windowToks[0])
stateName = windowToks[1]
compType = windowToks[2]
score = windowToks[3]
outBed = windowToks[4]
except:
raise RuntimeError("value passed to --window is not in valid format")
if compType == "base":
compIdx = 0
elif compType == "interval":
compIdx = 1
elif compType == "weighted":
compIdx = 2
else:
raise RuntimeError("invalid compType, %s, passed to --window" % compType)
if score != "f1" and score != "precision" and score != "recall":
raise RuntimeError("invalid score, %s, passed to --window" % score)
try:
outFile = open(outBed, "w")
except:
raise RuntimeError("invalid outBed, %s, passed to --window" % outBed)
tempBed = getLocalTempPath("Temp_region", ".bed")
runShellCommand("mergeBed -i %s > %s" % (args.bed1, tempBed))
chunkBed = getLocalTempPath("Temp_chunkBed", ".bed")
runShellCommand("chunkBedRegions.py %s %d --overlap .5 > %s" % (
tempBed, windowSize, chunkBed))
window = getLocalTempPath("Temp_window", ".bed")
slice1 = getLocalTempPath("Temp_slice1", ".bed")
slice2 = getLocalTempPath("Temp_slice2", ".bed")
compFile = getLocalTempPath("Temp_compFile", ".bed")
compOpts = ""
winIdx = argv.index("--window")
assert winIdx > 0 and winIdx < len(argv) -1 and argv[winIdx + 1] == args.window
for i in xrange(3, len(argv)):
if i != winIdx and i != winIdx + 1:
compOpts += " " + argv[i]
for chunk in readBedIntervals(chunkBed):
runShellCommand("echo \"%s\t%d\t%d\" > %s" % (chunk[0], chunk[1], chunk[2],
window))
runShellCommand("intersectBed -a %s -b %s | sortBed > %s" % (
args.bed1, window, slice1))
runShellCommand("intersectBed -a %s -b %s | sortBed > %s" % (
args.bed2, window, slice2))
runShellCommand("compareBedStates.py %s %s %s > %s" % (
slice1, slice2, compOpts, compFile))
stats = extractCompStatsFromFile(compFile)[compIdx]
if stateName not in stats:
stats[stateName] = (0,0)
f1 = 0.
prec, rec = stats[stateName]
if prec + rec > 0:
f1 = (2. * prec * rec) / (prec + rec)
val = f1
if score == "precision":
val = prec
elif score == "recall":
val = rec
outFile.write("%s\t%d\t%d\t%f\n" % (chunk[0], chunk[1], chunk[2], val))
runShellCommand("rm -f %s %s %s %s %s %s" % (tempBed, chunkBed, window,
slice1, slice2, compFile))
outFile.close()
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Remove ltr_finder ids from 4th column")
parser.add_argument("inBed", help="bed with ltr results to process")
parser.add_argument("outBed", help="bed to write output to. Will also "
"write outBed_sym.bed outBed_tsd_as_gap.bed etc.")
parser.add_argument("--keepOl", help="by default, if LTR elements "
"overlap, the one with the highest score (length "
"in event of tie) is kept. This option disables"
" this logic.", action="store_true", default=False)
parser.add_argument("--all", help="write _sym, _tsd_as_gap, etc. versions"
" of output", action="store_true", default=False)
parser.add_argument("--weak", help="score threshold such that any elemetns"
" with a score lower or equal to will be assigned the"
" prefix WEAK_ to their names.", type=float,
default=-1)
parser.add_argument("--weakIgnore", help="dont apply --weak to state names"
" that contain given keywords (defined as comma-separated"
" list", default=None)
args = parser.parse_args()
tempBedToolPath = initBedTool()
assert os.path.exists(args.inBed)
baseOut, ext = os.path.splitext(args.outBed)
if args.weakIgnore is not None:
args.weakIgnore = args.weakIgnore.split(",")
else:
args.weakIgnore = []
inBed = args.inBed
toRm = []
if not args.keepOl:
inBed = getLocalTempPath("Temp", ".bed")
removeOverlaps(args.inBed, inBed, args)
toRm.append(inBed)
os.system("sed -e \"s/|LTR_TE|[0-9]*//g\" -e \"s/|-//g\" %s > %s" % (
inBed, args.outBed))
if args.all:
symBed = baseOut + "_sym" + ext
os.system("sed -e \"s/|left//g\" -e \"s/|right//g\" %s > %s" % (args.outBed,
symBed))
tsd_as_gapsBed = baseOut + "_tsd_as_gap" + ext
os.system("grep -v TSD %s > %s" % (args.outBed, tsd_as_gapsBed))
sym_tsd_as_gapsBed = baseOut + "_sym_tsd_as_gap" + ext
os.system("grep -v TSD %s > %s" % (symBed, sym_tsd_as_gapsBed))
tsd_as_ltrBed = baseOut + "_tsd_as_ltr" + ext
os.system("sed -e \"s/TSD/LTR/g\" %s > %s" % (args.outBed, tsd_as_ltrBed))
sym_tsd_as_ltrBed = baseOut + "_sym_tsd_as_ltr" + ext
os.system("sed -e \"s/TSD/LTR/g\" %s > %s" % (symBed, sym_tsd_as_ltrBed))
singleBed = baseOut + "_single" + ext
os.system("sed -e \"s/LTR/inside/g\" %s > %s" % (sym_tsd_as_ltrBed,
singleBed))
for path in toRm:
runShellCommand("rm -f %s" % path)
cleanBedTool(tempBedToolPath)
def main(argv=None):
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Combine a bunch of non-numeric BED tracks into"
" single file using fitStateNames.py to try to keep names "
"consistent. Idea is to be used as baseline to compare"
" hmm to (via base-by-base statistics, primarily, since"
" this procedure could induce some fragmentation)")
parser.add_argument("tracksXML", help="Path of Tracks Info file "
"containing paths to genome annotation tracks")
parser.add_argument("regionBed", help="BED file representing "
"target region (best if whole genome)")
parser.add_argument("outBed", help="Output bed")
parser.add_argument("--tracks", help="Comma-separated list of "
"track names to use. All tracks will be"
" used by default", default=None)
parser.add_argument("--outside", help="Name to give non-annotated"
"regions", default="Outside")
parser.add_argument("--fitThresh", help="Min map percentage (0,1)"
" in order to rename (see --qualThresh option"
"of fitStateNames.py", type=float,
default=0.5)
addLoggingOptions(parser)
args = parser.parse_args()
setLoggingFromOptions(args)
tempBedToolPath = initBedTool()
inputTrackList = TrackList(args.tracksXML)
iter = 0
# get regionBed where all intervals are merged when possible
regionIntervals = getMergedBedIntervals(args.regionBed, sort=True)
tempRegionPath = getLocalTempPath("Temp", "_reg.bed")
tempRegionFile = open(tempRegionPath, "w")
for interval in regionIntervals:
tempRegionFile.write("\t".join([str(x) for x in interval]) + "\n")
tempRegionFile.close()
# accumulate tracks in temp file
tempOutPath = getLocalTempPath("Temp", "_out.bed")
for track in inputTrackList:
if track.shift is not None or track.scale is not None or\
track.logScale is not None or track.dist == "gaussian" or\
os.path.splitext(track.getPath())[1].lower() != ".bed":
logger.warning("Skipping numeric track %s" % track.getName())
elif args.tracks is None or track.getName() in args.tracks.split(","):
combineTrack(track, tempOutPath, tempRegionPath, iter, args)
iter += 1
# nothing got written, make everything outside
if iter == 0:
tempOutFile = open(tempOutPath, "w")
for interval in regionIntervals:
tempOutFile.write("%s\t%s\t%s\t%s\n" % (interval[0], interval[1],
interval[2], args.outside))
tempOutFile.close()
runShellCommand("mv %s %s" % (tempOutPath, args.outBed))
runShellCommand("rm -f %s" % (tempRegionPath))
cleanBedTool(tempBedToolPath)
def runTsd(args, tempTracksInfo):
""" run addTsdTrack on termini and chaux to generate tsd track"""
if args.noTsd is True:
return
origTrackList = TrackList(args.tracksInfo)
outTrackList = TrackList(tempTracksInfo)
tempFiles = []
tsdInputFiles = []
tsdInputTracks = []
# preprocess termini
lastzTracks = [origTrackList.getTrackByName(args.ltr_termini),
origTrackList.getTrackByName(args.tir)]
for terminiTrack in lastzTracks:
if terminiTrack is not None:
inFile = terminiTrack.getPath()
fillFile = getLocalTempPath("Temp_fill", ".bed")
tempBed = None
if inFile[-3:] == ".bb":
tempBed = getLocalTempPath("Temp_termini", ".bed")
runShellCommand("bigBedToBed %s %s" % (inFile, tempBed))
inFile = tempBed
runShellCommand("fillTermini.py %s %s" % (inFile, fillFile))
tsdInputFiles.append(fillFile)
tsdInputTracks.append(terminiTrack.getName())
tempFiles.append(fillFile)
if tempBed is not None:
runShellCommand("rm -f %s" % tempBed)
else:
logger.warning("Could not find termini track")
# add repeat_modeler
repeat_modelerTrack = outTrackList.getTrackByName(args.repeat_modeler)
if repeat_modelerTrack is not None:
tsdInputFiles.append(repeat_modelerTrack.getPath())
tsdInputTracks.append(repeat_modelerTrack.getName())
# run addTsdTrack (appending except first time)
# note we override input track paths in each case
assert len(tsdInputFiles) == len(tsdInputTracks)
for i in xrange(len(tsdInputFiles)):
optString = ""
if i > 0:
optString += " --append"
# really rough hardcoded params based on
# (A unified classification system for eukaryotic transposable elements
# Wicker et. al 2007)
if tsdInputTracks[i] == args.repeat_modeler:
optString += " --names LINE,SINE,Unknown"
optString += " --maxScore 20"
optString += " --left 20"
optString += " --right 20"
optString += " --min 5"
optString += " --max 20"
optString += " --overlap 20"
elif tsdInputTracks[i] == args.ltr_termini:
optString += " --maxScore 3"
optString += " --left 8"
optString += " --right 8"
optString += " --min 3"
optString += " --max 6"
elif tsdInputTracks[i] == args.tir:
optString += " --maxScore 3"
optString += " --left 15"
optString += " --right 15"
optString += " --min 3"
optString += " --max 12"
tempXMLOut = getLocalTempPath("Temp_tsd_xml", ".xml")
runShellCommand("addTsdTrack.py %s %s %s %s %s %s --inPath %s %s %s --numProc %d" % (
tempTracksInfo,
args.cleanTrackPath,
tempXMLOut,
tsdInputTracks[i],
args.sequence,
args.tsd,
tsdInputFiles[i],
optString,
args.logOpString,
args.numProc))
runShellCommand("mv %s %s" % (tempXMLOut, tempTracksInfo))
for i in xrange(len(tempFiles)):
runShellCommand("rm %s" % tempFiles[i])
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Generate HMM-usable tracklist from raw tracklist. EX "
"used to transform mustang_alyrata_tracks.xml -> "
"mustang_alyrata_clean.xml. Runs cleanRM.py cleanLtrFinder.py and "
" cleanTermini.py and addTsdTrack.py and setTrackScaling.py (also runs "
" removeBedOverlaps.py before each of the clean scripts)")
parser.add_argument("tracksInfo", help="Path of Tracks Info file "
"containing paths to genome annotation tracks")
parser.add_argument("allBed", help="Bed file spanning entire genome")
parser.add_argument("cleanTrackPath", help="Directory to write cleaned BED"
" tracks to")
parser.add_argument("outTracksInfo", help="Path to write modified tracks XML"
" to.")
parser.add_argument("--numBins", help="Maximum number of bins after scaling",
default=10, type=int)
parser.add_argument("--scaleTracks", help="Comma-separated list of tracks "
"to process for scaling. If not set, all"
" tracks listed as having a multinomial distribution"
" (since this is the default value, this includes "
"tracks with no distribution attribute) or gaussian "
"distribution will be processed.", default=None)
parser.add_argument("--skipScale", help="Comma-separated list of tracks to "
"skip for scaling.", default=None)
parser.add_argument("--ltr_termini", help="Name of termini track (appy tsd)",
default="ltr_termini")
parser.add_argument("--repeat_modeler", help="Name of repeat_modeler track (appy tsd)",
default="repeat_modeler")
parser.add_argument("--sequence", help="Name of fasta sequence track",
default="sequence")
parser.add_argument("--tsd", help="Name of tsd track to generate (appy cleanTermini.py)",
default="tsd")
parser.add_argument("--tir", help="Name of tir_termini track (appy cleanTermini.py)",
default="tir_termini")
parser.add_argument("--noScale", help="Dont do any scaling", default=False,
action="store_true")
parser.add_argument("--noTsd", help="Dont generate TSD track. NOTE:"
" TSD track is hardcoded to be generated from "
"termini and (non-LTR elements of ) chaux",
default=False, action="store_true")
parser.add_argument("--numProc", help="Number of processes to use for tsdFinder.py",
default=1, type=int)
addLoggingOptions(parser)
args = parser.parse_args()
setLoggingFromOptions(args)
tempBedToolPath = initBedTool()
args.logOpString = "--logLevel %s" % getLogLevelString()
if args.logFile is not None:
args.logOpString += " --logFile %s" % args.logFile
try:
os.makedirs(args.cleanTrackPath)
except:
pass
if not os.path.isdir(args.cleanTrackPath):
raise RuntimeError("Unable to find or create cleanTrack dir %s" %
args.cleanTrackPath)
tempTracksInfo = getLocalTempPath("Temp_mustang_alyrata_clean", "xml")
runCleaning(args, tempTracksInfo)
assert os.path.isfile(tempTracksInfo)
runTsd(args, tempTracksInfo)
runScaling(args, tempTracksInfo)
runShellCommand("rm -f %s" % tempTracksInfo)
cleanBedTool(tempBedToolPath)
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Remove ltr_finder ids from 4th column")
parser.add_argument("inBed", help="bed with ltr results to process")
parser.add_argument("outBed",
help="bed to write output to. Will also "
"write outBed_sym.bed outBed_tsd_as_gap.bed etc.")
parser.add_argument("--keepOl",
help="by default, if LTR elements "
"overlap, the one with the highest score (length "
"in event of tie) is kept. This option disables"
" this logic.",
action="store_true",
default=False)
parser.add_argument("--all",
help="write _sym, _tsd_as_gap, etc. versions"
" of output",
action="store_true",
default=False)
parser.add_argument("--weak",
help="score threshold such that any elemetns"
" with a score lower or equal to will be assigned the"
" prefix WEAK_ to their names.",
type=float,
default=-1)
parser.add_argument(
"--weakIgnore",
help="dont apply --weak to state names"
" that contain given keywords (defined as comma-separated"
" list",
default=None)
args = parser.parse_args()
tempBedToolPath = initBedTool()
assert os.path.exists(args.inBed)
baseOut, ext = os.path.splitext(args.outBed)
if args.weakIgnore is not None:
args.weakIgnore = args.weakIgnore.split(",")
else:
args.weakIgnore = []
inBed = args.inBed
toRm = []
if not args.keepOl:
inBed = getLocalTempPath("Temp", ".bed")
removeOverlaps(args.inBed, inBed, args)
toRm.append(inBed)
os.system("sed -e \"s/|LTR_TE|[0-9]*//g\" -e \"s/|-//g\" %s > %s" %
(inBed, args.outBed))
if args.all:
symBed = baseOut + "_sym" + ext
os.system("sed -e \"s/|left//g\" -e \"s/|right//g\" %s > %s" %
(args.outBed, symBed))
tsd_as_gapsBed = baseOut + "_tsd_as_gap" + ext
os.system("grep -v TSD %s > %s" % (args.outBed, tsd_as_gapsBed))
sym_tsd_as_gapsBed = baseOut + "_sym_tsd_as_gap" + ext
os.system("grep -v TSD %s > %s" % (symBed, sym_tsd_as_gapsBed))
tsd_as_ltrBed = baseOut + "_tsd_as_ltr" + ext
os.system("sed -e \"s/TSD/LTR/g\" %s > %s" %
(args.outBed, tsd_as_ltrBed))
sym_tsd_as_ltrBed = baseOut + "_sym_tsd_as_ltr" + ext
os.system("sed -e \"s/TSD/LTR/g\" %s > %s" %
(symBed, sym_tsd_as_ltrBed))
singleBed = baseOut + "_single" + ext
os.system("sed -e \"s/LTR/inside/g\" %s > %s" %
(sym_tsd_as_ltrBed, singleBed))
for path in toRm:
runShellCommand("rm -f %s" % path)
cleanBedTool(tempBedToolPath)
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Filter overlapping intervals out")
parser.add_argument("inputBed", help="Bed file to filter")
parser.add_argument("--bed12", help="Use bed12 exons instead of start/end"
" if present (equivalent to running bed12ToBed6 on"
" input first).", action="store_true", default=False)
parser.add_argument("--rm", help="Make sure intervals that are labeled as TE "
"by rm2State.sh script are never cut by ones that are not",
default=False, action='store_true')
addLoggingOptions(parser)
args = parser.parse_args()
setLoggingFromOptions(args)
assert os.path.isfile(args.inputBed)
tempBedToolPath = initBedTool()
# do the --rm filter. by splitting into TE / non-TE
# then removing everything in non-TE that overlaps
# TE. The adding the remainder back to TE.
inputPath = args.inputBed
if args.rm is True:
tempPath = getLocalTempPath("Temp_", ".bed")
tePath = getLocalTempPath("Temp_te_", ".bed")
runShellCommand("rm2State.sh %s |grep TE | sortBed > %s" % (
args.inputBed, tempPath))
runShellCommand("intersectBed -a %s -b %s | sortBed > %s" %(
args.inputBed, tempPath, tePath))
otherPath = getLocalTempPath("Temp_other_", ".bed")
runShellCommand("rm2State.sh %s |grep -v TE | sortBed > %s" % (
args.inputBed, tempPath))
runShellCommand("intersectBed -a %s -b %s | sortBed > %s" %(
args.inputBed, tempPath, otherPath))
if os.path.getsize(tePath) > 0 and\
os.path.getsize(otherPath) > 0:
filterPath = getLocalTempPath("Temp_filter_", ".bed")
runShellCommand("subtractBed -a %s -b %s | sortBed > %s" % (
otherPath, tePath, filterPath))
inputPath = getLocalTempPath("Temp_input_", ".bed")
runShellCommand("cat %s %s | sortBed > %s" % (
tePath, filterPath, inputPath))
runShellCommand("rm -f %s" % filterPath)
runShellCommand("rm -f %s %s %s" % (tePath, otherPath, tempPath))
bedIntervals = BedTool(inputPath).sort()
if args.bed12 is True:
bedIntervals = bedIntervals.bed6()
prevInterval = None
# this code has been way to buggy for something so simple
# keep extra list to check for sure even though it's a waste of
# time and space
sanity = []
for interval in bedIntervals:
if (prevInterval is not None and
interval.chrom == prevInterval.chrom and
interval.start < prevInterval.end):
logger.debug("Replace %d bases of \n%s with\n%s" % (
prevInterval.end - interval.start,
str(interval), str(prevInterval)))
interval.start = prevInterval.end
if interval.end > interval.start:
sys.stdout.write("%s" % str(interval))
sanity.append(interval)
prevInterval = interval
for i in xrange(len(sanity) - 1):
if sanity[i].chrom == sanity[i+1].chrom:
assert sanity[i+1].start >= sanity[i].end
cleanBedTool(tempBedToolPath)
if args.inputBed != inputPath:
runShellCommand("rm -f %s" % inputPath)
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Fill in masked intervals of an hmm prediction "
"(from teHmmEval.py) with state corresponding to surrounding"
" intervals.")
parser.add_argument("tracksXML",
help="XML track list (used to id masking"
" tracks")
parser.add_argument("allBed",
help="Target scope. Masked intervals outside"
" of these regions will not be included")
parser.add_argument(
"inBed",
help="TE prediction BED file. State labels"
" should probably be mapped (ie with fitStateNames.py)")
parser.add_argument("outBed",
help="Output BED. Will be equivalent to"
" the input bed except all gaps corresponding to "
"masked intervals will be filled")
parser.add_argument(
"--maxLen",
help="Maximum length of a masked interval"
" to fill (inclusive). Use --delMask option with same value"
"if running compareBedStates.py after.",
type=int,
default=sys.maxint)
parser.add_argument("--default",
help="Default label to give to masked "
"region if no label can be determined",
default="0")
parser.add_argument(
"--tgts",
help="Only relabel gaps that "
"are flanked on both sides by the same state, and this state"
" is in this comma- separated list. --default used for other"
" gaps. If not targetst specified then all states checked.",
default=None)
parser.add_argument(
"--oneSidedTgts",
help="Only relabel gaps that "
"are flanked on at least one side by a state in this comma-"
"separated list --default used for other gaps",
default=None)
parser.add_argument(
"--onlyDefault",
help="Add the default state (--default) no"
" no all masked gaps no matter what. ie ignoring all other "
"logic",
action="store_true",
default=False)
parser.add_argument(
"--cut",
help="Cut out gaps for masked tracks from the input."
" By default, the input is expected to come from the HMM "
"with mask intervals already absent, and will crash on with"
" an assertion error if an overlap is detected.",
action="store_true",
default=False)
addLoggingOptions(parser)
args = parser.parse_args()
setLoggingFromOptions(args)
tempBedToolPath = initBedTool()
# make sets
tgtSet = set()
if args.tgts is not None:
tgtSet = set(args.tgts.split(","))
oneSidedTgtSet = set()
if args.oneSidedTgts is not None:
oneSidedTgtSet = set(args.oneSidedTgts.split(","))
assert len(tgtSet.intersection(oneSidedTgtSet)) == 0
# read the track list
trackList = TrackList(args.tracksXML)
maskTracks = trackList.getMaskTracks()
# read the input bed
inBed = args.inBed
if args.cut is True:
inBed = cutOutMaskIntervals(inBed, -1, args.maxLen + 1, args.tracksXML)
inputIntervals = readBedIntervals(inBed, ncol=4, sort=True)
if args.cut is True:
runShellCommand("rm -f %s" % inBed)
if len(maskTracks) == 0 or len(inputIntervals) == 0:
runShellCommand("cp %s %s" % (args.inBed, args.outBed))
logger.warning("No mask tracks located in %s or"
" %s empty" % (args.tracksXML, args.inBed))
return 0
# make a temporary, combined, merged masking bed file
tempMaskBed = getLocalTempPath("Temp_mb", ".bed")
for maskTrack in maskTracks:
assert os.path.isfile(maskTrack.getPath())
runShellCommand(
"cat %s | setBedCol.py 3 mask | awk \'{print $1\"\t\"$2\"\t\"$3}\'>> %s"
% (maskTrack.getPath(), tempMaskBed))
maskedIntervals = getMergedBedIntervals(tempMaskBed, sort=True)
resolvedMasks = 0
if len(inputIntervals) == 0:
logger.warning("No mask tracks located in %s" % args.tracksXML)
return
inputIdx = 0
rightFlank = inputIntervals[inputIdx]
tempOutMask = getLocalTempPath("Temp_om", ".bed")
tempOutMaskFile = open(tempOutMask, "w")
for maskIdx, maskInterval in enumerate(maskedIntervals):
if maskInterval[2] - maskInterval[1] > args.maxLen:
continue
# find candidate right flank
while rightFlank < maskInterval:
if inputIdx == len(inputIntervals) - 1:
rightFlank = None
break
else:
inputIdx += 1
rightFlank = inputIntervals[inputIdx]
# candidate left flank
leftFlank = None
if inputIdx > 0:
leftFlank = inputIntervals[inputIdx - 1]
# identify flanking states if the intervals perfectly abut
leftState = None
if leftFlank is not None:
if leftFlank[0] == maskInterval[0] and leftFlank[
2] == maskInterval[1]:
leftState = str(leftFlank[3])
else:
assert intersectSize(leftFlank, maskInterval) == 0
rightState = None
if rightFlank is not None:
if rightFlank[0] == maskInterval[0] and rightFlank[
1] == maskInterval[2]:
rightState = str(rightFlank[3])
else:
assert intersectSize(rightFlank, maskInterval) == 0
# choose a state for the mask interval
maskState = str(args.default)
if args.onlyDefault is True:
pass
elif leftState is not None and leftState == rightState:
if len(tgtSet) == 0 or leftState in tgtSet:
maskState = leftState
elif leftState in oneSidedTgtSet:
maskState = leftState
elif rightState in oneSidedTgtSet:
maskState = rightState
# write our mask interval
tempOutMaskFile.write(
"%s\t%d\t%d\t%s\n" %
(maskInterval[0], maskInterval[1], maskInterval[2], maskState))
tempOutMaskFile.close()
tempMergePath1 = getLocalTempPath("Temp_mp", ".bed")
tempMergePath2 = getLocalTempPath("Temp_mp", ".bed")
runShellCommand("cp %s %s ; cat %s >> %s" %
(args.inBed, tempMergePath1, tempOutMask, tempMergePath1))
runShellCommand("cat %s | sortBed > %s" % (tempMergePath1, tempMergePath2))
tempScopePath = getLocalTempPath("temp_all", ".bed")
runShellCommand("mergeBed -i %s |sortBed > %s" %
(args.allBed, tempScopePath))
runShellCommand("intersectBed -a %s -b %s > %s" %
(tempMergePath2, tempScopePath, args.outBed))
runShellCommand("rm -f %s" % " ".join([
tempMaskBed, tempOutMask, tempMergePath1, tempMergePath2, tempScopePath
]))
cleanBedTool(tempBedToolPath)
def parallelDispatch(argv, args):
""" chunk up input with chrom option. recursivlely launch eval. merge
results """
jobList = []
chromIntervals = readBedIntervals(args.chroms, sort=True)
chromFiles = []
regionFiles = []
segFiles = []
statsFiles = []
offset = args.co
for chrom in chromIntervals:
cmdToks = copy.deepcopy(argv)
cmdToks[cmdToks.index("--chrom") + 1] = ""
cmdToks[cmdToks.index("--chrom")] = ""
chromPath = getLocalTempPath("TempChromPath", ".bed")
cpFile = open(chromPath, "w")
cpFile.write("%s\t%d\t%d\t0\t0\t.\n" % (chrom[0], chrom[1], chrom[2]))
cpFile.close()
regionPath = getLocalTempPath("Temp", ".bed")
runShellCommand("intersectBed -a %s -b %s | sortBed > %s" %
(args.allBed, chromPath, regionPath))
if os.path.getsize(regionPath) < 2:
continue
offset += int(chrom[2]) - int(chrom[1])
regionFiles.append(regionPath)
chromFiles.append(chromPath)
cmdToks[2] = regionPath
segPath = getLocalTempPath("Temp", ".bed")
cmdToks[3] = segPath
segFiles.append(segPath)
if "--co" in cmdToks:
cmdToks[cmdToks.index("--co") + 1] = str(offset)
else:
cmdToks.append("--co")
cmdToks.append(str(offset))
if args.stats is not None:
statsPath = getLocalTempPath("Temp", ".bed")
cmdToks[cmdToks.index("--stats") + 1] = statsPath
statsFiles.append(statsPath)
cmd = " ".join(cmdToks)
jobList.append(cmd)
runParallelShellCommands(jobList, args.proc)
for i in xrange(len(jobList)):
if i == 0:
ct = ">"
else:
ct = ">>"
runShellCommand("cat %s %s %s" % (segFiles[i], ct, args.outBed))
if len(statsFiles) > 0:
runShellCommand("cat %s %s %s" % (statsFiles[i], ct, args.stats))
for i in itertools.chain(chromFiles, regionFiles, segFiles, statsFiles):
runShellCommand("rm %s" % i)
def parallelDispatch(argv, args):
""" chunk up input with chrom option. recursivlely launch eval. merge
results """
jobList = []
chromIntervals = readBedIntervals(args.chroms, sort=True)
chromFiles = []
regionFiles = []
bedFiles = []
pdFiles = []
bicFiles = []
edFiles = []
for chrom in chromIntervals:
cmdToks = copy.deepcopy(argv)
cmdToks[cmdToks.index("--chrom") + 1] = ""
cmdToks[cmdToks.index("--chrom")] = ""
chromPath = getLocalTempPath("Temp", ".bed")
cpFile = open(chromPath, "w")
cpFile.write("%s\t%d\t%d\t0\t0\t.\n" % (chrom[0], chrom[1], chrom[2]))
cpFile.close()
regionPath = getLocalTempPath("Temp", ".bed")
runShellCommand("intersectBed -a %s -b %s | sortBed > %s" % (args.bedRegions,
chromPath,
regionPath))
if os.path.getsize(regionPath) < 2:
continue
regionFiles.append(regionPath)
chromFiles.append(chromPath)
cmdToks[3] = regionPath
if args.bed is not None:
bedPath = getLocalTempPath("Temp", ".bed")
cmdToks[cmdToks.index("--bed")+1] = bedPath
bedFiles.append(bedPath)
if args.pd is not None:
pdPath = getLocalTempPath("Temp", ".bed")
cmdToks[cmdToks.index("--pd")+1] = pdPath
pdFiles.append(pdPath)
if args.ed is not None:
edPath = getLocalTempPath("Temp", ".bed")
cmdToks[cmdToks.index("--ed")+1] = edPath
edFiles.append(edPath)
if args.bic is not None:
bicPath = getLocalTempPath("Temp", ".bic")
cmdToks[cmdToks.index("--bic")+1] = bicPath
bicFiles.append(bicPath)
cmd = " ".join(cmdToks)
jobList.append(cmd)
runParallelShellCommands(jobList, args.proc)
for i in xrange(len(jobList)):
if i == 0:
ct = ">"
else:
ct = ">>"
if len(bedFiles) > 0:
runShellCommand("cat %s %s %s" % (bedFiles[i], ct, args.bed))
if len(pdFiles) > 0:
runShellCommand("cat %s %s %s" % (pdFiles[i], ct, args.pd))
if len(edFiles) > 0:
runShellCommand("cat %s %s %s" % (edFiles[i], ct, args.ed))
if len(bicFiles) > 0:
runShellCommand("cat %s %s %s" % (bicFiles[i], ct, args.bic))
for i in itertools.chain(chromFiles, regionFiles, bedFiles, pdFiles, edFiles,
bicFiles):
runShellCommand("rm %s" % i)
def runPositionalComparison(argv, args):
""" hack to recursively exectute compareBedStates.py on a sliding window of the two
inputs and report accuracy in a BED file """
try:
windowToks = args.window.split(",")
assert len(windowToks) == 5
windowSize = int(windowToks[0])
stateName = windowToks[1]
compType = windowToks[2]
score = windowToks[3]
outBed = windowToks[4]
except:
raise RuntimeError("value passed to --window is not in valid format")
if compType == "base":
compIdx = 0
elif compType == "interval":
compIdx = 1
elif compType == "weighted":
compIdx = 2
else:
raise RuntimeError("invalid compType, %s, passed to --window" %
compType)
if score != "f1" and score != "precision" and score != "recall":
raise RuntimeError("invalid score, %s, passed to --window" % score)
try:
outFile = open(outBed, "w")
except:
raise RuntimeError("invalid outBed, %s, passed to --window" % outBed)
tempBed = getLocalTempPath("Temp_region", ".bed")
runShellCommand("mergeBed -i %s > %s" % (args.bed1, tempBed))
chunkBed = getLocalTempPath("Temp_chunkBed", ".bed")
runShellCommand("chunkBedRegions.py %s %d --overlap .5 > %s" %
(tempBed, windowSize, chunkBed))
window = getLocalTempPath("Temp_window", ".bed")
slice1 = getLocalTempPath("Temp_slice1", ".bed")
slice2 = getLocalTempPath("Temp_slice2", ".bed")
compFile = getLocalTempPath("Temp_compFile", ".bed")
compOpts = ""
winIdx = argv.index("--window")
assert winIdx > 0 and winIdx < len(argv) - 1 and argv[winIdx +
1] == args.window
for i in xrange(3, len(argv)):
if i != winIdx and i != winIdx + 1:
compOpts += " " + argv[i]
for chunk in readBedIntervals(chunkBed):
runShellCommand("echo \"%s\t%d\t%d\" > %s" %
(chunk[0], chunk[1], chunk[2], window))
runShellCommand("intersectBed -a %s -b %s | sortBed > %s" %
(args.bed1, window, slice1))
runShellCommand("intersectBed -a %s -b %s | sortBed > %s" %
(args.bed2, window, slice2))
runShellCommand("compareBedStates.py %s %s %s > %s" %
(slice1, slice2, compOpts, compFile))
stats = extractCompStatsFromFile(compFile)[compIdx]
if stateName not in stats:
stats[stateName] = (0, 0)
f1 = 0.
prec, rec = stats[stateName]
if prec + rec > 0:
f1 = (2. * prec * rec) / (prec + rec)
val = f1
if score == "precision":
val = prec
elif score == "recall":
val = rec
outFile.write("%s\t%d\t%d\t%f\n" % (chunk[0], chunk[1], chunk[2], val))
runShellCommand("rm -f %s %s %s %s %s %s" %
(tempBed, chunkBed, window, slice1, slice2, compFile))
outFile.close()
def runTsd(args, tempTracksInfo):
""" run addTsdTrack on termini and chaux to generate tsd track"""
if args.noTsd is True:
return
origTrackList = TrackList(args.tracksInfo)
outTrackList = TrackList(tempTracksInfo)
tempFiles = []
tsdInputFiles = []
tsdInputTracks = []
# preprocess termini
lastzTracks = [
origTrackList.getTrackByName(args.ltr_termini),
origTrackList.getTrackByName(args.tir)
]
for terminiTrack in lastzTracks:
if terminiTrack is not None:
inFile = terminiTrack.getPath()
fillFile = getLocalTempPath("Temp_fill", ".bed")
tempBed = None
if inFile[-3:] == ".bb":
tempBed = getLocalTempPath("Temp_termini", ".bed")
runShellCommand("bigBedToBed %s %s" % (inFile, tempBed))
inFile = tempBed
runShellCommand("fillTermini.py %s %s" % (inFile, fillFile))
tsdInputFiles.append(fillFile)
tsdInputTracks.append(terminiTrack.getName())
tempFiles.append(fillFile)
if tempBed is not None:
runShellCommand("rm -f %s" % tempBed)
else:
logger.warning("Could not find termini track")
# add repeat_modeler
repeat_modelerTrack = outTrackList.getTrackByName(args.repeat_modeler)
if repeat_modelerTrack is not None:
tsdInputFiles.append(repeat_modelerTrack.getPath())
tsdInputTracks.append(repeat_modelerTrack.getName())
# run addTsdTrack (appending except first time)
# note we override input track paths in each case
assert len(tsdInputFiles) == len(tsdInputTracks)
for i in xrange(len(tsdInputFiles)):
optString = ""
if i > 0:
optString += " --append"
# really rough hardcoded params based on
# (A unified classification system for eukaryotic transposable elements
# Wicker et. al 2007)
if tsdInputTracks[i] == args.repeat_modeler:
optString += " --names LINE,SINE,Unknown"
optString += " --maxScore 20"
optString += " --left 20"
optString += " --right 20"
optString += " --min 5"
optString += " --max 20"
optString += " --overlap 20"
elif tsdInputTracks[i] == args.ltr_termini:
optString += " --maxScore 3"
optString += " --left 8"
optString += " --right 8"
optString += " --min 3"
optString += " --max 6"
elif tsdInputTracks[i] == args.tir:
optString += " --maxScore 3"
optString += " --left 15"
optString += " --right 15"
optString += " --min 3"
optString += " --max 12"
tempXMLOut = getLocalTempPath("Temp_tsd_xml", ".xml")
runShellCommand(
"addTsdTrack.py %s %s %s %s %s %s --inPath %s %s %s --numProc %d" %
(tempTracksInfo, args.cleanTrackPath, tempXMLOut,
tsdInputTracks[i], args.sequence, args.tsd, tsdInputFiles[i],
optString, args.logOpString, args.numProc))
runShellCommand("mv %s %s" % (tempXMLOut, tempTracksInfo))
for i in xrange(len(tempFiles)):
runShellCommand("rm %s" % tempFiles[i])
cmd += " --segment %s" % trainSegPath
runShellCommand(cmd)
# eval ############
evalPath = "eval.bed"
if startPoint <= 3:
cmd = "teHmmEval.py %s %s %s --bed %s --segment %s" % (
trainTracksPath, modelPath, evalSegPath, evalPath, logOpts)
runShellCommand(cmd)
# fit ############
fitPath = "fit.bed"
fitFdrPath = "fitFdr.bed"
labelPath = "label.bed"
if startPoint <= 4:
tempPath = getLocalTempPath("Tempmask", ".bed")
runShellCommand("mergeBed -i %s | sortBed > %s" % (evalSegPath, tempPath))
runShellCommand("intersectBed -a %s -b %s | sortBed > %s" %
(modelerPath, tempPath, labelPath))
runShellCommand("rm -f %s" % tempPath)
fitCmd = "fitStateNames.py %s %s %s %s" % (labelPath, evalPath, fitPath,
fitFlags)
fitFdrCmd = "fitStateNames.py %s %s %s %s" % (labelPath, evalPath,
fitFdrPath, fitFlagsFdr)
runParallelShellCommands([fitCmd, fitFdrCmd], 2)
# compare ############
compDir = "comp"
if not os.path.exists(compDir):
runShellCommand("mkdir %s" % compDir)
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Cut names off at first |, /, ?, or _")
parser.add_argument("inBed", help="bed with chaux results to process")
parser.add_argument("--keepSlash", help="dont strip anything after slash "
"ex: DNA/HELITRONY1C -> DNA", action="store_true",
default=False)
parser.add_argument("--keepUnderscore", help="dont strip anything after _ ",
action="store_true", default=False)
parser.add_argument("--leaveNumbers", help="by default, numbers as the end"
" of names are trimmed off. ex: DNA/HELITRONY1C -> "
" DNA/HELITRONY. This option disables this behaviour",
default=False)
parser.add_argument("--mapPrefix", help="Rename all strings with given "
"prefix to just the prefix. ex: --mapPrefix DNA/HELI"
" would cause any instance of DNA/HELITRONY1C or "
"HELITRON2 to be mapped to just DNA/HELI. This option"
" overrides --keepSlash and --leaveNumbers for the"
" elements to which it applies. This option can be"
" specified more than once. ex --mapPrefix DNA/HELI "
"--maxPrefix DNA/ASINE.", action="append")
parser.add_argument("--minScore", help="Minimum score value to not filter"
" out", default=-sys.maxint, type=float)
parser.add_argument("--maxScore", help="Maximum score value to not filter"
" out", default=sys.maxint, type=float)
parser.add_argument("--overlap", help="Dont run removeBedOverlaps.py",
action="store_true", default=False)
args = parser.parse_args()
assert os.path.exists(args.inBed)
assert args.minScore <= args.maxScore
tempBedToolPath = initBedTool()
tempPath = getLocalTempPath("Temp_cleanOut", ".bed")
tempPath2 = getLocalTempPath("Temp2_", ".bed")
tempFile = open(tempPath, "w")
for interval in BedTool(args.inBed).sort():
# filter score if exists
try:
if interval.score is not None and\
(float(interval.score) < args.minScore or
float(interval.score) > args.maxScore):
continue
except:
pass
prefix = findPrefix(interval.name, args.mapPrefix)
if prefix is not None:
# prefix was specified with --mapPrefix, that's what we use
interval.name = prefix
else:
# otherwise, strip after |
if "|" in interval.name:
interval.name = interval.name[:interval.name.find("|")]
# strip after ?
if "?" in interval.name:
interval.name = interval.name[:interval.name.find("?")]
#strip after _ unlerss told not to
if "_" in interval.name and args.keepUnderscore is False:
interval.name = interval.name[:interval.name.find("_")]
# strip after "/" unless told not to
if "/" in interval.name and args.keepSlash is False:
interval.name = interval.name[:interval.name.find("/")]
# strip trailing digits (and anything after) unless told not to
if args.leaveNumbers is False:
m = re.search("\d", interval.name)
if m is not None:
interval.name = interval.name[:m.start()]
tempFile.write(str(interval))
tempFile.close()
if not args.overlap:
runShellCommand("removeBedOverlaps.py %s --rm > %s" % (tempPath,
tempPath2))
tempPath, tempPath2, = tempPath2, tempPath
tempFile = open(tempPath, "r")
for line in tempFile:
sys.stdout.write(line)
tempFile.close()
runShellCommand("rm -f %s %s" % (tempPath, tempPath2))
cleanBedTool(tempBedToolPath)
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Cut names off at first |, /, ?, or _")
parser.add_argument("inBed", help="bed with chaux results to process")
parser.add_argument("--keepSlash",
help="dont strip anything after slash "
"ex: DNA/HELITRONY1C -> DNA",
action="store_true",
default=False)
parser.add_argument("--keepUnderscore",
help="dont strip anything after _ ",
action="store_true",
default=False)
parser.add_argument("--leaveNumbers",
help="by default, numbers as the end"
" of names are trimmed off. ex: DNA/HELITRONY1C -> "
" DNA/HELITRONY. This option disables this behaviour",
default=False)
parser.add_argument("--mapPrefix",
help="Rename all strings with given "
"prefix to just the prefix. ex: --mapPrefix DNA/HELI"
" would cause any instance of DNA/HELITRONY1C or "
"HELITRON2 to be mapped to just DNA/HELI. This option"
" overrides --keepSlash and --leaveNumbers for the"
" elements to which it applies. This option can be"
" specified more than once. ex --mapPrefix DNA/HELI "
"--maxPrefix DNA/ASINE.",
action="append")
parser.add_argument("--minScore",
help="Minimum score value to not filter"
" out",
default=-sys.maxint,
type=float)
parser.add_argument("--maxScore",
help="Maximum score value to not filter"
" out",
default=sys.maxint,
type=float)
parser.add_argument("--overlap",
help="Dont run removeBedOverlaps.py",
action="store_true",
default=False)
args = parser.parse_args()
assert os.path.exists(args.inBed)
assert args.minScore <= args.maxScore
tempBedToolPath = initBedTool()
tempPath = getLocalTempPath("Temp_cleanOut", ".bed")
tempPath2 = getLocalTempPath("Temp2_", ".bed")
tempFile = open(tempPath, "w")
for interval in BedTool(args.inBed).sort():
# filter score if exists
try:
if interval.score is not None and\
(float(interval.score) < args.minScore or
float(interval.score) > args.maxScore):
continue
except:
pass
prefix = findPrefix(interval.name, args.mapPrefix)
if prefix is not None:
# prefix was specified with --mapPrefix, that's what we use
interval.name = prefix
else:
# otherwise, strip after |
if "|" in interval.name:
interval.name = interval.name[:interval.name.find("|")]
# strip after ?
if "?" in interval.name:
interval.name = interval.name[:interval.name.find("?")]
#strip after _ unlerss told not to
if "_" in interval.name and args.keepUnderscore is False:
interval.name = interval.name[:interval.name.find("_")]
# strip after "/" unless told not to
if "/" in interval.name and args.keepSlash is False:
interval.name = interval.name[:interval.name.find("/")]
# strip trailing digits (and anything after) unless told not to
if args.leaveNumbers is False:
m = re.search("\d", interval.name)
if m is not None:
interval.name = interval.name[:m.start()]
tempFile.write(str(interval))
tempFile.close()
if not args.overlap:
runShellCommand("removeBedOverlaps.py %s --rm > %s" %
(tempPath, tempPath2))
tempPath, tempPath2, = tempPath2, tempPath
tempFile = open(tempPath, "r")
for line in tempFile:
sys.stdout.write(line)
tempFile.close()
runShellCommand("rm -f %s %s" % (tempPath, tempPath2))
cleanBedTool(tempBedToolPath)
def main(argv=None):
if argv is None:
argv = sys.argv
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Generate HMM-usable tracklist from raw tracklist. EX "
"used to transform mustang_alyrata_tracks.xml -> "
"mustang_alyrata_clean.xml. Runs cleanRM.py cleanLtrFinder.py and "
" cleanTermini.py and addTsdTrack.py and setTrackScaling.py (also runs "
" removeBedOverlaps.py before each of the clean scripts)")
parser.add_argument("tracksInfo",
help="Path of Tracks Info file "
"containing paths to genome annotation tracks")
parser.add_argument("allBed", help="Bed file spanning entire genome")
parser.add_argument("cleanTrackPath",
help="Directory to write cleaned BED"
" tracks to")
parser.add_argument("outTracksInfo",
help="Path to write modified tracks XML"
" to.")
parser.add_argument("--numBins",
help="Maximum number of bins after scaling",
default=10,
type=int)
parser.add_argument("--scaleTracks",
help="Comma-separated list of tracks "
"to process for scaling. If not set, all"
" tracks listed as having a multinomial distribution"
" (since this is the default value, this includes "
"tracks with no distribution attribute) or gaussian "
"distribution will be processed.",
default=None)
parser.add_argument("--skipScale",
help="Comma-separated list of tracks to "
"skip for scaling.",
default=None)
parser.add_argument("--ltr_termini",
help="Name of termini track (appy tsd)",
default="ltr_termini")
parser.add_argument("--repeat_modeler",
help="Name of repeat_modeler track (appy tsd)",
default="repeat_modeler")
parser.add_argument("--sequence",
help="Name of fasta sequence track",
default="sequence")
parser.add_argument(
"--tsd",
help="Name of tsd track to generate (appy cleanTermini.py)",
default="tsd")
parser.add_argument(
"--tir",
help="Name of tir_termini track (appy cleanTermini.py)",
default="tir_termini")
parser.add_argument("--noScale",
help="Dont do any scaling",
default=False,
action="store_true")
parser.add_argument("--noTsd",
help="Dont generate TSD track. NOTE:"
" TSD track is hardcoded to be generated from "
"termini and (non-LTR elements of ) chaux",
default=False,
action="store_true")
parser.add_argument("--numProc",
help="Number of processes to use for tsdFinder.py",
default=1,
type=int)
addLoggingOptions(parser)
args = parser.parse_args()
setLoggingFromOptions(args)
tempBedToolPath = initBedTool()
args.logOpString = "--logLevel %s" % getLogLevelString()
if args.logFile is not None:
args.logOpString += " --logFile %s" % args.logFile
try:
os.makedirs(args.cleanTrackPath)
except:
pass
if not os.path.isdir(args.cleanTrackPath):
raise RuntimeError("Unable to find or create cleanTrack dir %s" %
args.cleanTrackPath)
tempTracksInfo = getLocalTempPath("Temp_mustang_alyrata_clean", "xml")
runCleaning(args, tempTracksInfo)
assert os.path.isfile(tempTracksInfo)
runTsd(args, tempTracksInfo)
runScaling(args, tempTracksInfo)
runShellCommand("rm -f %s" % tempTracksInfo)
cleanBedTool(tempBedToolPath)
def parallelDispatch(argv, args):
""" chunk up input with chrom option. recursivlely launch eval. merge
results """
jobList = []
chromIntervals = readBedIntervals(args.chroms, sort=True)
chromFiles = []
regionFiles = []
segFiles = []
statsFiles = []
offset = args.co
for chrom in chromIntervals:
cmdToks = copy.deepcopy(argv)
cmdToks[cmdToks.index("--chrom") + 1] = ""
cmdToks[cmdToks.index("--chrom")] = ""
chromPath = getLocalTempPath("TempChromPath", ".bed")
cpFile = open(chromPath, "w")
cpFile.write("%s\t%d\t%d\t0\t0\t.\n" % (chrom[0], chrom[1], chrom[2]))
cpFile.close()
regionPath = getLocalTempPath("Temp", ".bed")
runShellCommand("intersectBed -a %s -b %s | sortBed > %s" % (args.allBed,
chromPath,
regionPath))
if os.path.getsize(regionPath) < 2:
continue
offset += int(chrom[2]) - int(chrom[1])
regionFiles.append(regionPath)
chromFiles.append(chromPath)
cmdToks[2] = regionPath
segPath = getLocalTempPath("Temp", ".bed")
cmdToks[3] = segPath
segFiles.append(segPath)
if "--co" in cmdToks:
cmdToks[cmdToks.index("--co")+1] = str(offset)
else:
cmdToks.append("--co")
cmdToks.append(str(offset))
if args.stats is not None:
statsPath = getLocalTempPath("Temp", ".bed")
cmdToks[cmdToks.index("--stats")+1] = statsPath
statsFiles.append(statsPath)
cmd = " ".join(cmdToks)
jobList.append(cmd)
runParallelShellCommands(jobList, args.proc)
for i in xrange(len(jobList)):
if i == 0:
ct = ">"
else:
ct = ">>"
runShellCommand("cat %s %s %s" % (segFiles[i], ct, args.outBed))
if len(statsFiles) > 0:
runShellCommand("cat %s %s %s" % (statsFiles[i], ct, args.stats))
for i in itertools.chain(chromFiles, regionFiles, segFiles, statsFiles):
runShellCommand("rm %s" % i)