def checkGapOverlap(db, table, checkUnbridged=False):
"""Check if any items in a table over lap with bridged and unbridged gaps"""
tableBed = trackUtils.makeBedFromTable(db, table)
tableFileName = db + "." + table + ".bed.temp"
tableFile = open(tableFileName, "w")
tableFile.write(tableBed)
tableFile.close()
gapOverlapOut = ""
gapTableList = trackUtils.checkSplit(db, "gap")
gapFileName = str(db) + ".gap.bed.temp"
gapFile = open(gapFileName, "a")
for tbl in gapTableList:
gapBed = trackUtils.makeBedFromTable(db, "gap")
gapFile.write(gapBed)
gapFile.close()
gapOverFile = str(db) + ".gapOver.bed"
bedIntCmd = [
"bedIntersect", "-allowStartEqualEnd -aHitAny", gapFileName,
tableFileName, gapOverFile
]
qaUtils.runCommand(bedIntCmd)
gapOverUrls = constructOutputUrls(db, table, gapOverFile)
gapOverlapOut += gapOverUrls
# Remove intermediate files
os.remove(gapOverFile)
os.remove(gapFileName)
# If checkUnbridged is set, then also output overlap with unbridged gaps
if checkUnbridged == True:
# Create temp file to store unbridged gaps
gapUnbrFileName = str(db) + ".gap.unbr.bed.temp"
gapUnbrFile = open(gapUnbrFileName, "a")
# Get unbridged gaps
for tbl in gapTableList:
gapUnbrBed = trackUtils.makeBedFromTable(db,
"gap",
whereOpt="bridge='no'")
gapUnbrFile.write(gapUnbrBed)
gapUnbrFile.close()
gapUnbrOverFile = str(db) + ".gapOver.unbr.bed"
bedIntCmd = [
"bedIntersect", "-allowStartEqualEnd -aHitAny", gapUnbrFileName,
tableFileName, gapUnbrOverFile
]
qaUtils.runCommand(bedIntCmd)
gapUnbrOverUrls = constructOutputUrls(db, table, gapUnbrOverFile)
gapOverlapOut += gapUnbrOverUrls
# Remove intermediate files
os.remove(gapUnbrOverFile)
os.remove(gapUnbrFileName)
os.remove(tableFileName)
return gapOverlapOut
def __featureBits(self):
"""Adds featureBits output (both regular and overlap w/ gap) to sumRow."""
fbCommand = ["featureBits", "-countGaps", self.db, self.table]
fbOut, fbErr = qaUtils.runCommand(fbCommand)
# normal featureBits output actually goes to stderr
self.sumRow.setFeatureBits(fbErr.rstrip("in intersection\n"))
fbGapCommand = ["featureBits", "-countGaps", self.db, self.table, "gap"]
fbGapOut, fbGapErr = qaUtils.runCommand(fbGapCommand)
self.sumRow.setFeatureBitsGaps(fbGapErr.rstrip("in intersection\n"))
def __featureBits(self):
"""Adds featureBits output (both regular and overlap w/ gap) to sumRow."""
fbCommand = ["featureBits", "-countGaps", self.db, self.table]
fbOut, fbErr = qaUtils.runCommand(fbCommand)
# normal featureBits output actually goes to stderr
self.sumRow.setFeatureBits(fbErr.rstrip("in intersection\n"))
fbGapCommand = ["featureBits", "-countGaps", self.db, self.table, "gap"]
fbGapOut, fbGapErr = qaUtils.runCommand(fbGapCommand)
self.sumRow.setFeatureBitsGaps(fbGapErr.rstrip("in intersection\n"))
def checkGapOverlap(db, table, checkUnbridged=False):
"""Check if any items in a table over lap with bridged and unbridged gaps"""
tableBedTemp = makeBedFromTable(db, table)
gapOverlapOut = ""
gapTableList = checkSplit(db, "gap")
gapFileName = str(db) + ".gap.bed.temp"
gapFile = open(gapFileName, 'w')
for tbl in gapTableList:
hgsqlGapOut = qaUtils.callHgsql(
db, "select chrom, chromStart, chromEnd from " + tbl)
gapFile.write(hgsqlGapOut)
gapFile.close()
gapOverFile = str(db) + ".gapOver.bed"
bedIntCmd = [
"bedIntersect", "-aHitAny", gapFileName, tableBedTemp.name, gapOverFile
]
qaUtils.runCommand(bedIntCmd)
gapOverUrls = constructOutputUrls(db, table, gapOverFile)
gapOverlapOut += gapOverUrls
# Remove intermediate files
os.remove(gapOverFile)
os.remove(gapFileName)
# If checkUnbridged is set, then also output overlap with unbridged gaps
if checkUnbridged == True:
# Create temp file to store unbridged gaps
gapUnbrFileName = str(db) + ".gap.unbr.bed.temp"
gapUnbrFile = open(gapUnbrFileName, 'w')
# Get unbridged gaps
for tbl in gapTableList:
hgsqlGapOut = qaUtils.callHgsql(
db, "select chrom, chromStart, chromEnd from " + tbl +
" where bridge='no'")
gapUnbrFile.write(hgsqlGapOut)
gapUnbrFile.close()
gapUnbrOverFile = str(db) + ".gapOver.unbr.bed"
bedIntCmd = [
"bedIntersect", "-aHitAny", gapUnbrFileName, tableBedTemp.name,
gapUnbrOverFile
]
qaUtils.runCommand(bedIntCmd)
gapUnbrOverUrls = constructOutputUrls(db, table, gapUnbrOverFile)
gapOverlapOut += gapUnbrOverUrls
# Remove intermediate files
os.remove(gapUnbrOverFile)
os.remove(gapUnbrFileName)
return gapOverlapOut
def checkGapOverlap(db, table, checkUnbridged=False):
"""Check if any items in a table over lap with bridged and unbridged gaps"""
tableBed = trackUtils.makeBedFromTable(db, table)
tableFileName = db + "." + table + ".bed.temp"
tableFile = open(tableFileName, "w")
tableFile.write(tableBed)
tableFile.close()
gapOverlapOut = ""
gapTableList = trackUtils.checkSplit(db, "gap")
gapFileName = str(db) + ".gap.bed.temp"
gapFile = open(gapFileName, "a")
for tbl in gapTableList:
gapBed = trackUtils.makeBedFromTable(db, "gap")
gapFile.write(gapBed)
gapFile.close()
gapOverFile = str(db) + ".gapOver.bed"
bedIntCmd = ["bedIntersect", "-allowStartEqualEnd", "-aHitAny", gapFileName, tableFileName, gapOverFile]
qaUtils.runCommand(bedIntCmd)
gapOverUrls = constructOutputUrls(db, table, gapOverFile)
gapOverlapOut += gapOverUrls
# Remove intermediate files
os.remove(gapOverFile)
os.remove(gapFileName)
# If checkUnbridged is set, then also output overlap with unbridged gaps
if checkUnbridged == True:
# Create temp file to store unbridged gaps
gapUnbrFileName = str(db) + ".gap.unbr.bed.temp"
gapUnbrFile = open(gapUnbrFileName, "a")
# Get unbridged gaps
for tbl in gapTableList:
gapUnbrBed = trackUtils.makeBedFromTable(db, "gap", whereOpt="bridge='no'")
gapUnbrFile.write(gapUnbrBed)
gapUnbrFile.close()
gapUnbrOverFile = str(db) + ".gapOver.unbr.bed"
bedIntCmd = ["bedIntersect", "-allowStartEqualEnd", "-aHitAny", gapUnbrFileName, tableFileName, gapUnbrOverFile]
qaUtils.runCommand(bedIntCmd)
gapUnbrOverUrls = constructOutputUrls(db, table, gapUnbrOverFile)
gapOverlapOut += gapUnbrOverUrls
# Remove intermediate files
os.remove(gapUnbrOverFile)
os.remove(gapUnbrFileName)
os.remove(tableFileName)
return gapOverlapOut
def __featureBits(self):
"""Adds featureBits output (both regular and overlap w/ gap) to sumRow.
Also records commands and results in reporter"""
fbCommand = ["featureBits", "-countGaps", self.db, self.table]
fbOut, fbErr = qaUtils.runCommand(fbCommand)
# normal featureBits output actually goes to stderr
self.reporter.writeLine(" ".join(fbCommand))
self.reporter.writeLine(str(fbErr))
self.sumRow.setFeatureBits(fbErr.rstrip("in intersection\n"))
fbGapCommand = ["featureBits", "-countGaps", self.db, self.table, "gap"]
fbGapOut, fbGapErr = qaUtils.runCommand(fbGapCommand)
self.reporter.writeLine(" ".join(fbGapCommand))
self.reporter.writeLine(str(fbGapErr))
self.sumRow.setFeatureBitsGaps(fbGapErr.rstrip("in intersection\n"))
def __featureBits(self):
"""Adds featureBits output (both regular and overlap w/ gap) to sumRow.
Also records commands and results in reporter"""
fbCommand = ["featureBits", "-countGaps", self.db, self.table]
fbOut, fbErr = qaUtils.runCommand(fbCommand)
# normal featureBits output actually goes to stderr
self.reporter.writeLine(' '.join(fbCommand))
self.reporter.writeLine(str(fbErr))
self.sumRow.setFeatureBits(fbErr.rstrip("in intersection\n"))
fbGapCommand = ["featureBits", "-countGaps", self.db, self.table, "gap"]
fbGapOut, fbGapErr = qaUtils.runCommand(fbGapCommand)
self.reporter.writeLine(' '.join(fbGapCommand))
self.reporter.writeLine(str(fbGapErr))
self.sumRow.setFeatureBitsGaps(fbGapErr.rstrip("in intersection\n"))
def __getAttributeForTrack(self, attribute, track):
"""Uses tdbQuery to get an attribute where track or table == 'track' and removes label."""
cmd = [
"tdbQuery",
"select " + attribute + " from " + self.db + " where table='" + track + "' or track='" + track + "'",
]
cmdout, cmderr = qaUtils.runCommand(cmd)
return cmdout.strip(attribute).strip()
def getAttributeForTrack(attribute, db, track):
"""Uses tdbQuery to get an attribute where track or table == 'track' and removes label."""
cmd = [
"tdbQuery", "select " + attribute + " from " + db + " where table='" +
track + "' or track='" + track + "'"
]
cmdout, cmderr = qaUtils.runCommand(cmd)
return cmdout.strip(attribute).strip()
def __pslCheck(self):
"""Runs pslCheck program on this table and sends result to reporter's filehandle."""
self.reporter.beginStep(self.db, self.table, "pslCheck")
# Get baseColorUseSequence setting
tdbCommand = ["tdbQuery", "select baseColorUseSequence from " + self.db +\
" where track='" + self.table +"' or table='"+ self.table +"'"]
tdbOut, tdbErr = qaUtils.runCommand(tdbCommand)
# Turn tdbOut into array so we can extract different pieces
tdbOut = tdbOut.replace("\n", "")
tdbOut = tdbOut.split(" ")
# tdbOut[1] should be what option is used for baseColorUseSequence
# We want to fully run pslCheck for those entries with "table" and "extFile"
if len(tdbOut) > 2:
if tdbOut[1] == "table":
# Get sizes of sequences in specified table
sizesOut = qaUtils.callHgsql(
self.db, "select name, length(seq) from " + tdbOut[2])
elif tdbOut[1] == "extFile":
# Extract sizes from named seq table
sizesOut = qaUtils.callHgsql(
self.db, "select acc, size from " + tdbOut[2])
# Write itemSizes to file
itemSizes = open("%s.sizes" % self.table, 'w')
itemSizes.write(sizesOut)
itemSizes.close()
# Check if $db.sizes file exists
if not os.path.isfile("%s.chrom.sizes" % self.db):
# If it doens't exist, get chrom sizes from chromInfo
chromSizesOut = qaUtils.callHgsql(self.db,
"select * from chromInfo")
chromSizes = open("%s.chrom.sizes" % self.db, 'w')
chromSizes.write(chromSizesOut)
chromSizes.close()
# Run more in-depth version of pslCheck
command = ("pslCheck", "-querySizes=%s.sizes" % self.table ,\
"-targetSizes=%s.chrom.sizes" % self.db, "-db=" + self.db, self.table)
self.reporter.writeCommand(command)
commandOut, commandErr, commandReturnCode = qaUtils.runCommandNoAbort(
command)
# Write output to file
self.reporter.fh.write(commandErr)
# Clean up intermediate item sizes file
os.remove("%s.sizes" % self.table)
# For everything else, use generic set of steps
else:
command = ["pslCheck", "db=" + self.db, self.table]
self.reporter.writeCommand(command)
commandOut, commandErr, commandReturnCode = qaUtils.runCommandNoAbort(
command)
self.reporter.fh.write(commandErr)
if commandReturnCode:
self.recordError()
else:
self.recordPass()
self.reporter.endStep()
def __featureBits(self):
"""Runs featureBits. Adds output to sumRow and records commands and results in reporter."""
fbCommand = ["featureBits", "-countGaps", self.db, self.table]
fbOut, fbErr = qaUtils.runCommand(fbCommand)
# normal featureBits output actually goes to stderr
self.reporter.writeLine(' '.join(fbCommand))
self.reporter.writeLine(str(fbErr))
self.sumRow.setFeatureBits(fbErr.rstrip("in intersection\n"))
fbGapCommand = ["featureBits", "-countGaps", self.db, self.table, "gap"]
fbGapOut, fbGapErr = qaUtils.runCommand(fbGapCommand)
self.reporter.writeLine(' '.join(fbGapCommand))
self.reporter.writeLine(str(fbGapErr))
self.sumRow.setFeatureBitsGaps(fbGapErr.rstrip("in intersection\n"))
# Check overlap with gaps, both bridged and unbridged
gapOverlapOut = checkGapOverlap.checkGapOverlap(self.db, self.table, True)
self.reporter.writeLine(gapOverlapOut)
def __featureBits(self):
"""Runs featureBits. Adds output to sumRow and records commands and results in reporter."""
fbCommand = ["featureBits", "-countGaps", self.db, self.table]
fbOut, fbErr = qaUtils.runCommand(fbCommand)
# normal featureBits output actually goes to stderr
self.reporter.writeLine(' '.join(fbCommand))
self.reporter.writeLine(str(fbErr))
self.sumRow.setFeatureBits(fbErr.rstrip("in intersection\n"))
fbGapCommand = ["featureBits", "-countGaps", self.db, self.table, "gap"]
fbGapOut, fbGapErr = qaUtils.runCommand(fbGapCommand)
self.reporter.writeLine(' '.join(fbGapCommand))
self.reporter.writeLine(str(fbGapErr))
self.sumRow.setFeatureBitsGaps(fbGapErr.rstrip("in intersection\n"))
# Check overlap with gaps, both bridged and unbridged
gapOverlapOut = ''
if self.table != 'gap':
gapOverlapOut = checkGapOverlap.checkGapOverlap(self.db, self.table, True)
self.reporter.writeLine(gapOverlapOut)
def __pslCheck(self):
"""Runs pslCheck program on this table and sends result to reporter's filehandle."""
self.reporter.beginStep(self.db, self.table, "pslCheck")
# Get baseColorUseSequence setting
tdbCommand = ["tdbQuery", "select baseColorUseSequence from " + self.db +\
" where track='" + self.table +"' or table='"+ self.table +"'"]
tdbOut, tdbErr = qaUtils.runCommand(tdbCommand)
# Turn tdbOut into array so we can extract different pieces
tdbOut = tdbOut.replace("\n","")
tdbOut = tdbOut.split(" ")
# tdbOut[1] should be what option is used for baseColorUseSequence
# We want to fully run pslCheck for those entries with "table" and "extFile"
if len(tdbOut) > 2:
if tdbOut[1] == "table":
# Get sizes of sequences in specified table
sizesOut = qaUtils.callHgsql(self.db, "select name, length(seq) from " + tdbOut[2])
elif tdbOut[1] == "extFile":
# Extract sizes from named seq table
sizesOut = qaUtils.callHgsql(self.db, "select acc, size from " + tdbOut[2])
# Write itemSizes to tempfile
itemSizesTemp = tempfile.NamedTemporaryFile(mode='w')
itemSizes = open(itemSizesTemp.name, 'w')
itemSizes.write(sizesOut)
itemSizes.close()
# Write chromSizes to tempfile
chromSizesTemp = tempfile.NamedTemporaryFile(mode='w')
chromSizesOut = qaUtils.callHgsql(self.db, "select chrom,size from chromInfo")
chromSizes = open(chromSizesTemp.name, 'w')
chromSizes.write(chromSizesOut)
chromSizes.close()
# Run more in-depth version of pslCheck
command = ("pslCheck", "-querySizes=" + itemSizesTemp.name ,\
"-targetSizes=" + chromSizesTemp.name, "-db=" + self.db, self.table)
self.reporter.writeCommand(command)
commandOut, commandErr, commandReturnCode = qaUtils.runCommandNoAbort(command)
# Write output to file
self.reporter.fh.write(commandErr)
# For everything else, use generic set of steps
else:
command = ["pslCheck", "db=" + self.db, self.table]
self.reporter.writeCommand(command)
commandOut, commandErr, commandReturnCode = qaUtils.runCommandNoAbort(command)
self.reporter.fh.write(commandErr)
if commandReturnCode:
self.recordError()
else:
self.recordPass()
self.reporter.endStep()