def main():
'''
main run call
'''
debug = False
from optparse import OptionParser
usage = "usage: %prog [options] -g [GENOME] -i [INPUT_REGION_GFF] -r [RANKBY_BAM_FILE] -o [OUTPUT_FOLDER] [OPTIONAL_FLAGS]"
parser = OptionParser(usage=usage)
# required flags
parser.add_option(
"-i",
"--i",
dest="input",
nargs=1,
default=None,
help="Enter a .gff or .bed file of binding sites used to make enhancers"
)
parser.add_option("-r",
"--rankby",
dest="rankby",
nargs=1,
default=None,
help="bamfile to rank enhancer by")
parser.add_option("-o",
"--out",
dest="out",
nargs=1,
default=None,
help="Enter an output folder")
parser.add_option("-g",
"--genome",
dest="genome",
nargs=1,
default=None,
help="Enter the genome build (MM9,MM8,HG18,HG19)")
# optional flags
parser.add_option(
"-b",
"--bams",
dest="bams",
nargs=1,
default=None,
help="Enter a comma separated list of additional bam files to map to")
parser.add_option("-c",
"--control",
dest="control",
nargs=1,
default=None,
help="bamfile to rank enhancer by")
parser.add_option(
"-s",
"--stitch",
dest="stitch",
nargs=1,
default='',
help=
"Enter a max linking distance for stitching. Default will determine optimal stitching parameter"
)
parser.add_option(
"-t",
"--tss",
dest="tss",
nargs=1,
default=0,
help="Enter a distance from TSS to exclude. 0 = no TSS exclusion")
parser.add_option(
"--mask",
dest="mask",
nargs=1,
default=None,
help=
"Mask a set of regions from analysis. Provide a .bed or .gff of masking regions"
)
# RETRIEVING FLAGS
(options, args) = parser.parse_args()
if not options.input or not options.rankby or not options.out or not options.genome:
print('hi there')
parser.print_help()
exit()
# making the out folder if it doesn't exist
outFolder = utils.formatFolder(options.out, True)
# figuring out folder schema
gffFolder = utils.formatFolder(outFolder + 'gff/', True)
mappedFolder = utils.formatFolder(outFolder + 'mappedGFF/', True)
# GETTING INPUT FILE
if options.input.split('.')[-1] == 'bed':
# CONVERTING A BED TO GFF
inputGFFName = options.input.split('/')[-1][0:-4]
inputGFFFile = '%s%s.gff' % (gffFolder, inputGFFName)
utils.bedToGFF(options.input, inputGFFFile)
elif options.input.split('.')[-1] == 'gff':
# COPY THE INPUT GFF TO THE GFF FOLDER
inputGFFFile = options.input
os.system('cp %s %s' % (inputGFFFile, gffFolder))
else:
print(
'WARNING: INPUT FILE DOES NOT END IN .gff or .bed. ASSUMING .gff FILE FORMAT'
)
# COPY THE INPUT GFF TO THE GFF FOLDER
inputGFFFile = options.input
os.system('cp %s %s' % (inputGFFFile, gffFolder))
# GETTING THE LIST OF BAMFILES TO PROCESS
if options.control:
bamFileList = [options.rankby, options.control]
else:
bamFileList = [options.rankby]
if options.bams:
bamFileList += options.bams.split(',')
#bamFileList = utils.uniquify(bamFileList) # makes sad when you have the same control bam over and over again
# optional args
# Stitch parameter
if options.stitch == '':
stitchWindow = ''
else:
stitchWindow = int(options.stitch)
# tss options
tssWindow = int(options.tss)
if tssWindow != 0:
removeTSS = True
else:
removeTSS = False
# GETTING THE BOUND REGION FILE USED TO DEFINE ENHANCERS
print('USING %s AS THE INPUT GFF' % (inputGFFFile))
inputName = inputGFFFile.split('/')[-1].split('.')[0]
# GETTING THE GENOME
genome = options.genome
print('USING %s AS THE GENOME' % genome)
# GETTING THE CORRECT ANNOT FILE
cwd = os.getcwd()
genomeDict = {
'HG18': '%s/annotation/hg18_refseq.ucsc' % (cwd),
'MM9': '%s/annotation/mm9_refseq.ucsc' % (cwd),
'HG19': '%s/annotation/hg19_refseq.ucsc' % (cwd),
'MM8': '%s/annotation/mm8_refseq.ucsc' % (cwd),
'MM10': '%s/annotation/mm10_refseq.ucsc' % (cwd),
'RN4': '%s/annotation/rn4_refseq.ucsc' % (cwd),
'RN6': '%s/annotation/rn6_refseq.ucsc' % (cwd),
}
annotFile = genomeDict[genome.upper()]
# MAKING THE START DICT
print('MAKING START DICT')
startDict = utils.makeStartDict(annotFile)
#GET CHROMS FOUND IN THE BAMS
print('GETTING CHROMS IN BAMFILES')
bamChromList = getBamChromList(bamFileList)
print("USING THE FOLLOWING CHROMS")
print(bamChromList)
#LOADING IN THE GFF AND FILTERING BY CHROM
print('LOADING AND FILTERING THE GFF')
inputGFF = filterGFF(inputGFFFile, bamChromList)
# LOADING IN THE BOUND REGION REFERENCE COLLECTION
print('LOADING IN GFF REGIONS')
referenceCollection = utils.gffToLocusCollection(inputGFF)
print('CHECKING REFERENCE COLLECTION:')
checkRefCollection(referenceCollection)
# MASKING REFERENCE COLLECTION
# see if there's a mask
if options.mask:
maskFile = options.mask
# if it's a bed file
if maskFile.split('.')[-1].upper() == 'BED':
maskGFF = utils.bedToGFF(maskFile)
elif maskFile.split('.')[-1].upper() == 'GFF':
maskGFF = utils.parseTable(maskFile, '\t')
else:
print("MASK MUST BE A .gff or .bed FILE")
sys.exit()
maskCollection = utils.gffToLocusCollection(maskGFF)
# now mask the reference loci
referenceLoci = referenceCollection.getLoci()
filteredLoci = [
locus for locus in referenceLoci
if len(maskCollection.getOverlap(locus, 'both')) == 0
]
print("FILTERED OUT %s LOCI THAT WERE MASKED IN %s" %
(len(referenceLoci) - len(filteredLoci), maskFile))
referenceCollection = utils.LocusCollection(filteredLoci, 50)
# NOW STITCH REGIONS
print('STITCHING REGIONS TOGETHER')
stitchedCollection, debugOutput, stitchWindow = regionStitching(
referenceCollection, inputName, outFolder, stitchWindow, tssWindow,
annotFile, removeTSS)
# NOW MAKE A STITCHED COLLECTION GFF
print('MAKING GFF FROM STITCHED COLLECTION')
stitchedGFF = utils.locusCollectionToGFF(stitchedCollection)
# making sure start/stop ordering are correct
for i in range(len(stitchedGFF)):
line = stitchedGFF[i]
start = int(line[3])
stop = int(line[4])
if start > stop:
line[3] = stop
line[4] = start
print(stitchWindow)
print(type(stitchWindow))
if not removeTSS:
stitchedGFFFile = '%s%s_%sKB_STITCHED.gff' % (gffFolder, inputName,
str(stitchWindow / 1000))
stitchedGFFName = '%s_%sKB_STITCHED' % (inputName,
str(stitchWindow / 1000))
debugOutFile = '%s%s_%sKB_STITCHED.debug' % (gffFolder, inputName,
str(stitchWindow / 1000))
else:
stitchedGFFFile = '%s%s_%sKB_STITCHED_TSS_DISTAL.gff' % (
gffFolder, inputName, str(stitchWindow / 1000))
stitchedGFFName = '%s_%sKB_STITCHED_TSS_DISTAL' % (
inputName, str(stitchWindow / 1000))
debugOutFile = '%s%s_%sKB_STITCHED_TSS_DISTAL.debug' % (
gffFolder, inputName, str(stitchWindow / 1000))
# WRITING DEBUG OUTPUT TO DISK
if debug:
print('WRITING DEBUG OUTPUT TO DISK AS %s' % (debugOutFile))
utils.unParseTable(debugOutput, debugOutFile, '\t')
# WRITE THE GFF TO DISK
print('WRITING STITCHED GFF TO DISK AS %s' % (stitchedGFFFile))
utils.unParseTable(stitchedGFF, stitchedGFFFile, '\t')
# SETTING UP THE OVERALL OUTPUT FILE
outputFile1 = outFolder + stitchedGFFName + '_ENHANCER_REGION_MAP.txt'
print('OUTPUT WILL BE WRITTEN TO %s' % (outputFile1))
# MAPPING TO THE NON STITCHED (ORIGINAL GFF)
# MAPPING TO THE STITCHED GFF
# Try to use the bamliquidatior_path.py script on cluster, otherwise, failover to local (in path), otherwise fail.
bamliquidator_path = 'bamliquidator_batch.py'
bamFileListUnique = list(bamFileList)
bamFileListUnique = utils.uniquify(bamFileListUnique)
#prevent redundant mapping
print("MAPPING TO THE FOLLOWING BAMS:")
print(bamFileListUnique)
for bamFile in bamFileListUnique:
bamFileName = bamFile.split('/')[-1]
# MAPPING TO THE STITCHED GFF
mappedOut1Folder = '%s%s_%s_MAPPED' % (mappedFolder, stitchedGFFName,
bamFileName)
mappedOut1File = '%s%s_%s_MAPPED/matrix.txt' % (
mappedFolder, stitchedGFFName, bamFileName)
if utils.checkOutput(mappedOut1File, 0.2, 0.2):
print("FOUND %s MAPPING DATA FOR BAM: %s" %
(stitchedGFFFile, mappedOut1File))
else:
cmd1 = bamliquidator_path + " --sense . -e 200 --match_bamToGFF -r %s -o %s %s" % (
stitchedGFFFile, mappedOut1Folder, bamFile)
print(cmd1)
os.system(cmd1)
if utils.checkOutput(mappedOut1File, 0.2, 5):
print("SUCCESSFULLY MAPPED TO %s FROM BAM: %s" %
(stitchedGFFFile, bamFileName))
else:
print("ERROR: FAILED TO MAP %s FROM BAM: %s" %
(stitchedGFFFile, bamFileName))
sys.exit()
print('BAM MAPPING COMPLETED NOW MAPPING DATA TO REGIONS')
# CALCULATE DENSITY BY REGION
# NEED TO FIX THIS FUNCTION TO ACCOUNT FOR DIFFERENT OUTPUTS OF LIQUIDATOR
mapCollection(stitchedCollection,
referenceCollection,
bamFileList,
mappedFolder,
outputFile1,
refName=stitchedGFFName)
print('CALLING AND PLOTTING SUPER-ENHANCERS')
if options.control:
rankbyName = options.rankby.split('/')[-1]
controlName = options.control.split('/')[-1]
cmd = 'R --no-save %s %s %s %s < ROSE2_callSuper.R' % (
outFolder, outputFile1, inputName, controlName)
else:
rankbyName = options.rankby.split('/')[-1]
controlName = 'NONE'
cmd = 'R --no-save %s %s %s %s < ROSE2_callSuper.R' % (
outFolder, outputFile1, inputName, controlName)
print(cmd)
os.system(cmd)
# calling the gene mapper
time.sleep(20)
superTableFile = "%s_SuperEnhancers.table.txt" % (inputName)
if options.control:
cmd = "python ROSE2_geneMapper.py -g %s -r %s -c %s -i %s%s &" % (
genome, options.rankby, options.control, outFolder, superTableFile)
else:
cmd = "python ROSE2_geneMapper.py -g %s -r %s -i %s%s &" % (
genome, options.rankby, outFolder, superTableFile)
os.system(cmd)
stretchTableFile = "%s_StretchEnhancers.table.txt" % (inputName)
if options.control:
cmd = "python ROSE2_geneMapper.py -g %s -r %s -c %s -i %s%s &" % (
genome, options.rankby, options.control, outFolder,
stretchTableFile)
else:
cmd = "python ROSE2_geneMapper.py -g %s -r %s -i %s%s &" % (
genome, options.rankby, outFolder, stretchTableFile)
os.system(cmd)
superStretchTableFile = "%s_SuperStretchEnhancers.table.txt" % (inputName)
if options.control:
cmd = "python ROSE2_geneMapper.py -g %s -r %s -c %s -i %s%s &" % (
genome, options.rankby, options.control, outFolder,
superStretchTableFile)
else:
cmd = "python ROSE2_geneMapper.py -g %s -r %s -i %s%s &" % (
genome, options.rankby, outFolder, superStretchTableFile)
os.system(cmd)
def main():
'''
main run call
'''
debug = False
from optparse import OptionParser
usage = "usage: %prog [options] -g [GENOME] -i [INPUT_REGION_GFF] -r [RANKBY_BAM_FILE] -o [OUTPUT_FOLDER] [OPTIONAL_FLAGS]"
parser = OptionParser(usage=usage)
# required flags
parser.add_option(
"-i",
"--i",
dest="input",
nargs=1,
default=None,
help=
"Enter a comma separated list of .gff or .bed file of binding sites used to make enhancers"
)
parser.add_option("-r",
"--rankby",
dest="rankby",
nargs=1,
default=None,
help="Enter a comma separated list of bams to rank by")
parser.add_option("-o",
"--out",
dest="out",
nargs=1,
default=None,
help="Enter an output folder")
parser.add_option("-g",
"--genome",
dest="genome",
nargs=1,
default=None,
help="Enter the genome build (MM9,MM8,HG18,HG19)")
# optional flags
parser.add_option(
"-n",
"--name",
dest="name",
nargs=1,
default=None,
help="Provide a name for the analysis otherwise ROSE will guess")
parser.add_option(
"-c",
"--control",
dest="control",
nargs=1,
default=None,
help=
"Enter a comma separated list of control bams. Can either provide a single control bam for all rankby bams, or provide a control bam for each individual bam"
)
parser.add_option(
"-s",
"--stitch",
dest="stitch",
nargs=1,
default='',
help=
"Enter a max linking distance for stitching. Default will determine optimal stitching parameter"
)
parser.add_option(
"-t",
"--tss",
dest="tss",
nargs=1,
default=0,
help="Enter a distance from TSS to exclude. 0 = no TSS exclusion")
parser.add_option(
"--mask",
dest="mask",
nargs=1,
default=None,
help=
"Mask a set of regions from analysis. Provide a .bed or .gff of masking regions"
)
# RETRIEVING FLAGS
(options, args) = parser.parse_args()
if not options.input or not options.rankby or not options.out or not options.genome:
print('hi there')
parser.print_help()
exit()
# making the out folder if it doesn't exist
outFolder = utils.formatFolder(options.out, True)
# figuring out folder schema
gffFolder = utils.formatFolder(outFolder + 'gff/', True)
mappedFolder = utils.formatFolder(outFolder + 'mappedGFF/', True)
# GETTING INPUT FILE(s)
inputList = [
inputFile for inputFile in options.input.split(',')
if len(inputFile) > 1
]
#converting all input files into GFFs and moving into the GFF folder
inputGFFList = []
for inputFile in inputList:
if inputFile.split('.')[-1] == 'bed':
# CONVERTING A BED TO GFF
inputGFFName = inputFile.split('/')[-1][
0:-4] #strips the last 4 characters i.e. '.bed'
inputGFFFile = '%s%s.gff' % (gffFolder, inputGFFName)
utils.bedToGFF(inputFile, inputGFFFile)
elif options.input.split('.')[-1] == 'gff':
# COPY THE INPUT GFF TO THE GFF FOLDER
os.system('cp %s %s' % (inputFile, gffFolder))
inputGFFFile = '%s%s' % (gffFolder, inputFile.split('/')[-1])
else:
print(
'WARNING: INPUT FILE DOES NOT END IN .gff or .bed. ASSUMING .gff FILE FORMAT'
)
# COPY THE INPUT GFF TO THE GFF FOLDER
os.system('cp %s %s' % (inputFile, gffFolder))
inputGFFFile = '%s%s' % (gffFolder, inputFile.split('/')[-1])
inputGFFList.append(inputGFFFile)
# GETTING THE LIST OF BAMFILES TO PROCESS
#either same number of bams for rankby and control
#or only 1 control #or none!
#bamlist should be all rankby bams followed by control bams
bamFileList = []
if options.control:
controlBamList = [
bam for bam in options.control.split(',') if len(bam) > 0
]
rankbyBamList = [
bam for bam in options.rankby.split(',') if len(bam) > 0
]
if len(controlBamList) == len(rankbyBamList):
#case where an equal number of backgrounds are given
bamFileList = rankbyBamList + controlBamList
elif len(controlBamList) == 1:
#case where a universal background is applied
bamFileList = rankbyBamList + controlBamList * len(rankbyBamList)
else:
print(
'ERROR: EITHER PROVIDE A SINGLE CONTROL BAM FOR ALL SAMPLES, OR ONE CONTROL BAM FOR EACH SAMPLE'
)
sys.exit()
else:
bamFileList = [
bam for bam in options.rankby.split(',') if len(bam) > 0
]
# Stitch parameter
if options.stitch == '':
stitchWindow = ''
else:
stitchWindow = int(options.stitch)
# tss options
tssWindow = int(options.tss)
if tssWindow != 0:
removeTSS = True
else:
removeTSS = False
# GETTING THE GENOME
genome = string.upper(options.genome)
print('USING %s AS THE GENOME' % (genome))
# GETTING THE CORRECT ANNOT FILE
genomeDict = {
'HG18': '%s/annotation/hg18_refseq.ucsc' % (pipeline_dir),
'MM9': '%s/annotation/mm9_refseq.ucsc' % (pipeline_dir),
'HG19': '%s/annotation/hg19_refseq.ucsc' % (pipeline_dir),
'MM8': '%s/annotation/mm8_refseq.ucsc' % (pipeline_dir),
'MM10': '%s/annotation/mm10_refseq.ucsc' % (pipeline_dir),
'RN4': '%s/annotation/rn4_refseq.ucsc' % (pipeline_dir),
}
try:
annotFile = genomeDict[genome.upper()]
except KeyError:
print('ERROR: UNSUPPORTED GENOMES TYPE %s' % (genome))
sys.exit()
#FINDING THE ANALYSIS NAME
if options.name:
inputName = options.name
else:
inputName = inputGFFList[0].split('/')[-1].split('.')[0]
print('USING %s AS THE ANALYSIS NAME' % (inputName))
print('FORMATTING INPUT REGIONS')
# MAKING THE RAW INPUT FILE FROM THE INPUT GFFs
#use a simpler unique region naming system
if len(inputGFFList) == 1:
inputGFF = utils.parseTable(inputGFFList[0], '\t')
else:
inputLoci = []
for gffFile in inputGFFList:
print('\tprocessing %s' % (gffFile))
gff = utils.parseTable(gffFile, '\t')
gffCollection = utils.gffToLocusCollection(gff, 50)
inputLoci += gffCollection.getLoci()
inputCollection = utils.LocusCollection(inputLoci, 50)
inputCollection = inputCollection.stitchCollection(
) # stitches to produce unique regions
inputGFF = utils.locusCollectionToGFF(inputCollection)
formattedGFF = []
#now number things appropriately
for i, line in enumerate(inputGFF):
#use the coordinates to make a new id inputname_chr_sense_start_stop
chrom = line[0]
coords = [int(line[3]), int(line[4])]
sense = line[6]
lineID = '%s_%s' % (inputName, str(i + 1)) #1 indexing
newLine = [
chrom, lineID, lineID,
min(coords),
max(coords), '', sense, '', lineID
]
formattedGFF.append(newLine)
#name of the master input gff file
masterGFFFile = '%s%s_%s_ALL_-0_+0.gff' % (gffFolder, string.upper(genome),
inputName)
utils.unParseTable(formattedGFF, masterGFFFile, '\t')
print('USING %s AS THE INPUT GFF' % (masterGFFFile))
# MAKING THE START DICT
print('MAKING START DICT')
startDict = utils.makeStartDict(annotFile)
#GET CHROMS FOUND IN THE BAMS
print('GETTING CHROMS IN BAMFILES')
bamChromList = getBamChromList(bamFileList)
print("USING THE FOLLOWING CHROMS")
print(bamChromList)
#LOADING IN THE GFF AND FILTERING BY CHROM
print('LOADING AND FILTERING THE GFF')
inputGFF = filterGFF(masterGFFFile, bamChromList)
# LOADING IN THE BOUND REGION REFERENCE COLLECTION
print('LOADING IN GFF REGIONS')
referenceCollection = utils.gffToLocusCollection(inputGFF)
print('CHECKING REFERENCE COLLECTION:')
checkRefCollection(referenceCollection)
# MASKING REFERENCE COLLECTION
# see if there's a mask
if options.mask:
maskFile = options.mask
# if it's a bed file
if maskFile.split('.')[-1].upper() == 'BED':
maskGFF = utils.bedToGFF(maskFile)
elif maskFile.split('.')[-1].upper() == 'GFF':
maskGFF = utils.parseTable(maskFile, '\t')
else:
print("MASK MUST BE A .gff or .bed FILE")
sys.exit()
maskCollection = utils.gffToLocusCollection(maskGFF)
# now mask the reference loci
referenceLoci = referenceCollection.getLoci()
filteredLoci = [
locus for locus in referenceLoci
if len(maskCollection.getOverlap(locus, 'both')) == 0
]
print("FILTERED OUT %s LOCI THAT WERE MASKED IN %s" %
(len(referenceLoci) - len(filteredLoci), maskFile))
referenceCollection = utils.LocusCollection(filteredLoci, 50)
# NOW STITCH REGIONS
print('STITCHING REGIONS TOGETHER')
stitchedCollection, debugOutput, stitchWindow = regionStitching(
referenceCollection, inputName, outFolder, stitchWindow, tssWindow,
annotFile, removeTSS)
# NOW MAKE A STITCHED COLLECTION GFF
print('MAKING GFF FROM STITCHED COLLECTION')
stitchedGFF = utils.locusCollectionToGFF(stitchedCollection)
print(stitchWindow)
print(type(stitchWindow))
if not removeTSS:
stitchedGFFFile = '%s%s_%sKB_STITCHED.gff' % (gffFolder, inputName,
str(stitchWindow / 1000))
stitchedGFFName = '%s_%sKB_STITCHED' % (inputName,
str(stitchWindow / 1000))
debugOutFile = '%s%s_%sKB_STITCHED.debug' % (gffFolder, inputName,
str(stitchWindow / 1000))
else:
stitchedGFFFile = '%s%s_%sKB_STITCHED_TSS_DISTAL.gff' % (
gffFolder, inputName, str(stitchWindow / 1000))
stitchedGFFName = '%s_%sKB_STITCHED_TSS_DISTAL' % (
inputName, str(stitchWindow / 1000))
debugOutFile = '%s%s_%sKB_STITCHED_TSS_DISTAL.debug' % (
gffFolder, inputName, str(stitchWindow / 1000))
# WRITING DEBUG OUTPUT TO DISK
if debug:
print('WRITING DEBUG OUTPUT TO DISK AS %s' % (debugOutFile))
utils.unParseTable(debugOutput, debugOutFile, '\t')
# WRITE THE GFF TO DISK
print('WRITING STITCHED GFF TO DISK AS %s' % (stitchedGFFFile))
utils.unParseTable(stitchedGFF, stitchedGFFFile, '\t')
# SETTING UP THE OVERALL OUTPUT FILE
outputFile1 = outFolder + stitchedGFFName + '_ENHANCER_REGION_MAP.txt'
print('OUTPUT WILL BE WRITTEN TO %s' % (outputFile1))
# MAPPING TO THE NON STITCHED (ORIGINAL GFF)
# MAPPING TO THE STITCHED GFF
# Try to use the bamliquidatior_path.py script on cluster, otherwise, failover to local (in path), otherwise fail.
bamFileListUnique = list(bamFileList)
bamFileListUnique = utils.uniquify(bamFileListUnique)
#prevent redundant mapping
print("MAPPING TO THE FOLLOWING BAMS:")
print(bamFileListUnique)
for bamFile in bamFileListUnique:
bamFileName = bamFile.split('/')[-1]
# MAPPING TO THE STITCHED GFF
mappedOut1Folder = '%s%s_%s_MAPPED' % (mappedFolder, stitchedGFFName,
bamFileName)
mappedOut1File = '%s%s_%s_MAPPED/matrix.txt' % (
mappedFolder, stitchedGFFName, bamFileName)
if utils.checkOutput(mappedOut1File, 0.2, 0.2):
print("FOUND %s MAPPING DATA FOR BAM: %s" %
(stitchedGFFFile, mappedOut1File))
else:
cmd1 = bamliquidator_path + " --sense . -e 200 --match_bamToGFF -r %s -o %s %s" % (
stitchedGFFFile, mappedOut1Folder, bamFile)
print(cmd1)
os.system(cmd1)
if utils.checkOutput(mappedOut1File, 0.2, 5):
print("SUCCESSFULLY MAPPED TO %s FROM BAM: %s" %
(stitchedGFFFile, bamFileName))
else:
print("ERROR: FAILED TO MAP %s FROM BAM: %s" %
(stitchedGFFFile, bamFileName))
sys.exit()
print('BAM MAPPING COMPLETED NOW MAPPING DATA TO REGIONS')
# CALCULATE DENSITY BY REGION
# NEED TO FIX THIS FUNCTION TO ACCOUNT FOR DIFFERENT OUTPUTS OF LIQUIDATOR
mapCollection(stitchedCollection,
referenceCollection,
bamFileList,
mappedFolder,
outputFile1,
refName=stitchedGFFName)
print('FINDING AVERAGE SIGNAL AMONGST BAMS')
metaOutputFile = collapseRegionMap(outputFile1,
inputName + '_MERGED_SIGNAL',
controlBams=options.control)
#now try the merging
print('CALLING AND PLOTTING SUPER-ENHANCERS')
rankbyName = inputName + '_MERGED_SIGNAL'
controlName = 'NONE'
cmd = 'Rscript %sROSE2_callSuper.R %s %s %s %s' % (
pipeline_dir, outFolder, metaOutputFile, inputName, controlName)
print(cmd)
os.system(cmd)
# calling the gene mapper
print('CALLING GENE MAPPING')
superTableFile = "%s_SuperEnhancers.table.txt" % (inputName)
#for now don't use ranking bam to call top genes
cmd = "python %sROSE2_geneMapper.py -g %s -i %s%s -f" % (
pipeline_dir, genome, outFolder, superTableFile)
print(cmd)
os.system(cmd)
stretchTableFile = "%s_StretchEnhancers.table.txt" % (inputName)
cmd = "python %sROSE2_geneMapper.py -g %s -i %s%s -f" % (
pipeline_dir, genome, outFolder, stretchTableFile)
print(cmd)
os.system(cmd)
superStretchTableFile = "%s_SuperStretchEnhancers.table.txt" % (inputName)
cmd = "python %sROSE2_geneMapper.py -g %s -i %s%s -f" % (
pipeline_dir, genome, outFolder, superStretchTableFile)
os.system(cmd)
def make_shep_on_mycn_landscape(shep_on_dataFile):
'''
finds mycn peaks in shep21 that are conserved in nb and segregates them into promoter or enhancer
'''
dataDict = pipeline_dfci.loadDataTable(shep_on_dataFile)
print('LOADING SHEP ON MYCN SITES')
#load all of the shep_on sites
# shep_on_gff_path = '%smeta_rose/SHEP_ON_MYC/gff/HG19_SHEP_ON_MYC_ALL_-0_+0.gff' % (projectFolder)
# shep_on_gff = utils.parseTable(shep_on_gff_path,'\t')
shep_on_bed_path = '%sSHEP_6HR_MYCN_peaks.bed' % (macsEnrichedFolder)
shep_on_bed = utils.parseTable(shep_on_bed_path,'\t')
shep_on_gff = utils.bedToGFF(shep_on_bed)
#now get the conserved NB MYCN regions
nb_conserved_mycn_gff_file = '%sHG19_NB_MYCN_CONSERVED_-0_+0.gff' % (gffFolder)
nb_conserved_mycn_collection = utils.gffToLocusCollection(nb_conserved_mycn_gff_file)
print('LOADING SHEP ACTIVE ENHANCERS')
#make a collection of enhancers
shep_enhancer_file = '%smeta_rose/SHEP_ON_H3K27AC/SHEP_ON_H3K27AC_AllEnhancers.table.txt' % (projectFolder)
shep_enhancer_collection = utils.makeSECollection(shep_enhancer_file,'SHEP_H3K27AC')
#now get the active promoters
print('LOADING SHEP ACTIVE PROMOTERS')
startDict = utils.makeStartDict(annotFile)
shep_transcribed_file = '%sHG19_SHEP_ON_H3K27AC_ACTIVE.txt' % (geneListFolder)
shep_transcribed_table = utils.parseTable(shep_transcribed_file,'\t')
transcribedList = [line[1] for line in shep_transcribed_table]
tssLoci = []
for refID in transcribedList:
tssLoci.append(utils.makeTSSLocus(refID,startDict,1000,1000))
shep_tss_collection = utils.LocusCollection(tssLoci,50)
#now initialize the 6 gffs we will need
shep_mycn_gff = []
shep_mycn_gff_5kb = []
shep_mycn_gff_1kb = []
shep_mycn_promoter_gff = []
shep_mycn_promoter_gff_1kb = []
shep_mycn_promoter_gff_5kb = []
shep_mycn_enhancer_gff = []
shep_mycn_enhancer_gff_1kb = []
shep_mycn_enhancer_gff_5kb = []
#and their respective file names
shep_mycn_gff_file = '%sHG19_SHEP_MYCN_CONSERVED_-0_+0.gff' % (gffFolder)
shep_mycn_gff_5kb_file = '%sHG19_SHEP_MYCN_CONSERVED_-5kb_+5kb.gff' % (gffFolder)
shep_mycn_gff_1kb_file = '%sHG19_SHEP_MYCN_CONSERVED_-1kb_+1kb.gff' % (gffFolder)
shep_mycn_promoter_gff_file = '%sHG19_SHEP_MYCN_CONSERVED_PROMOTER_-0_+0.gff' % (gffFolder)
shep_mycn_promoter_gff_5kb_file = '%sHG19_SHEP_MYCN_CONSERVED_PROMOTER_-5kb_+5kb.gff' % (gffFolder)
shep_mycn_promoter_gff_1kb_file = '%sHG19_SHEP_MYCN_CONSERVED_PROMOTER_-1kb_+1kb.gff' % (gffFolder)
shep_mycn_enhancer_gff_file = '%sHG19_SHEP_MYCN_CONSERVED_ENHANCER_-0_+0.gff' % (gffFolder)
shep_mycn_enhancer_gff_5kb_file = '%sHG19_SHEP_MYCN_CONSERVED_ENHANCER_-5kb_+5kb.gff' % (gffFolder)
shep_mycn_enhancer_gff_1kb_file = '%sHG19_SHEP_MYCN_CONSERVED_ENHANCER_-1kb_+1kb.gff' % (gffFolder)
print('ITERATING THROUGH SHEP MYCN PEAKS')
ticker = 0
enhancer = 0
promoter = 0
other = 0
for line in shep_on_gff:
if ticker % 1000 == 0:
print ticker
ticker+=1
peakID = '%s_%s' % ('SHEP_MYCN',str(ticker))
lineLocus = utils.Locus(line[0],line[3],line[4],'.',peakID)
if nb_conserved_mycn_collection.getOverlap(lineLocus):
gffLine = [line[0],peakID,peakID,line[3],line[4],'','.','',peakID]
peakCenter = (int(line[3]) + int(line[4]))/2
gffLine_5kb = [line[0],peakID,peakID,peakCenter - 5000,peakCenter + 5000,'','.','',peakID]
#the 1kb is not a center +/- but a flank
gffLine_1kb = [line[0],peakID,peakID,int(line[3]) - 1000,int(line[4]) + 1000,'','.','',peakID]
shep_mycn_gff.append(gffLine)
shep_mycn_gff_5kb.append(gffLine_5kb)
shep_mycn_gff_1kb.append(gffLine_1kb)
#tss overlap should take precedence over enhancer overlap
if shep_tss_collection.getOverlap(lineLocus,'both'):
shep_mycn_promoter_gff.append(gffLine)
shep_mycn_promoter_gff_5kb.append(gffLine_5kb)
shep_mycn_promoter_gff_1kb.append(gffLine_1kb)
promoter+=1
#now check for enhancer overlap
elif shep_enhancer_collection.getOverlap(lineLocus,'both'):
shep_mycn_enhancer_gff.append(gffLine)
shep_mycn_enhancer_gff_5kb.append(gffLine_5kb)
shep_mycn_enhancer_gff_1kb.append(gffLine_1kb)
enhancer+=1
else:
other+=1
print('Of %s shep on mycn peaks' % (len(shep_on_gff)))
print('%s are promoter' % (promoter))
print('%s are enhancer' % (enhancer))
print('%s are other' % (other))
#now write out the gffs
utils.unParseTable(shep_mycn_gff,shep_mycn_gff_file,'\t')
utils.unParseTable(shep_mycn_gff_5kb,shep_mycn_gff_5kb_file,'\t')
utils.unParseTable(shep_mycn_gff_1kb,shep_mycn_gff_1kb_file,'\t')
utils.unParseTable(shep_mycn_promoter_gff,shep_mycn_promoter_gff_file,'\t')
utils.unParseTable(shep_mycn_promoter_gff_5kb,shep_mycn_promoter_gff_5kb_file,'\t')
utils.unParseTable(shep_mycn_promoter_gff_1kb,shep_mycn_promoter_gff_1kb_file,'\t')
utils.unParseTable(shep_mycn_enhancer_gff,shep_mycn_enhancer_gff_file,'\t')
utils.unParseTable(shep_mycn_enhancer_gff_5kb,shep_mycn_enhancer_gff_5kb_file,'\t')
utils.unParseTable(shep_mycn_enhancer_gff_1kb,shep_mycn_enhancer_gff_1kb_file,'\t')
def main():
'''
main run call
'''
debug = False
from optparse import OptionParser
usage = "usage: %prog [options] -g [GENOME] -i [INPUT_REGION_GFF] -r [RANKBY_BAM_FILE] -o [OUTPUT_FOLDER] [OPTIONAL_FLAGS]"
parser = OptionParser(usage=usage)
# required flags
parser.add_option("-i", "--i", dest="input", nargs=1, default=None,
help="Enter a comma separated list of .gff or .bed file of binding sites used to make enhancers")
parser.add_option("-r", "--rankby", dest="rankby", nargs=1, default=None,
help="Enter a comma separated list of bams to rank by")
parser.add_option("-o", "--out", dest="out", nargs=1, default=None,
help="Enter an output folder")
parser.add_option("-g", "--genome", dest="genome", nargs=1, default=None,
help="Enter the genome build (MM9,MM8,HG18,HG19)")
# optional flags
parser.add_option("-n", "--name", dest="name", nargs=1, default=None,
help="Provide a name for the analysis otherwise ROSE will guess")
parser.add_option("-c", "--control", dest="control", nargs=1, default=None,
help="Enter a comma separated list of control bams. Can either provide a single control bam for all rankby bams, or provide a control bam for each individual bam")
parser.add_option("-s", "--stitch", dest="stitch", nargs=1, default='',
help="Enter a max linking distance for stitching. Default will determine optimal stitching parameter")
parser.add_option("-t", "--tss", dest="tss", nargs=1, default=0,
help="Enter a distance from TSS to exclude. 0 = no TSS exclusion")
parser.add_option("--mask", dest="mask", nargs=1, default=None,
help="Mask a set of regions from analysis. Provide a .bed or .gff of masking regions")
# RETRIEVING FLAGS
(options, args) = parser.parse_args()
if not options.input or not options.rankby or not options.out or not options.genome:
print('hi there')
parser.print_help()
exit()
# making the out folder if it doesn't exist
outFolder = utils.formatFolder(options.out, True)
# figuring out folder schema
gffFolder = utils.formatFolder(outFolder + 'gff/', True)
mappedFolder = utils.formatFolder(outFolder + 'mappedGFF/', True)
# GETTING INPUT FILE(s)
inputList = [inputFile for inputFile in options.input.split(',') if len(inputFile) > 1]
#converting all input files into GFFs and moving into the GFF folder
inputGFFList = []
for inputFile in inputList:
if inputFile.split('.')[-1] == 'bed':
# CONVERTING A BED TO GFF
inputGFFName = inputFile.split('/')[-1][0:-4] #strips the last 4 characters i.e. '.bed'
inputGFFFile = '%s%s.gff' % (gffFolder, inputGFFName)
utils.bedToGFF(inputFile, inputGFFFile)
elif options.input.split('.')[-1] == 'gff':
# COPY THE INPUT GFF TO THE GFF FOLDER
os.system('cp %s %s' % (inputFile, gffFolder))
inputGFFFile = '%s%s' % (gffFolder,inputFile.split('/')[-1])
else:
print('WARNING: INPUT FILE DOES NOT END IN .gff or .bed. ASSUMING .gff FILE FORMAT')
# COPY THE INPUT GFF TO THE GFF FOLDER
os.system('cp %s %s' % (inputFile, gffFolder))
inputGFFFile = '%s%s' % (gffFolder,inputFile.split('/')[-1])
inputGFFList.append(inputGFFFile)
# GETTING THE LIST OF BAMFILES TO PROCESS
#either same number of bams for rankby and control
#or only 1 control #or none!
#bamlist should be all rankby bams followed by control bams
bamFileList = []
if options.control:
controlBamList = [bam for bam in options.control.split(',') if len(bam) >0]
rankbyBamList = [bam for bam in options.rankby.split(',') if len(bam) >0]
if len(controlBamList) == len(rankbyBamList):
#case where an equal number of backgrounds are given
bamFileList = rankbyBamList + controlBamList
elif len(controlBamList) == 1:
#case where a universal background is applied
bamFileList = rankbyBamList + controlBamList*len(rankbyBamList)
else:
print('ERROR: EITHER PROVIDE A SINGLE CONTROL BAM FOR ALL SAMPLES, OR ONE CONTROL BAM FOR EACH SAMPLE')
sys.exit()
else:
bamFileList = [bam for bam in options.rankby.split(',') if len(bam) > 0]
# Stitch parameter
if options.stitch == '':
stitchWindow = ''
else:
stitchWindow = int(options.stitch)
# tss options
tssWindow = int(options.tss)
if tssWindow != 0:
removeTSS = True
else:
removeTSS = False
# GETTING THE GENOME
genome = string.upper(options.genome)
print('USING %s AS THE GENOME' % (genome))
# GETTING THE CORRECT ANNOT FILE
genomeDict = {
'HG18': '%s/annotation/hg18_refseq.ucsc' % (pipeline_dir),
'MM9': '%s/annotation/mm9_refseq.ucsc' % (pipeline_dir),
'HG19': '%s/annotation/hg19_refseq.ucsc' % (pipeline_dir),
'MM8': '%s/annotation/mm8_refseq.ucsc' % (pipeline_dir),
'MM10': '%s/annotation/mm10_refseq.ucsc' % (pipeline_dir),
'RN4': '%s/annotation/rn4_refseq.ucsc' % (pipeline_dir),
}
try:
annotFile = genomeDict[genome.upper()]
except KeyError:
print('ERROR: UNSUPPORTED GENOMES TYPE %s' % (genome))
sys.exit()
#FINDING THE ANALYSIS NAME
if options.name:
inputName = options.name
else:
inputName = inputGFFList[0].split('/')[-1].split('.')[0]
print('USING %s AS THE ANALYSIS NAME' % (inputName))
print('FORMATTING INPUT REGIONS')
# MAKING THE RAW INPUT FILE FROM THE INPUT GFFs
#use a simpler unique region naming system
if len(inputGFFList) == 1:
inputGFF = utils.parseTable(inputGFFList[0],'\t')
else:
inputLoci = []
for gffFile in inputGFFList:
print('\tprocessing %s' % (gffFile))
gff = utils.parseTable(gffFile,'\t')
gffCollection = utils.gffToLocusCollection(gff,50)
inputLoci += gffCollection.getLoci()
inputCollection = utils.LocusCollection(inputLoci,50)
inputCollection = inputCollection.stitchCollection() # stitches to produce unique regions
inputGFF = utils.locusCollectionToGFF(inputCollection)
formattedGFF = []
#now number things appropriately
for i,line in enumerate(inputGFF):
#use the coordinates to make a new id inputname_chr_sense_start_stop
chrom = line[0]
coords = [int(line[3]) ,int(line[4])]
sense = line[6]
lineID = '%s_%s' % (inputName,str(i+1)) #1 indexing
newLine = [chrom,lineID,lineID,min(coords),max(coords),'',sense,'',lineID]
formattedGFF.append(newLine)
#name of the master input gff file
masterGFFFile = '%s%s_%s_ALL_-0_+0.gff' % (gffFolder,string.upper(genome),inputName)
utils.unParseTable(formattedGFF,masterGFFFile,'\t')
print('USING %s AS THE INPUT GFF' % (masterGFFFile))
# MAKING THE START DICT
print('MAKING START DICT')
startDict = utils.makeStartDict(annotFile)
#GET CHROMS FOUND IN THE BAMS
print('GETTING CHROMS IN BAMFILES')
bamChromList = getBamChromList(bamFileList)
print("USING THE FOLLOWING CHROMS")
print(bamChromList)
#LOADING IN THE GFF AND FILTERING BY CHROM
print('LOADING AND FILTERING THE GFF')
inputGFF = filterGFF(masterGFFFile,bamChromList)
# LOADING IN THE BOUND REGION REFERENCE COLLECTION
print('LOADING IN GFF REGIONS')
referenceCollection = utils.gffToLocusCollection(inputGFF)
print('CHECKING REFERENCE COLLECTION:')
checkRefCollection(referenceCollection)
# MASKING REFERENCE COLLECTION
# see if there's a mask
if options.mask:
maskFile = options.mask
# if it's a bed file
if maskFile.split('.')[-1].upper() == 'BED':
maskGFF = utils.bedToGFF(maskFile)
elif maskFile.split('.')[-1].upper() == 'GFF':
maskGFF = utils.parseTable(maskFile, '\t')
else:
print("MASK MUST BE A .gff or .bed FILE")
sys.exit()
maskCollection = utils.gffToLocusCollection(maskGFF)
# now mask the reference loci
referenceLoci = referenceCollection.getLoci()
filteredLoci = [locus for locus in referenceLoci if len(maskCollection.getOverlap(locus, 'both')) == 0]
print("FILTERED OUT %s LOCI THAT WERE MASKED IN %s" % (len(referenceLoci) - len(filteredLoci), maskFile))
referenceCollection = utils.LocusCollection(filteredLoci, 50)
# NOW STITCH REGIONS
print('STITCHING REGIONS TOGETHER')
stitchedCollection, debugOutput, stitchWindow = regionStitching(referenceCollection, inputName, outFolder, stitchWindow, tssWindow, annotFile, removeTSS)
# NOW MAKE A STITCHED COLLECTION GFF
print('MAKING GFF FROM STITCHED COLLECTION')
stitchedGFF = utils.locusCollectionToGFF(stitchedCollection)
print(stitchWindow)
print(type(stitchWindow))
if not removeTSS:
stitchedGFFFile = '%s%s_%sKB_STITCHED.gff' % (gffFolder, inputName, str(stitchWindow / 1000))
stitchedGFFName = '%s_%sKB_STITCHED' % (inputName, str(stitchWindow / 1000))
debugOutFile = '%s%s_%sKB_STITCHED.debug' % (gffFolder, inputName, str(stitchWindow / 1000))
else:
stitchedGFFFile = '%s%s_%sKB_STITCHED_TSS_DISTAL.gff' % (gffFolder, inputName, str(stitchWindow / 1000))
stitchedGFFName = '%s_%sKB_STITCHED_TSS_DISTAL' % (inputName, str(stitchWindow / 1000))
debugOutFile = '%s%s_%sKB_STITCHED_TSS_DISTAL.debug' % (gffFolder, inputName, str(stitchWindow / 1000))
# WRITING DEBUG OUTPUT TO DISK
if debug:
print('WRITING DEBUG OUTPUT TO DISK AS %s' % (debugOutFile))
utils.unParseTable(debugOutput, debugOutFile, '\t')
# WRITE THE GFF TO DISK
print('WRITING STITCHED GFF TO DISK AS %s' % (stitchedGFFFile))
utils.unParseTable(stitchedGFF, stitchedGFFFile, '\t')
# SETTING UP THE OVERALL OUTPUT FILE
outputFile1 = outFolder + stitchedGFFName + '_ENHANCER_REGION_MAP.txt'
print('OUTPUT WILL BE WRITTEN TO %s' % (outputFile1))
# MAPPING TO THE NON STITCHED (ORIGINAL GFF)
# MAPPING TO THE STITCHED GFF
# Try to use the bamliquidatior_path.py script on cluster, otherwise, failover to local (in path), otherwise fail.
bamFileListUnique = list(bamFileList)
bamFileListUnique = utils.uniquify(bamFileListUnique)
#prevent redundant mapping
print("MAPPING TO THE FOLLOWING BAMS:")
print(bamFileListUnique)
for bamFile in bamFileListUnique:
bamFileName = bamFile.split('/')[-1]
# MAPPING TO THE STITCHED GFF
mappedOut1Folder = '%s%s_%s_MAPPED' % (mappedFolder, stitchedGFFName, bamFileName)
mappedOut1File = '%s%s_%s_MAPPED/matrix.txt' % (mappedFolder, stitchedGFFName, bamFileName)
if utils.checkOutput(mappedOut1File, 0.2, 0.2):
print("FOUND %s MAPPING DATA FOR BAM: %s" % (stitchedGFFFile, mappedOut1File))
else:
cmd1 = bamliquidator_path + " --sense . -e 200 --match_bamToGFF -r %s -o %s %s" % (stitchedGFFFile, mappedOut1Folder, bamFile)
print(cmd1)
os.system(cmd1)
if utils.checkOutput(mappedOut1File,0.2,5):
print("SUCCESSFULLY MAPPED TO %s FROM BAM: %s" % (stitchedGFFFile, bamFileName))
else:
print("ERROR: FAILED TO MAP %s FROM BAM: %s" % (stitchedGFFFile, bamFileName))
sys.exit()
print('BAM MAPPING COMPLETED NOW MAPPING DATA TO REGIONS')
# CALCULATE DENSITY BY REGION
# NEED TO FIX THIS FUNCTION TO ACCOUNT FOR DIFFERENT OUTPUTS OF LIQUIDATOR
mapCollection(stitchedCollection, referenceCollection, bamFileList, mappedFolder, outputFile1, refName=stitchedGFFName)
print('FINDING AVERAGE SIGNAL AMONGST BAMS')
metaOutputFile = collapseRegionMap(outputFile1,inputName + '_MERGED_SIGNAL',controlBams=options.control)
#now try the merging
print('CALLING AND PLOTTING SUPER-ENHANCERS')
rankbyName = inputName + '_MERGED_SIGNAL'
controlName = 'NONE'
cmd = 'Rscript %sROSE2_callSuper.R %s %s %s %s' % (pipeline_dir,outFolder, metaOutputFile, inputName, controlName)
print(cmd)
os.system(cmd)
# calling the gene mapper
print('CALLING GENE MAPPING')
superTableFile = "%s_SuperEnhancers.table.txt" % (inputName)
#for now don't use ranking bam to call top genes
cmd = "python %sROSE2_geneMapper.py -g %s -i %s%s -f" % (pipeline_dir,genome, outFolder, superTableFile)
print(cmd)
os.system(cmd)
stretchTableFile = "%s_StretchEnhancers.table.txt" % (inputName)
cmd = "python %sROSE2_geneMapper.py -g %s -i %s%s -f" % (pipeline_dir,genome, outFolder, stretchTableFile)
print(cmd)
os.system(cmd)
superStretchTableFile = "%s_SuperStretchEnhancers.table.txt" % (inputName)
cmd = "python %sROSE2_geneMapper.py -g %s -i %s%s -f" % (pipeline_dir,genome, outFolder, superStretchTableFile)
os.system(cmd)
def main():
'''
main run call
'''
debug = False
from optparse import OptionParser
usage = "usage: %prog [options] -g [GENOME] -i [INPUT_REGION_GFF] -r [RANKBY_BAM_FILE] -o [OUTPUT_FOLDER] [OPTIONAL_FLAGS]"
parser = OptionParser(usage=usage)
# required flags
parser.add_option("-i", "--i", dest="input", nargs=1, default=None,
help="Enter a .gff or .bed file of binding sites used to make enhancers")
parser.add_option("-r", "--rankby", dest="rankby", nargs=1, default=None,
help="bamfile to rank enhancer by")
parser.add_option("-o", "--out", dest="out", nargs=1, default=None,
help="Enter an output folder")
parser.add_option("-g", "--genome", dest="genome", nargs=1, default=None,
help="Enter the genome build (MM9,MM8,HG18,HG19)")
# optional flags
parser.add_option("-b", "--bams", dest="bams", nargs=1, default=None,
help="Enter a comma separated list of additional bam files to map to")
parser.add_option("-c", "--control", dest="control", nargs=1, default=None,
help="bamfile to rank enhancer by")
parser.add_option("-s", "--stitch", dest="stitch", nargs=1, default='',
help="Enter a max linking distance for stitching. Default will determine optimal stitching parameter")
parser.add_option("-t", "--tss", dest="tss", nargs=1, default=0,
help="Enter a distance from TSS to exclude. 0 = no TSS exclusion")
parser.add_option("--mask", dest="mask", nargs=1, default=None,
help="Mask a set of regions from analysis. Provide a .bed or .gff of masking regions")
# RETRIEVING FLAGS
(options, args) = parser.parse_args()
if not options.input or not options.rankby or not options.out or not options.genome:
print('hi there')
parser.print_help()
exit()
# making the out folder if it doesn't exist
outFolder = utils.formatFolder(options.out, True)
# figuring out folder schema
gffFolder = utils.formatFolder(outFolder + 'gff/', True)
mappedFolder = utils.formatFolder(outFolder + 'mappedGFF/', True)
# GETTING INPUT FILE
if options.input.split('.')[-1] == 'bed':
# CONVERTING A BED TO GFF
inputGFFName = options.input.split('/')[-1][0:-4]
inputGFFFile = '%s%s.gff' % (gffFolder, inputGFFName)
utils.bedToGFF(options.input, inputGFFFile)
elif options.input.split('.')[-1] == 'gff':
# COPY THE INPUT GFF TO THE GFF FOLDER
inputGFFFile = options.input
os.system('cp %s %s' % (inputGFFFile, gffFolder))
else:
print('WARNING: INPUT FILE DOES NOT END IN .gff or .bed. ASSUMING .gff FILE FORMAT')
# COPY THE INPUT GFF TO THE GFF FOLDER
inputGFFFile = options.input
os.system('cp %s %s' % (inputGFFFile, gffFolder))
# GETTING THE LIST OF BAMFILES TO PROCESS
if options.control:
bamFileList = [options.rankby, options.control]
else:
bamFileList = [options.rankby]
if options.bams:
bamFileList += options.bams.split(',')
bamFileList = utils.uniquify(bamFileList)
# optional args
# Stitch parameter
if options.stitch == '':
stitchWindow = ''
else:
stitchWindow = int(options.stitch)
# tss options
tssWindow = int(options.tss)
if tssWindow != 0:
removeTSS = True
else:
removeTSS = False
# GETTING THE BOUND REGION FILE USED TO DEFINE ENHANCERS
print('USING %s AS THE INPUT GFF' % (inputGFFFile))
inputName = inputGFFFile.split('/')[-1].split('.')[0]
# GETTING THE GENOME
genome = options.genome
print('USING %s AS THE GENOME' % genome)
# GETTING THE CORRECT ANNOT FILE
cwd = os.getcwd()
genomeDict = {
'HG18': '%s/annotation/hg18_refseq.ucsc' % (cwd),
'MM9': '%s/annotation/mm9_refseq.ucsc' % (cwd),
'HG19': '%s/annotation/hg19_refseq.ucsc' % (cwd),
'MM8': '%s/annotation/mm8_refseq.ucsc' % (cwd),
'MM10': '%s/annotation/mm10_refseq.ucsc' % (cwd),
}
annotFile = genomeDict[genome.upper()]
# MAKING THE START DICT
print('MAKING START DICT')
startDict = utils.makeStartDict(annotFile)
# LOADING IN THE BOUND REGION REFERENCE COLLECTION
print('LOADING IN GFF REGIONS')
referenceCollection = utils.gffToLocusCollection(inputGFFFile)
# MASKING REFERENCE COLLECTION
# see if there's a mask
if options.mask:
maskFile = options.mask
# if it's a bed file
if maskFile.split('.')[-1].upper() == 'BED':
maskGFF = utils.bedToGFF(maskFile)
elif maskFile.split('.')[-1].upper() == 'GFF':
maskGFF = utils.parseTable(maskFile, '\t')
else:
print("MASK MUST BE A .gff or .bed FILE")
sys.exit()
maskCollection = utils.gffToLocusCollection(maskGFF)
# now mask the reference loci
referenceLoci = referenceCollection.getLoci()
filteredLoci = [locus for locus in referenceLoci if len(maskCollection.getOverlap(locus, 'both')) == 0]
print("FILTERED OUT %s LOCI THAT WERE MASKED IN %s" % (len(referenceLoci) - len(filteredLoci), maskFile))
referenceCollection = utils.LocusCollection(filteredLoci, 50)
# NOW STITCH REGIONS
print('STITCHING REGIONS TOGETHER')
stitchedCollection, debugOutput, stitchWindow = regionStitching(inputGFFFile, inputName, outFolder, stitchWindow, tssWindow, annotFile, removeTSS)
# NOW MAKE A STITCHED COLLECTION GFF
print('MAKING GFF FROM STITCHED COLLECTION')
stitchedGFF = utils.locusCollectionToGFF(stitchedCollection)
# making sure start/stop ordering are correct
for i in range(len(stitchedGFF)):
line = stitchedGFF[i]
start = int(line[3])
stop = int(line[4])
if start > stop:
line[3] = stop
line[4] = start
print(stitchWindow)
print(type(stitchWindow))
if not removeTSS:
stitchedGFFFile = '%s%s_%sKB_STITCHED.gff' % (gffFolder, inputName, str(stitchWindow / 1000))
stitchedGFFName = '%s_%sKB_STITCHED' % (inputName, str(stitchWindow / 1000))
debugOutFile = '%s%s_%sKB_STITCHED.debug' % (gffFolder, inputName, str(stitchWindow / 1000))
else:
stitchedGFFFile = '%s%s_%sKB_STITCHED_TSS_DISTAL.gff' % (gffFolder, inputName, str(stitchWindow / 1000))
stitchedGFFName = '%s_%sKB_STITCHED_TSS_DISTAL' % (inputName, str(stitchWindow / 1000))
debugOutFile = '%s%s_%sKB_STITCHED_TSS_DISTAL.debug' % (gffFolder, inputName, str(stitchWindow / 1000))
# WRITING DEBUG OUTPUT TO DISK
if debug:
print('WRITING DEBUG OUTPUT TO DISK AS %s' % (debugOutFile))
utils.unParseTable(debugOutput, debugOutFile, '\t')
# WRITE THE GFF TO DISK
print('WRITING STITCHED GFF TO DISK AS %s' % (stitchedGFFFile))
utils.unParseTable(stitchedGFF, stitchedGFFFile, '\t')
# SETTING UP THE OVERALL OUTPUT FILE
outputFile1 = outFolder + stitchedGFFName + '_ENHANCER_REGION_MAP.txt'
print('OUTPUT WILL BE WRITTEN TO %s' % (outputFile1))
# MAPPING TO THE NON STITCHED (ORIGINAL GFF)
# MAPPING TO THE STITCHED GFF
# Try to use the bamliquidatior_path.py script on cluster, otherwise, failover to local (in path), otherwise fail.
bamliquidator_path = '/ark/home/jdm/pipeline/bamliquidator_batch.py'
if not os.path.isfile(bamliquidator_path):
bamliquidator_path = 'bamliquidator_batch.py'
if not os.path.isfile(bamliquidator_path):
raise ValueError('bamliquidator_batch.py not found in path')
for bamFile in bamFileList:
bamFileName = bamFile.split('/')[-1]
# MAPPING TO THE STITCHED GFF
mappedOut1Folder = '%s%s_%s_MAPPED' % (mappedFolder, stitchedGFFName, bamFileName)
mappedOut1File = '%s%s_%s_MAPPED/matrix.gff' % (mappedFolder, stitchedGFFName, bamFileName)
if utils.checkOutput(mappedOut1File, 0.2, 0.2):
print("FOUND %s MAPPING DATA FOR BAM: %s" % (stitchedGFFFile, mappedOut1File))
else:
cmd1 = "python " + bamliquidator_path + " --sense . -e 200 --match_bamToGFF -r %s -o %s %s" % (stitchedGFFFile, mappedOut1Folder, bamFile)
print(cmd1)
output1 = subprocess.Popen(cmd1, stdout=subprocess.PIPE, shell=True)
output1 = output1.communicate()
if len(output1[0]) > 0: # test if mapping worked correctly
print("SUCCESSFULLY MAPPED TO %s FROM BAM: %s" % (stitchedGFFFile, bamFileName))
else:
print("ERROR: FAILED TO MAP %s FROM BAM: %s" % (stitchedGFFFile, bamFileName))
sys.exit()
# MAPPING TO THE ORIGINAL GFF
mappedOut2Folder = '%s%s_%s_MAPPED' % (mappedFolder, inputName, bamFileName)
mappedOut2File = '%s%s_%s_MAPPED/matrix.gff' % (mappedFolder, inputName, bamFileName)
if utils.checkOutput(mappedOut2File, 0.2, 0.2):
print("FOUND %s MAPPING DATA FOR BAM: %s" % (stitchedGFFFile, mappedOut2File))
else:
cmd2 = "python " + bamliquidator_path + " --sense . -e 200 --match_bamToGFF -r %s -o %s %s" % (inputGFFFile, mappedOut2Folder, bamFile)
print(cmd2)
output2 = subprocess.Popen(cmd2, stdout=subprocess.PIPE, shell=True)
output2 = output2.communicate()
if len(output2[0]) > 0: # test if mapping worked correctly
print("SUCCESSFULLY MAPPED TO %s FROM BAM: %s" % (inputGFFFile, bamFileName))
else:
print("ERROR: FAILED TO MAP %s FROM BAM: %s" % (inputGFFFile, bamFileName))
sys.exit()
print('BAM MAPPING COMPLETED NOW MAPPING DATA TO REGIONS')
# CALCULATE DENSITY BY REGION
# NEED TO FIX THIS FUNCTION TO ACCOUNT FOR DIFFERENT OUTPUTS OF LIQUIDATOR
mapCollection(stitchedCollection, referenceCollection, bamFileList, mappedFolder, outputFile1, refName=stitchedGFFName)
print('CALLING AND PLOTTING SUPER-ENHANCERS')
if options.control:
rankbyName = options.rankby.split('/')[-1]
controlName = options.control.split('/')[-1]
cmd = 'R --no-save %s %s %s %s < ROSE2_callSuper.R' % (outFolder, outputFile1, inputName, controlName)
else:
rankbyName = options.rankby.split('/')[-1]
controlName = 'NONE'
cmd = 'R --no-save %s %s %s %s < ROSE2_callSuper.R' % (outFolder, outputFile1, inputName, controlName)
print(cmd)
os.system(cmd)
# calling the gene mapper
time.sleep(20)
superTableFile = "%s_SuperEnhancers.table.txt" % (inputName)
if options.control:
cmd = "python ROSE2_geneMapper.py -g %s -r %s -c %s -i %s%s" % (genome, options.rankby, options.control, outFolder, superTableFile)
else:
cmd = "python ROSE2_geneMapper.py -g %s -r %s -i %s%s" % (genome, options.rankby, outFolder, superTableFile)
os.system(cmd)
stretchTableFile = "%s_StretchEnhancers.table.txt" % (inputName)
if options.control:
cmd = "python ROSE2_geneMapper.py -g %s -r %s -c %s -i %s%s" % (genome, options.rankby, options.control, outFolder, stretchTableFile)
else:
cmd = "python ROSE2_geneMapper.py -g %s -r %s -i %s%s" % (genome, options.rankby, outFolder, stretchTableFile)
os.system(cmd)
superStretchTableFile = "%s_SuperStretchEnhancers.table.txt" % (inputName)
if options.control:
cmd = "python ROSE2_geneMapper.py -g %s -r %s -c %s -i %s%s" % (genome, options.rankby, options.control, outFolder, superStretchTableFile)
else:
cmd = "python ROSE2_geneMapper.py -g %s -r %s -i %s%s" % (genome, options.rankby, outFolder, superStretchTableFile)
os.system(cmd)
def regionStitching(inputGFF, name, outFolder, stitchWindow, tssWindow, annotFile, removeTSS=True):
print('PERFORMING REGION STITCHING')
# first have to turn bound region file into a locus collection
# need to make sure this names correctly... each region should have a unique name
boundCollection = utils.gffToLocusCollection(inputGFF)
debugOutput = []
# filter out all bound regions that overlap the TSS of an ACTIVE GENE
if removeTSS:
print('REMOVING TSS FROM REGIONS USING AN EXCLUSION WINDOW OF %sBP' % (tssWindow))
# first make a locus collection of TSS
startDict = utils.makeStartDict(annotFile)
# now makeTSS loci for active genes
removeTicker = 0
# this loop makes a locus centered around +/- tssWindow of transcribed genes
# then adds it to the list tssLoci
tssLoci = []
for geneID in startDict.keys():
tssLoci.append(utils.makeTSSLocus(geneID, startDict, tssWindow, tssWindow))
# this turns the tssLoci list into a LocusCollection
# 50 is the internal parameter for LocusCollection and doesn't really matter
tssCollection = utils.LocusCollection(tssLoci, 50)
# gives all the loci in boundCollection
boundLoci = boundCollection.getLoci()
# this loop will check if each bound region is contained by the TSS exclusion zone
# this will drop out a lot of the promoter only regions that are tiny
# typical exclusion window is around 2kb
for locus in boundLoci:
if len(tssCollection.getContainers(locus, 'both')) > 0:
# if true, the bound locus overlaps an active gene
boundCollection.remove(locus)
debugOutput.append([locus.__str__(), locus.ID(), 'CONTAINED'])
removeTicker += 1
print('REMOVED %s LOCI BECAUSE THEY WERE CONTAINED BY A TSS' % (removeTicker))
# boundCollection is now all enriched region loci that don't overlap an active TSS
if stitchWindow == '':
print('DETERMINING OPTIMUM STITCHING PARAMTER')
optCollection = copy.deepcopy(boundCollection)
stitchWindow = optimizeStitching(optCollection, name, outFolder, stepSize=500)
print('USING A STITCHING PARAMETER OF %s' % stitchWindow)
stitchedCollection = boundCollection.stitchCollection(stitchWindow, 'both')
if removeTSS:
# now replace any stitched region that overlap 2 distinct genes
# with the original loci that were there
fixedLoci = []
tssLoci = []
for geneID in startDict.keys():
tssLoci.append(utils.makeTSSLocus(geneID, startDict, 50, 50))
# this turns the tssLoci list into a LocusCollection
# 50 is the internal parameter for LocusCollection and doesn't really matter
tssCollection = utils.LocusCollection(tssLoci, 50)
removeTicker = 0
originalTicker = 0
for stitchedLocus in stitchedCollection.getLoci():
overlappingTSSLoci = tssCollection.getOverlap(stitchedLocus, 'both')
tssNames = [startDict[tssLocus.ID()]['name'] for tssLocus in overlappingTSSLoci]
tssNames = utils.uniquify(tssNames)
if len(tssNames) > 2:
# stitchedCollection.remove(stitchedLocus)
originalLoci = boundCollection.getOverlap(stitchedLocus, 'both')
originalTicker += len(originalLoci)
fixedLoci += originalLoci
debugOutput.append([stitchedLocus.__str__(), stitchedLocus.ID(), 'MULTIPLE_TSS'])
removeTicker += 1
else:
fixedLoci.append(stitchedLocus)
print('REMOVED %s STITCHED LOCI BECAUSE THEY OVERLAPPED MULTIPLE TSSs' % (removeTicker))
print('ADDED BACK %s ORIGINAL LOCI' % (originalTicker))
fixedCollection = utils.LocusCollection(fixedLoci, 50)
return fixedCollection, debugOutput, stitchWindow
else:
return stitchedCollection, debugOutput, stitchWindow
def rank_eboxes(nb_all_chip_dataFile,mycn_gff_path,macsFolder,genomeDirectory,window = 100):
'''
uses the conserved MYCN sites and ranks eboxes within them
by average background subtracted signal
searches 100bp (window variable) from mycn summits
'''
window = int(window)
#bring in the conserved mycn region
print('making gff of nb mycn summits')
nb_mycn_gff = utils.parseTable(mycn_gff_path,'\t')
nb_mycn_collection = utils.gffToLocusCollection(nb_mycn_gff,50)
dataDict =pipeline_dfci.loadDataTable(nb_all_chip_dataFile)
names_list = [name for name in dataDict.keys() if name.count('MYCN') == 1]
names_list.sort()
summit_loci = []
#first makes a gff of all summits +/- 100bp for all nb mycn datasets
for name in names_list:
summit_bed_path = '%s%s/%s_summits.bed' % (macsFolder,name,name)
summit_bed = utils.parseTable(summit_bed_path,'\t')
for line in summit_bed:
summit_locus = utils.Locus(line[0],int(line[1])-window,int(line[2])+window,'.',line[3])
if len(nb_mycn_collection.getOverlap(summit_locus)) > 0:
summit_loci.append(summit_locus)
summit_collection =utils.LocusCollection(summit_loci,50)
summit_merged_collection = summit_collection.stitchCollection()
summit_gff = utils.locusCollectionToGFF(summit_merged_collection)
summit_gff_path = '%sHG19_NB_MYCN_SUMMITS_-%s_+%s.gff' % (gffFolder,window,window)
utils.unParseTable(summit_gff,summit_gff_path,'\t')
#this is borrowed from above and maps chip-seq signal to the gff
print('mapping to nb mycn summits and making signal dict')
gffList = [summit_gff_path]
summit_signal_path = pipeline_dfci.map_regions(nb_all_chip_dataFile,gffList)
mycnSignalTable = utils.parseTable(summit_signal_path,'\t')
#making a signal dictionary for MYCN binding
names_list = ['BE2C_MYCN','KELLY_MYCN','NGP_MYCN','SHEP21_0HR_MYCN_NOSPIKE']
background_list = [dataDict[name]['background'] for name in names_list]
header = mycnSignalTable[0]
chip_columns = [header.index(name) for name in names_list]
background_columns = [header.index(background_name) for background_name in background_list]
mycn_sig_dict = {}
for line in mycnSignalTable[1:]:
line_sig = []
for i in range(len(names_list)):
line_sig.append(float(line[chip_columns[i]]) - float(line[background_columns[i]]))
region_id = line[1]
coords = [int(x) for x in line[1].split(':')[-1].split('-')]
line_length = coords[1]-coords[0]
mycn_sig_dict[region_id] = numpy.mean(line_sig)*line_length
#now for each region find the eboxes and then add up the signal
print('making ebox ranking')
ebox_list = ['CACGTG','CAGTTG','CAAGTG','CAGGTG','CAATTG','CAAATG','CATCTG','CAGCTG','CATGTG','CATATG']
eboxDict = {}
for ebox in ebox_list:
eboxDict[ebox] = []
ticker = 0
for line in summit_gff:
if ticker % 1000 == 0:
print(ticker)
ticker+=1
chrom = line[0]
sense = '.'
start = int(line[3])
end = int(line[4])
region_id = '%s(%s):%s-%s' % (line[0],line[6],line[3],line[4])
signal = mycn_sig_dict[region_id]
sequenceLine = utils.fetchSeq(genomeDirectory,chrom,start,end,True)
motifVector = []
matches = re.finditer('CA..TG',str.upper(sequenceLine))
if matches:
for match in matches:
motifVector.append(match.group())
#count only 1 of each motif type per line
#motifVector = utils.uniquify(motifVector)
for motif in motifVector:
if ebox_list.count(motif) > 0:
eboxDict[motif].append(signal)
else:
eboxDict[utils.revComp(motif)].append(signal)
eboxTable =[]
eboxTableOrdered =[['EBOX','OCCURENCES','AVG_HEIGHT']]
for ebox in eboxDict.keys():
newLine = [ebox,len(eboxDict[ebox]),numpy.mean(eboxDict[ebox])]
eboxTable.append(newLine)
occurenceOrder = utils.order([line[2] for line in eboxTable],decreasing=True)
for x in occurenceOrder:
eboxTableOrdered.append(eboxTable[x])
print(eboxTableOrdered)
ebox_outfile = '%sHG19_NB_MYCN_CONSERVED_SUMMITS_-%s_+%s_EBOX_RANK.txt' % (tableFolder,window,window)
utils.unParseTable(eboxTableOrdered,ebox_outfile,'\t')
return ebox_outfile
def findCanidateTFs(genome, enhancer_gff, expressedNM, expressionDictNM,
bamFile, TFlist, refseqToNameDict, projectFolder, projectName, promoter):
'''
Assign each Super-Enhancer to the closest active TSS to its center
Return a dictionary keyed by TF that points to a list of loci
'''
#loading in the enhancer gff regions
enhancer_collection = utils.gffToLocusCollection(enhancer_gff)
enhancer_loci = enhancer_collection.getLoci()
#loading in the genome and TF info
annot_file = genome.returnFeature('annot_file')
startDict = utils.makeStartDict(annot_file)
tf_table = utils.parseTable(genome.returnFeature('tf_file'),'\t')
refID_list = [line[0] for line in tf_table] #creates a list of all NM IDs for TFs
#make a collection of all TF TSSs
tssLoci = []
for refID in refID_list:
tssLoci.append(utils.makeTSSLocus(refID,startDict,0,0)) #this is a precise 1 coordinate TSS locus
tssCollection = utils.LocusCollection(tssLoci,50)
enhancerTable = [['ENHANCER_ID','CHROM','START','STOP','GENE_LIST']]
gene_to_enhancer_dict = defaultdict(list)
# Loop through enhancers
#all gene nnames stored by refID
for enhancer in enhancer_loci:
# If the enhancer overlaps a TSS, save it
overlapping_loci = tssCollection.getOverlap(enhancer, 'both')
overlapping_refIDs =[locus.ID() for locus in overlapping_loci]
# Find all gene TSS within 100 kb
proximal_loci = tssCollection.getOverlap(utils.makeSearchLocus(enhancer,100000,100000),'both')
proximal_refIDs =[locus.ID() for locus in proximal_loci]
# If no genes are within 100 kb, find the closest active gene within 1 million bp
closest_refID = []
if len(overlapping_refIDs) == 0 and len(proximal_refIDs) == 0:
distal_loci = tssCollection.getOverlap(utils.makeSearchLocus(enhancer,1000000,1000000),'both')
distal_refIDs =[locus.ID() for locus in distal_loci]
enhancerCenter = (int(enhancer.start()) + int(enhancer.end())) / 2
distance_list = [abs(enhancerCenter - startDict[geneID]['start'][0])
for geneID in distal_refIDs]
if len(distance_list) > 0:
closest_refID = [distalGenes[distance_list.index(min(distance_list))]]
#now we have all potential gene cases
all_refIDs = overlappingGenes + proximalGenes + closest_refID
#now we get all names and refIDs
all_refIDs = utils.uniquify([refID for refID in all_refIDs if len(refID) > 0 ])
all_names = utils.uniquify([startDict[refID]['name'] for refID in all_refIDs])
#first do enhancer level assignment
names_string = ','.join(all_names)
enhancer_table.append([enhancer.ID(),enhancer.chr(),enhancer.start(),enhancer.end(),names_string])
#now do gene level assignment
for refID in all_refIDs:
gene_to_enhancer_dict[refID].append(enhancer.ID())
#an enhancer can be assigned to multiple genes
#a promoter can only be assigned to 1 gene
#promoters don't have enhancerIDs so don't add them yet
#this should just be an enhancer level table
#followed by a gene level table
overlappingGenes = utils.uniquify(overlappingGenes)
proximalGenes = utils.uniquify(proximalGenes)
for refID in overlappingGenes:
if proximalGenes.count(refID) == 1:
proximalGenes.remove(refID)
# If a TSS overlaps an enhancer, assign them together
if overlappingGenes:
for gene in overlappingGenes:
if gene in tf_list:
TFtoEnhancerDict[gene].append(enhancer)
enhancerAssignment.append([gene, enhancer.chr(), enhancer.start(), enhancer.end(), enhancer.ID()])
# Otherwise, assign the enhancer to the most active gene in 100 kb
elif not overlappingGenes and proximalGenes:
highestGene = ''
highestActivity = 0
for gene in proximalGenes:
if expressionDictNM[gene] > highestActivity:
highestActivity = expressionDictNM[gene]
highestGene = gene
if highestGene in TFlist:
TFtoEnhancerDict[gene].append(enhancer)
enhancerAssignment.append([gene, enhancer.chr(), enhancer.start(), enhancer.end(), enhancer.ID()])
elif not overlappingGenes and not proximalGenes and closestGene:
if closestGene in TFlist:
gene = closestGene
TFtoEnhancerDict[gene].append(enhancer)
enhancerAssignment.append([gene, enhancer.chr(), enhancer.start(), enhancer.end(), enhancer.ID()])
# Add promoter is it's not contained in the super
if promoter:
for gene in TFtoEnhancerDict.keys():
promoter = utils.Locus(startDict[gene]['chr'], int(startDict[gene]['start'][0]) - 2000,
int(startDict[gene]['start'][0]) + 2000, startDict[gene]['sense'])
overlapBool = False
for enhancer in TFtoEnhancerDict[gene]:
if promoter.overlaps(enhancer):
overlapBool = True
if not overlapBool:
TFtoEnhancerDict[gene].append(promoter)
seAssignmentFile = projectFolder + projectName + '_ENHANCER_ASSIGNMENT.txt'
utils.unParseTable(enhancerAssignment, seAssignmentFile, '\t')
return TFtoEnhancerDict
def main():
print('main analysis for MYCN project')
print('changing directory to project folder')
os.chdir(projectFolder)
print('\n\n')
print(
'#======================================================================'
)
print(
'#======================I, LOADING DATA ANNOTATION======================'
)
print(
'#======================================================================'
)
print('\n\n')
#This section sanity checks each data table and makes sure both bam and .bai files are accessible
#for ChIP-Seq
pipeline_dfci.summary(mouse_dataFile)
print('\n\n')
print(
'#======================================================================'
)
print(
'#==========================II. CALLING MACS============================'
)
print(
'#======================================================================'
)
print('\n\n')
#running peak finding using macs 1.4.2 on all chip datasets
#this usually takes ~2-3 hours on a reasonably fast machine
#a 3 hour time out on this entire operation is set
#if peak calling takes longer than 3 hours, simply run the script again after completion
#run_macs(mouse_dataFile)
print('\n\n')
print(
'#======================================================================'
)
print(
'#=================II. DEFINING ACTIVE GENES IN MOUSE==================='
)
print(
'#======================================================================'
)
print('\n\n')
#here we will identify active promoters in various contexts as those with
#an H3K27AC peak in the +/- 1kb tss region
#UCSC refseq annotations are used for all genes
#make_active_gene_lists(mouse_dataFile)
print('\n\n')
print(
'#======================================================================'
)
print(
'#==================III. CALLING ROSE TO MAP ENHANCERS=================='
)
print(
'#======================================================================'
)
print('\n\n')
# #for SCG_H3K27AC
# analysisName = 'SCG_H3K27AC'
# namesList = ['SCG_H3K27Ac']
# bashFileName,region_map_path,namesList=define_enhancer_landscape(mouse_dataFile,analysisName,namesList)
# #for CG_H3K27AC
# analysisName = 'CG_H3K27AC'
# namesList = ['CG_H3K27Ac']
# bashFileName,region_map_path,namesList=define_enhancer_landscape(mouse_dataFile,analysisName,namesList)
# #for GANGLIA_H3K27AC
# analysisName = 'GANGLIA_H3K27AC'
# namesList = ['CG_H3K27Ac','SCG_H3K27Ac']
# bashFileName,region_map_path,namesList=define_enhancer_landscape(mouse_dataFile,analysisName,namesList)
# #for THMYCN
# analysisName = 'THMYCN_H3K27AC'
# namesList = ['THMYCN_139076_H3K27Ac','THMYCN_139423_H3K27Ac','THMYCN1_H3K27Ac']
# bashFileName,region_map_path,namesList=define_enhancer_landscape(mouse_dataFile,analysisName,namesList)
print('\n\n')
print(
'#======================================================================'
)
print(
'#=================IV. LIFTING OVER NB CONSERVED REGIONS================'
)
print(
'#======================================================================'
)
print('\n\n')
# #liftover a pair of gffs
# #first convert to bed
# nb_promoter_gff_path = '%sgff/HG19_NB_MYCN_CONSERVED_PROMOTER_-5000_+5000.gff' % (hg19_projectFolder)
# nb_enhancer_gff_path = '%sgff/HG19_NB_MYCN_CONSERVED_ENHANCER_-5000_+5000.gff' % (hg19_projectFolder)
# nb_promoter_bed_path ='%sbeds/HG19_NB_MYCN_CONSERVED_PROMOTER_-5000_+5000.bed' % (hg19_projectFolder)
# nb_enhancer_bed_path ='%sbeds/HG19_NB_MYCN_CONSERVED_ENHANCER_-5000_+5000.bed' % (hg19_projectFolder)
# nb_promoter_gff = utils.parseTable(nb_promoter_gff_path,'\t')
# nb_enhancer_gff = utils.parseTable(nb_enhancer_gff_path,'\t')
# utils.gffToBed(nb_promoter_gff,nb_promoter_bed_path)
# utils.gffToBed(nb_enhancer_gff,nb_enhancer_bed_path)
# print('converted NB conserved gffs to beds at %s and %s' % (nb_promoter_bed_path,nb_enhancer_bed_path))
# #note, now you have to liftover manually to create beds
# mm9_promoter_bed_path = '%sMM9_NB_MYCN_CONSERVED_PROMOTER_-5000_+5000.bed' % (bedFolder)
# mm9_enhancer_bed_path = '%sMM9_NB_MYCN_CONSERVED_ENHANCER_-5000_+5000.bed' % (bedFolder)
# mm9_promoter_gff_path = '%sMM9_NB_MYCN_CONSERVED_PROMOTER_-5000_+5000.gff' % (gffFolder)
# mm9_enhancer_gff_path = '%sMM9_NB_MYCN_CONSERVED_ENHANCER_-5000_+5000.gff' % (gffFolder)
# utils.bedToGFF(mm9_promoter_bed_path,mm9_promoter_gff_path)
# utils.bedToGFF(mm9_enhancer_bed_path,mm9_enhancer_gff_path)
# print('writing mm9 nb mycn sites to %s and %s' % (mm9_promoter_gff_path,mm9_enhancer_gff_path))
print('\n\n')
print(
'#======================================================================'
)
print(
'#======================V. MAPPING ENRICHED TO GFFS====================='
)
print(
'#======================================================================'
)
print('\n\n')
# setName = 'THMYCN'
# gffList = [mm9_promoter_gff_path,mm9_enhancer_gff_path]
# cellTypeList = ['THMYCN1','THMYCN2','THMYCN','CG','SCG']
# mapList = ['CG_H3K27Ac',
# 'SCG_H3K27Ac',
# 'THMYCN1_H3K27Ac',
# 'THMYCN_139423_H3K27Ac',
# 'THMYCN_139076_H3K27Ac',
# ]
# #pipeline_dfci.mapEnrichedToGFF(mouse_dataFile,setName,gffList,cellTypeList,macsEnrichedFolder,mappedEnrichedFolder,macs=True,namesList=mapList,useBackground=True)
# #summarize info for venn diagrams for each
# promoter_mapped_path = '%sMM9_NB_MYCN_CONSERVED_PROMOTER_-5000_+5000/MM9_NB_MYCN_CONSERVED_PROMOTER_-5000_+5000_THMYCN.txt' % (mappedEnrichedFolder)
# promoter_venn_path = '%sMM9_NB_MYCN_CONSERVED_PROMOTER_-5000_+5000_VENN.txt' % (tableFolder)
# summarizeVenn(promoter_mapped_path,group_list = ['CG','THMYCN'],output=promoter_venn_path)
# enhancer_mapped_path = '%sMM9_NB_MYCN_CONSERVED_ENHANCER_-5000_+5000/MM9_NB_MYCN_CONSERVED_ENHANCER_-5000_+5000_THMYCN.txt' % (mappedEnrichedFolder)
# enhancer_venn_path = '%sMM9_NB_MYCN_CONSERVED_ENHANCER_-5000_+5000_VENN.txt' % (tableFolder)
# summarizeVenn(enhancer_mapped_path,group_list = ['CG','THMYCN'],output=enhancer_venn_path)
print('\n\n')
print(
'#======================================================================'
)
print(
'#=====================VI. MAKING MYCN REGIONS GFF======================'
)
print(
'#======================================================================'
)
print('\n\n')
dataDict = pipeline_dfci.loadDataTable(mouse_dataFile)
names_list = [
'THMYCN2_MYCN',
'THMYCN_139076_MYCN',
'THMYCN_139423_MYCN',
]
mycn_loci = []
for name in names_list:
mycn_collection = utils.importBoundRegion(
'%s%s' % (macsEnrichedFolder, dataDict[name]['enrichedMacs']),
name)
mycn_loci += mycn_collection.getLoci()
mycn_collection = utils.LocusCollection(mycn_loci, 50)
mycn_collection.stitchCollection()
mycn_gff = utils.locusCollectionToGFF(mycn_collection)
mycn_gff_path = '%sMM9_THMYCN_MYCN_-0_+0.gff' % (gffFolder)
utils.unParseTable(mycn_gff, mycn_gff_path, '\t')
#make collections
promoter_collection = utils.gffToLocusCollection(
'%sMM9_NB_MYCN_CONSERVED_PROMOTER_-5000_+5000.gff' % (gffFolder))
enhancer_collection = utils.gffToLocusCollection(
'%sMM9_NB_MYCN_CONSERVED_ENHANCER_-5000_+5000.gff' % (gffFolder))
#make the overlap table
overlap_table = [['PROMOTER', 'ENHANCER', 'NONE']]
promoter_count = 0
enhancer_count = 0
none_count = 0
for line in mycn_gff:
locus = utils.Locus(line[0],
int(line[3]) - 10000,
int(line[4]) + 10000, '.')
if enhancer_collection.getOverlap(locus, 'both'):
enhancer_count += 1
continue
if promoter_collection.getOverlap(locus, 'both'):
promoter_count += 1
else:
none_count += 1
overlap_table.append([promoter_count, enhancer_count, none_count])
overlap_table_path = '%sMM9_THMYCN_OVERLAP.txt' % (tableFolder)
utils.unParseTable(overlap_table, overlap_table_path, '\t')
print('\n\n')
print(
'#======================================================================'
)
print(
'#=====================VI. MAPPING GFFS FOR HEATMAP====================='
)
print(
'#======================================================================'
)
print('\n\n')
#map_for_heatmap(mouse_dataFile)
print('\n\n')
print(
'#======================================================================'
)
print(
'#=====================VII. AVERAGING MAPPED SIGNAL====================='
)
print(
'#======================================================================'
)
print('\n\n')
# set_list = ['GANGLIA_H3K27AC','THMYCN_H3K27AC','THMYCN_MYCN']
# set_names = [
# ['CG_H3K27Ac','SCG_H3K27Ac'],
# ['THMYCN1_H3K27Ac','THMYCN_139423_H3K27Ac','THMYCN_139076_H3K27Ac'],
# ['THMYCN2_MYCN','THMYCN_139076_MYCN','THMYCN_139423_MYCN']
# ]
# for i in range(len(set_list)):
# setName = set_list[i]
# names_list =set_names[i]
# print(setName)
# print(names_list)
# #for promoters
# mapped_list = ['%sMM9_NB_MYCN_CONSERVED_PROMOTER_-5000_+5000/MM9_NB_MYCN_CONSERVED_PROMOTER_-5000_+5000_%s.gff' % (mappedFolder,name) for name in names_list]
# output_path = '%sMM9_NB_MYCN_CONSERVED_PROMOTER_-5000_+5000/MM9_NB_MYCN_CONSERVED_PROMOTER_-5000_+5000_%s.gff' % (mappedFolder,setName)
# print(output_path)
# averagingMappedSignal(mapped_list,output_path,setName)
# #for enhancers
# mapped_list = ['%sMM9_NB_MYCN_CONSERVED_ENHANCER_-5000_+5000/MM9_NB_MYCN_CONSERVED_ENHANCER_-5000_+5000_%s.gff' % (mappedFolder,name) for name in names_list]
# output_path = '%sMM9_NB_MYCN_CONSERVED_ENHANCER_-5000_+5000/MM9_NB_MYCN_CONSERVED_ENHANCER_-5000_+5000_%s.gff' % (mappedFolder,setName)
# print(output_path)
# averagingMappedSignal(mapped_list,output_path,setName)
print('\n\n')
print(
'#======================================================================'
)
print(
'#=====================VIII. MAKING HEATMAPS/METAS======================'
)
print(
'#======================================================================'
)
print('\n\n')
def mergeCollections(enhancerFile1,enhancerFile2,name1,name2,output='',inputGFF=''):
'''
merges them collections
'''
print(enhancerFile1)
print(enhancerFile2)
name1Collection = makeSECollection(enhancerFile1,name1)
name2Collection = makeSECollection(enhancerFile2,name2)
print(len(name1Collection))
print(len(name2Collection))
print('weeeeeee')
if len(inputGFF) == 0:
#now merge them
mergedLoci = name1Collection.getLoci() + name2Collection.getLoci()
mergedCollection = utils.LocusCollection(mergedLoci,50)
#stitch the collection together
stitchedCollection = mergedCollection.stitchCollection()
stitchedLoci = stitchedCollection.getLoci()
else:
locusCollection = utils.gffToLocusCollection(inputGFF)
stitchedCollection = locusCollection.stitchCollection()
stitchedLoci = stitchedCollection.getLoci()
#rename loci by presence in group1 or group2
renamedLoci =[]
conserved_ticker = 1
name1_ticker = 1
name2_ticker = 1
for locus in stitchedLoci:
if len(name1Collection.getOverlap(locus)) > 0 and len(name1Collection.getOverlap(locus)) > 0:
newID = 'CONSERVED_%s' % (str(conserved_ticker))
conserved_ticker +=1
elif len(name1Collection.getOverlap(locus)) > 0 and len(name1Collection.getOverlap(locus)) == 0:
newID = '%s_%s' % (name1,str(name1_ticker))
name1_ticker +=1
else:
newID = '%s_%s' % (name2,str(name2_ticker))
name2_ticker +=1
locus._ID = newID
renamedLoci.append(locus)
#now we turn this into a gff and write it out
gff = utils.locusCollectionToGFF(utils.LocusCollection(renamedLoci,50))
if len(output) == 0:
return gff
else:
print "writing merged gff to %s" % (output)
utils.unParseTable(gff,output,'\t')
return output
def make_shep21_mycn_landscape(nb_all_chip_dataFile):
'''
finds mycn peaks in shep21 that are conserved in nb and segregates them into promoter or enhancer
'''
#first get the shep21 regions
print('LOADING SHEP21 MYCN SITES')
dataDict = pipeline_dfci.loadDataTable(nb_all_chip_dataFile)
shep21_0hr_mycn_enriched_file = '%s%s' % (
macsEnrichedFolder,
dataDict['SHEP21_0HR_MYCN_NOSPIKE']['enrichedMacs'])
shep21_0hr_mycn_bed = utils.parseTable(shep21_0hr_mycn_enriched_file, '\t')
#now get the conserved NB MYCN regions
nb_conserved_mycn_gff_file = '%sHG19_NB_MYCN_CONSERVED_-0_+0.gff' % (
gffFolder)
nb_conserved_mycn_collection = utils.gffToLocusCollection(
nb_conserved_mycn_gff_file)
print('LOADING SHEP21 ACTIVE ENHANCERS')
#make a collection of enhancers
shep21_enhancer_file = '%senhancer_rose/SHEP21_0HR_H3K27AC_NOSPIKE_ROSE/SHEP21_0HR_H3K27AC_NOSPIKE_peaks_AllEnhancers.table.txt' % (
projectFolder)
shep21_enhancer_collection = utils.makeSECollection(
shep21_enhancer_file, 'SHEP21_0HR_H3K27AC_NOSPIKE')
#now get the active promoters
print('LOADING SHEP21 ACTIVE PROMOTERS')
startDict = utils.makeStartDict(annotFile)
shep21_transcribed_file = '%sHG19_SHEP21_H3K27AC_TRANSCRIBED.txt' % (
geneListFolder)
shep21_transcribed_table = utils.parseTable(shep21_transcribed_file, '\t')
transcribedList = [line[1] for line in shep21_transcribed_table]
tssLoci = []
for refID in transcribedList:
tssLoci.append(utils.makeTSSLocus(refID, startDict, 1000, 1000))
shep21_tss_collection = utils.LocusCollection(tssLoci, 50)
#now initialize the 6 gffs we will need
shep21_mycn_conserved_gff = []
shep21_mycn_conserved_gff_5kb = []
shep21_mycn_conserved_gff_1kb = []
shep21_mycn_conserved_promoter_gff = []
shep21_mycn_conserved_promoter_gff_1kb = []
shep21_mycn_conserved_promoter_gff_5kb = []
shep21_mycn_conserved_enhancer_gff = []
shep21_mycn_conserved_enhancer_gff_1kb = []
shep21_mycn_conserved_enhancer_gff_5kb = []
#and their respective file names
shep21_mycn_conserved_gff_file = '%sHG19_SHEP21_0HR_MYCN_NOSPIKE_CONSERVED_-0_+0.gff' % (
gffFolder)
shep21_mycn_conserved_gff_5kb_file = '%sHG19_SHEP21_0HR_MYCN_NOSPIKE_CONSERVED_-5kb_+5kb.gff' % (
gffFolder)
shep21_mycn_conserved_gff_1kb_file = '%sHG19_SHEP21_0HR_MYCN_NOSPIKE_CONSERVED_-1kb_+1kb.gff' % (
gffFolder)
shep21_mycn_conserved_promoter_gff_file = '%sHG19_SHEP21_0HR_MYCN_NOSPIKE_CONSERVED_PROMOTER_-0_+0.gff' % (
gffFolder)
shep21_mycn_conserved_promoter_gff_5kb_file = '%sHG19_SHEP21_0HR_MYCN_NOSPIKE_CONSERVED_PROMOTER_-5kb_+5kb.gff' % (
gffFolder)
shep21_mycn_conserved_promoter_gff_1kb_file = '%sHG19_SHEP21_0HR_MYCN_NOSPIKE_CONSERVED_PROMOTER_-1kb_+1kb.gff' % (
gffFolder)
shep21_mycn_conserved_enhancer_gff_file = '%sHG19_SHEP21_0HR_MYCN_NOSPIKE_CONSERVED_ENHANCER_-0_+0.gff' % (
gffFolder)
shep21_mycn_conserved_enhancer_gff_5kb_file = '%sHG19_SHEP21_0HR_MYCN_NOSPIKE_CONSERVED_ENHANCER_-5kb_+5kb.gff' % (
gffFolder)
shep21_mycn_conserved_enhancer_gff_1kb_file = '%sHG19_SHEP21_0HR_MYCN_NOSPIKE_CONSERVED_ENHANCER_-1kb_+1kb.gff' % (
gffFolder)
print('ITERATING THROUGH SHEP21 MYCN PEAKS')
ticker = 0
for line in shep21_0hr_mycn_bed:
if ticker % 1000 == 0:
print ticker
ticker += 1
peakID = '%s_%s' % ('SHEP21_0HR_MYCN_NOSPIKE', str(ticker))
lineLocus = utils.Locus(line[0], line[1], line[2], '.', peakID)
if nb_conserved_mycn_collection.getOverlap(lineLocus):
gffLine = [
line[0], peakID, peakID, line[1], line[2], '', '.', '', peakID
]
peakCenter = (int(line[1]) + int(line[2])) / 2
gffLine_5kb = [
line[0], peakID, peakID, peakCenter - 5000, peakCenter + 5000,
'', '.', '', peakID
]
#the 1kb is not a center +/- but a flank
gffLine_1kb = [
line[0], peakID, peakID,
int(line[1]) - 1000,
int(line[2]) + 1000, '', '.', '', peakID
]
shep21_mycn_conserved_gff.append(gffLine)
shep21_mycn_conserved_gff_5kb.append(gffLine_5kb)
shep21_mycn_conserved_gff_1kb.append(gffLine_1kb)
#tss overlap should take precedence over enhancer overlap
if shep21_tss_collection.getOverlap(lineLocus, 'both'):
shep21_mycn_conserved_promoter_gff.append(gffLine)
shep21_mycn_conserved_promoter_gff_5kb.append(gffLine_5kb)
shep21_mycn_conserved_promoter_gff_1kb.append(gffLine_1kb)
#now check for enhancer overlap
elif shep21_enhancer_collection.getOverlap(lineLocus, 'both'):
shep21_mycn_conserved_enhancer_gff.append(gffLine)
shep21_mycn_conserved_enhancer_gff_5kb.append(gffLine_5kb)
shep21_mycn_conserved_enhancer_gff_1kb.append(gffLine_1kb)
#now write out the gffs
utils.unParseTable(shep21_mycn_conserved_gff,
shep21_mycn_conserved_gff_file, '\t')
utils.unParseTable(shep21_mycn_conserved_gff_5kb,
shep21_mycn_conserved_gff_5kb_file, '\t')
utils.unParseTable(shep21_mycn_conserved_gff_1kb,
shep21_mycn_conserved_gff_1kb_file, '\t')
utils.unParseTable(shep21_mycn_conserved_promoter_gff,
shep21_mycn_conserved_promoter_gff_file, '\t')
utils.unParseTable(shep21_mycn_conserved_promoter_gff_5kb,
shep21_mycn_conserved_promoter_gff_5kb_file, '\t')
utils.unParseTable(shep21_mycn_conserved_promoter_gff_1kb,
shep21_mycn_conserved_promoter_gff_1kb_file, '\t')
utils.unParseTable(shep21_mycn_conserved_enhancer_gff,
shep21_mycn_conserved_enhancer_gff_file, '\t')
utils.unParseTable(shep21_mycn_conserved_enhancer_gff_5kb,
shep21_mycn_conserved_enhancer_gff_5kb_file, '\t')
utils.unParseTable(shep21_mycn_conserved_enhancer_gff_1kb,
shep21_mycn_conserved_enhancer_gff_1kb_file, '\t')