def parClean(fN):
if '/' in fN:
dirName = os.path.dirname(fN)
else:
dirName = os.environ['PWD']
basename = os.path.basename(fN)
#remove the original file
os.remove(fN)
#remove the exit signals
for eFN in bioLibCG.recurseDir(dirName, start = basename, end = 'exitSignal'):
pass
os.remove(eFN)
#cat the SORTED range files.
rangeFiles = bioLibCG.recurseDir(dirName, start = basename, within = 'range')
rangeFiles.sort(key = lambda x: int(x.split('.')[-2]))
f = open(fN, 'w')
for rFN in rangeFiles:
subprocess.Popen(['cat', rFN], stdout = f).wait()
f.close()
#remove the range files
for rFN in rangeFiles:
os.remove(rFN)
def probe(tcc, conf=None):
if not conf:
mConf = c.cgConfig('Main.conf')
smallPath = mConf.conf['smallPath']
chrom, strand, start, end = cg.tccSplit(tcc)
total = 0
for lib in cg.recurseDir(smallPath, end='mapped.%s.wig' % strand):
try:
eLevels = stepVectorScan.scanVectorsFile(lib, [tcc])
except:
print lib, 'index failed'
continue
#find highest expression level
highest = 0
for coord in eLevels:
if eLevels[coord] > highest:
highest = eLevels[coord]
if highest > 0:
print lib, highest
total += highest
#print eLevels
print total
def scanSequence(seqList, dirName): '''Given list of sequences --> get all reads that have sequence ''' fileNames = cg.recurseDir(dirName, end = '.sequence') if len(fileNames) > 1: print fileNames print 'there is more than one sequence file in this directory' return 1 else: fN = fileNames[0] #for seq in seqList: seq = seqList fIndex = cgIndex.lineIndex(fN, header = False) fIndex.passCheckFunction(cgIndex.mapSequenceCheckFunction) fIndex.binarySearch(seq) #places file pointer at beginning of sequence line #extend and report fIndex.extendUp(seq) finalReads = [] for line in fIndex.file: if fIndex.checkFunction(seq, line) == 0: finalReads.append(line.strip()) else: return finalReads
def scanSequence(seqList, dirName):
'''Given list of sequences --> get all reads that have sequence
'''
fileNames = cg.recurseDir(dirName, end='.sequence')
if len(fileNames) > 1:
print fileNames
print 'there is more than one sequence file in this directory'
return 1
else:
fN = fileNames[0]
#for seq in seqList:
seq = seqList
fIndex = cgIndex.lineIndex(fN, header=False)
fIndex.passCheckFunction(cgIndex.mapSequenceCheckFunction)
fIndex.binarySearch(
seq) #places file pointer at beginning of sequence line
#extend and report
fIndex.extendUp(seq)
finalReads = []
for line in fIndex.file:
if fIndex.checkFunction(seq, line) == 0:
finalReads.append(line.strip())
else:
return finalReads
def createMTrack(dirName): '''merge all mapped tracks in directory and create a single wig file''' fileList = cg.recurseDir(dirName, end = '.out') chroms = cg.humanChromosomes print 'Making Bed File vectors' cvg = HTSeq.GenomicArray(chroms, stranded=True, typecode='i') for fName in fileList: print fName alignment_file = HTSeq.BowtieReader(fName) for alngt in alignment_file: if alngt.aligned: try: cvg.add_value( 1, alngt.iv ) #iv is the genomic interval.. except KeyError: pass bedNamePos = dirName + '/Merge.' + 'hg19' + '.1.wig' bedNameNeg = dirName + '/Merge.' + 'hg19' + '.-1.wig' print 'Writing Bed File' cvg.write_bedgraph_file(bedNamePos, "+" ) cvg.write_bedgraph_file(bedNameNeg, "-" ) #Now extend it updateWigLength(bedNamePos, 'hg19') updateWigLength(bedNameNeg, 'hg19') #Now Sort it. cgSort.wigSort(bedNamePos) cgSort.wigSort(bedNameNeg)
def probe(tcc, conf = None):
if not conf:
mConf = c.cgConfig('Main.conf')
smallPath = mConf.conf['smallPath']
chrom, strand, start, end = cg.tccSplit(tcc)
total = 0
for lib in cg.recurseDir(smallPath, end = 'mapped.%s.wig' % strand):
try:
eLevels = stepVectorScan.scanVectorsFile(lib, [tcc])
except:
print lib, 'index failed'
continue
#find highest expression level
highest = 0
for coord in eLevels:
if eLevels[coord] > highest:
highest = eLevels[coord]
if highest > 0:
print lib, highest
total += highest
#print eLevels
print total
def mixWig(directory, assembly, name=None):
'''Does it by chromosome --> faster, less memory'''
if not name: name = 'Merge'
#gather all chromosomes
chromList = []
for fN in cg.recurseDir(directory, end='.wig'):
chrom = cg.getBaseFileName(fN).strip().split('.')[-3]
if chrom not in chromList:
chromList.append(chrom)
print chromList
for chrom in chromList:
print chrom
#Gather all the values from all the files
hitDict = {} # chrom : { strand : coord
for fN in cg.recurseDir(directory, end='.wig'):
fChrom = cg.getBaseFileName(fN).strip().split('.')[-3]
if fChrom != chrom: continue
print ' ', fN, fChrom
f = open(fN, 'r')
f.readline() #header
strand = cg.getBaseFileName(fN).strip().split('.')[-2]
for line in f:
lChrom, start, end, val = (line.strip().split('\t'))
start, end, val = int(start), int(end), int(val)
if val < 1: continue
#print start, end, val
for i in range(start, end):
try:
hitDict[lChrom][strand][i] += val
except (KeyError, TypeError):
if not lChrom in hitDict:
hitDict[lChrom] = {}
if not strand in hitDict[lChrom]:
hitDict[lChrom][strand] = {}
hitDict[lChrom][strand][i] = val
#write results to wig file
writeWigFromHitDict(hitDict, assembly, name, directory)
def mixWig(directory, assembly, name = None):
'''Does it by chromosome --> faster, less memory'''
if not name: name = 'Merge'
#gather all chromosomes
chromList = []
for fN in cg.recurseDir(directory, end = '.wig'):
chrom = cg.getBaseFileName(fN).strip().split('.')[-3]
if chrom not in chromList:
chromList.append(chrom)
print chromList
for chrom in chromList:
print chrom
#Gather all the values from all the files
hitDict = {} # chrom : { strand : coord
for fN in cg.recurseDir(directory, end = '.wig'):
fChrom = cg.getBaseFileName(fN).strip().split('.')[-3]
if fChrom != chrom: continue
print ' ', fN, fChrom
f = open(fN, 'r')
f.readline() #header
strand = cg.getBaseFileName(fN).strip().split('.')[-2]
for line in f:
lChrom, start, end, val = (line.strip().split('\t'))
start, end, val = int(start), int(end), int(val)
if val < 1: continue
#print start, end, val
for i in range(start, end):
try:
hitDict[lChrom][strand][i] += val
except (KeyError,TypeError):
if not lChrom in hitDict:
hitDict[lChrom] = {}
if not strand in hitDict[lChrom]:
hitDict[lChrom][strand] = {}
hitDict[lChrom][strand][i] = val
#write results to wig file
writeWigFromHitDict(hitDict, assembly, name, directory)
def parCleanSplit(fN):
'''Remove the exit signals for a split continuation run'''
if '/' in fN:
dirName = os.path.dirname(fN)
else:
dirName = os.environ['PWD']
basename = os.path.basename(fN)
#remove the exit signals
for eFN in bioLibCG.recurseDir(dirName, start = basename, end = 'exitSignal'):
os.remove(eFN)
def parCleanSplit(fN):
'''Remove the exit signals for a split continuation run'''
if '/' in fN:
dirName = os.path.dirname(fN)
else:
dirName = os.environ['PWD']
basename = os.path.basename(fN)
#remove the exit signals
for eFN in bioLibCG.recurseDir(dirName, start=basename, end='exitSignal'):
os.remove(eFN)
def parClean(fN):
if '/' in fN:
dirName = os.path.dirname(fN)
else:
dirName = os.environ['PWD']
basename = os.path.basename(fN)
#remove the original file
print '..removing original file, if present'
try:
os.remove(fN)
except OSError:
pass
#remove the exit signals
print '..removing exit signals'
for eFN in bioLibCG.recurseDir(dirName, start = basename, end = 'exitSignal'):
os.remove(eFN)
#cat the SORTED range files.
rangeFiles = bioLibCG.recurseDir(dirName, start = basename, within = 'range')
rangeFiles.sort(key = lambda x: int(x.split('.')[-2]))
print '..catting files together'
f = open(fN, 'w')
for rFN in rangeFiles:
print '....', rFN
subprocess.Popen(['cat', rFN], stdout = f).wait()
f.close()
#remove the range files
print '..removing packets'
for rFN in rangeFiles:
print '....', rFN
os.remove(rFN)
def getAll(chrom, strand, point): fNs = cg.recurseDir(mConf.conf['smallPath'], end = '.wig') for file in fNs: if 'WIG' in file: fNs.remove(file) elif file == '/home/chrisgre/smallLibs/WIGS/Merge.mouse.1.wig' or '/home/chrisgre/smallLibs/WIGS/Merge.human.1.wig': fNs.remove(file) for fN in fNs: fStrand = cg.ss(fN, '.')[-2] if str(fStrand) == str(strand): val = getWigValue(chrom, point, fN) if val > 0: print fN, val
def getAll(chrom, strand, point):
fNs = cg.recurseDir(mConf.conf['smallPath'], end='.wig')
for file in fNs:
if 'WIG' in file:
fNs.remove(file)
elif file == '/home/chrisgre/smallLibs/WIGS/Merge.mouse.1.wig' or '/home/chrisgre/smallLibs/WIGS/Merge.human.1.wig':
fNs.remove(file)
for fN in fNs:
fStrand = cg.ss(fN, '.')[-2]
if str(fStrand) == str(strand):
val = getWigValue(chrom, point, fN)
if val > 0:
print fN, val
def parClean(fN):
if '/' in fN:
dirName = os.path.dirname(fN)
else:
dirName = os.environ['PWD']
basename = os.path.basename(fN)
#remove the original file
print '..removing original file, if present'
try:
os.remove(fN)
except OSError:
pass
#remove the exit signals
print '..removing exit signals'
for eFN in bioLibCG.recurseDir(dirName, start=basename, end='exitSignal'):
os.remove(eFN)
#cat the SORTED range files.
rangeFiles = bioLibCG.recurseDir(dirName, start=basename, within='range')
rangeFiles.sort(key=lambda x: int(x.split('.')[-2]))
print '..catting files together'
f = open(fN, 'w')
for rFN in rangeFiles:
print '....', rFN
subprocess.Popen(['cat', rFN], stdout=f).wait()
f.close()
#remove the range files
print '..removing packets'
for rFN in rangeFiles:
print '....', rFN
os.remove(rFN)
def load(self):
#get schema
attName_field = getClassScheme(self.mappingClass)
id_obj = {}
selectedAttNames = []
#load defined attributes
for fN in bioLibCG.recurseDir(self.aDir):
baseName = fN.strip().split('/')[-1]
if baseName.startswith('a.'):
#get atribute name/type
bs = baseName.split('.')
attName = bs[1]
attType = attName_field[attName].dataType
casteFxn = getCasteFunction(attType)
selectedAttNames.append(attName)
#now set the attributes
f = open(fN, 'r')
for line in f:
ls = line.strip().split('\t')
id = int(ls[0])
att = ls[1]
if attType in listTypes:
att = att.split(',')
att = [casteFxn(x) for x in att]
else:
att = casteFxn(att)
#now set the attribute property for the id
if id not in id_obj:
id_obj[id] = self.mappingClass(id)
o = id_obj[id]
setattr(o, attName, att)
f.close()
#Now initialize objects---It's important to do this after all the data has been loaded...
for obj in id_obj.values():
loadedAttNames = obj.__dict__.keys()
for attName in selectedAttNames:
if attName not in loadedAttNames:
setattr(obj, attName, copy.copy(attName_field[attName].dataDefault))
return id_obj
def checkDups(dir):
id_seqs = {}
for i in bioLibCG.recurseDir(dir, end='.simSeqs'):
f = open(i, 'r')
for line in f:
ls = line.strip().split('\t')
id, seq = ls[0], ls[1]
id_seqs.setdefault(id, []).append(seq)
for id, seqs in id_seqs.items():
if len(seqs) != len(list(set(seqs))):
print 'fail'
print id, seqs
def load(self):
#get schema
attName_field = getClassScheme(self.mappingClass)
id_obj = {}
selectedAttNames = []
#load defined attributes
for fN in bioLibCG.recurseDir(self.aDir):
baseName = fN.strip().split('/')[-1]
if baseName.startswith('a.'):
#get atribute name/type
bs = baseName.split('.')
attName = bs[1]
attType = attName_field[attName].dataType
casteFxn = getCasteFunction(attType)
selectedAttNames.append(attName)
#now set the attributes
f = open(fN, 'r')
for line in f:
ls = line.strip().split('\t')
id = int(ls[0])
att = ls[1]
if attType in listTypes:
att = att.split(',')
att = [casteFxn(x) for x in att]
else:
att = casteFxn(att)
#now set the attribute property for the id
if id not in id_obj:
id_obj[id] = self.mappingClass(id)
o = id_obj[id]
setattr(o, attName, att)
f.close()
#Now initialize objects---It's important to do this after all the data has been loaded...
for obj in id_obj.values():
loadedAttNames = obj.__dict__.keys()
for attName in selectedAttNames:
if attName not in loadedAttNames:
setattr(obj, attName,
copy.copy(attName_field[attName].dataDefault))
return id_obj
def checkDups(dir):
id_seqs = {}
for i in bioLibCG.recurseDir(dir, end = '.simSeqs'):
f = open(i, 'r')
for line in f:
ls = line.strip().split('\t')
id, seq = ls[0], ls[1]
id_seqs.setdefault(id, []).append(seq)
for id, seqs in id_seqs.items():
if len(seqs) != len(list(set(seqs))):
print 'fail'
print id, seqs
def mapFastQInDirQ(dirName, overwrite = True):
'''Every Q function has a corresponding shell script'''
wrapperShell = '/home/chrisgre/scripts/mapping/mapFastQ.sh'
for file in cg.recurseDir(dirName, end = 'clipped.fastq'):
print file
putativeN = file.replace('.clipped.fastq','.clipped.fastq.mapped')
if os.path.isfile(putativeN):
if overwrite:
print ' Overwriting file', putativeN
os.remove(putativeN)
else:
print ' \nNOT OVERWRITING FILE', putativeN
continue
#check if mouse or human
baseFName = cg.getBaseFileName(file, naked = True)
org = 'None'
for metaFileName in metaFileNames:
mFile = open(metaFileName, 'r')
for line in mFile:
fields = line.strip().split('\t')
if baseFName == fields[0]:
if fields[2] == 'NONE':
print ' NO ORG KNOWN FOR', file
continue
else:
org = fields[2]
print ' USING ORG', org, file
mFile.close()
#check if there is an organism, must check due to files not in metaFile
if org == 'None':
print ' NO org', file
continue
while True:
#submit job if there are less than ten
if clusterCheck.queryNumJobsQ('chrisgre') < 40:
#subprocess.Popen(['qsub', '-q', 'xiao', '-l', 'mem=4G', '-V', '-o', mainConf.conf['outLog'], '-e', mainConf.conf['errorLog'], wrapperShell, file, org ])
subprocess.Popen(['qsub', '-l', 'mem=4G', '-V', '-o', mainConf.conf['outLog'], '-e', mainConf.conf['errorLog'], wrapperShell, file, org ])
time.sleep(.2) #give it time to update qstat
break
else:#wait 10 secs...
time.sleep(20)
def checkExit(fN, numPackets):
numPackets = int(numPackets)
dirName = os.path.dirname(fN)
baseName = os.path.basename(fN)
if os.environ['PWD'] not in dirName:
dirName = os.path.dirname(os.environ['PWD'] + '/' + fN)
sleepTime = 1
pbar = ProgressBar(widgets=[' ', SimpleProgress(), ' ', Timer(), ' ', Bar()], maxval=numPackets).start()
while True:
time.sleep(sleepTime)
exitSignals = bioLibCG.recurseDir(dirName, start = baseName , end = 'exitSignal')
pbar.update(len(exitSignals))
if len(exitSignals) == numPackets:
pbar.finish()
print 'Jobs all finished!'
break
def checkExit(fN, numPackets):
numPackets = int(numPackets)
dirName = os.path.dirname(fN)
baseName = os.path.basename(fN)
if os.environ['PWD'] not in dirName:
dirName = os.path.dirname(os.environ['PWD'] + '/' + fN)
sleepTime = 10
iteration = 1
while True:
time.sleep(sleepTime)
exitSignals = bioLibCG.recurseDir(dirName, start = baseName , end = 'exitSignal')
print 'waiting...', str(len(exitSignals)), '/', str(numPackets), '%s' % bioLibCG.prettyTime(sleepTime * iteration), fN
if len(exitSignals) == numPackets:
print 'Jobs all finished!'
break
iteration += 1
def clipAdapterInDirQ(dirName):
'''The Q if for doing it on a cluster using qsub
Every Q function has a corresponding shell script'''
for file in cg.recurseDir(dirName, end = '.fastq'):
#check if it isn't a clipped file:
if 'clipped' in file:
continue
while True:
#submit job if there are less than ten
if clusterCheck.queryNumJobsQ('chrisgre') < 100:
subprocess.Popen(['qsub', '-V', '-cwd', '-o', 'errors', '-e', 'errors', wrapperShell, file])
time.sleep(.2) #give it time to update qstat
break
else:#wait 10 secs...
time.sleep(10)
def mergeInputs(cName, eLevel):
conf = c.getConfig(cName)
assembly = conf.conf['assembly']
ending = '%s.%s' % (eLevel, assembly)
print 'merging all files with ending', ending
newLines = []
for fN in cg.recurseDir('out', end = ending):
print os.getcwd(), fN
fN = os.getcwd() + '/' + fN
f = open(fN, 'r')
newLines.extend(f.readlines())
f.close()
f = open('peakData.%s.%s' % (eLevel, assembly), 'w')
f.writelines(newLines)
f.close()
def mergeInputs(cName, eLevel):
conf = c.getConfig(cName)
assembly = conf.conf['assembly']
ending = '%s.%s' % (eLevel, assembly)
print 'merging all files with ending', ending
newLines = []
for fN in cg.recurseDir('out', end=ending):
print os.getcwd(), fN
fN = os.getcwd() + '/' + fN
f = open(fN, 'r')
newLines.extend(f.readlines())
f.close()
f = open('peakData.%s.%s' % (eLevel, assembly), 'w')
f.writelines(newLines)
f.close()
def createMultiTrackDir(dirName, organism):
'''THIS DIFFERS FROM ABOVE BECAUSE IT DOESN't REQUIRE META INFO
IT JUST MAKES A MERGED WIG FOR EVERYTHING IN THE DIRECTORY'''
mainConf = c.cgConfig('Main.conf')
fileList = []
for file in cg.recurseDir(dirName, end = '.mapped'):
fileList.append(file)
#make merged wig
if organism == 'human':
chroms = cg.humanChromosomes
assembly = 'hg19'
elif organism == 'mouse':
chroms = cg.mouseChromosomes
assembly = 'mm9'
elif organism == 'zebrafish':
chroms = cg.zebrafishChromosomes
assembly = 'danRer6'
print 'Making Bed File vectors'
cvg = HTSeq.GenomicArray(chroms, stranded=True, typecode='i')
for fName in fileList:
alignment_file = HTSeq.BowtieReader(fName)
for alngt in alignment_file:
if alngt.aligned:
cvg.add_value( 1, alngt.iv ) #iv is the genomic interval..
bedNamePos = dirName + '/Merge.' + organism + '.1.wig'
bedNameNeg = dirName + '/Merge.' + organism + '.-1.wig'
print 'Writing Bed File'
cvg.write_bedgraph_file(bedNamePos, "+" )
cvg.write_bedgraph_file(bedNameNeg, "-" )
#Now extend it
updateWigLength(bedNamePos, assembly)
updateWigLength(bedNameNeg, assembly)
#Now Sort it.
cgSort.wigSort(bedNamePos)
cgSort.wigSort(bedNameNeg)
def checkExit(fN, numPackets):
numPackets = int(numPackets)
dirName = os.path.dirname(fN)
baseName = os.path.basename(fN)
if os.environ["PWD"] not in dirName:
dirName = os.path.dirname(os.environ["PWD"] + "/" + fN)
sleepTime = 10
iteration = 1
while True:
time.sleep(sleepTime)
exitSignals = bioLibCG.recurseDir(dirName, start=baseName, end="exitSignal")
print "waiting...", str(len(exitSignals)), "/", str(numPackets), "%s" % bioLibCG.prettyTime(
sleepTime * iteration
), fN
if len(exitSignals) == numPackets:
print "Jobs all finished!"
break
iteration += 1
def clipAdapterInDirQ(dirName):
'''The Q if for doing it on a cluster using qsub
Every Q function has a corresponding shell script'''
for file in cg.recurseDir(dirName, end='.fastq'):
#check if it isn't a clipped file:
if 'clipped' in file:
continue
while True:
#submit job if there are less than ten
if clusterCheck.queryNumJobsQ('chrisgre') < 100:
subprocess.Popen([
'qsub', '-V', '-cwd', '-o', 'errors', '-e', 'errors',
wrapperShell, file
])
time.sleep(.2) #give it time to update qstat
break
else: #wait 10 secs...
time.sleep(10)
def createTrackInDir(dirName):
'''Every Q function has a corresponding shell script
Make wig file for all mapped files, for all organisms'''
wrapperShell = '/home/chrisgre/scripts/mapping/createTrack.sh'
mainConf = c.cgConfig('Main.conf')
metaFileName = mainConf.conf['metaFileName']
for file in cg.recurseDir(dirName, end = '.mapped'):
#check if mouse or human
baseFName = cg.getBaseFileName(file)
baseFName = baseFName.split('.')[0]
metaDict = cg.getMetaFileDict(metaFileName)
org = 'None'
if baseFName in metaDict:
if metaDict[baseFName][1] == 'NONE':
print ' NO ORG KNOWN FOR', file
continue
else:
org = metaDict[baseFName][1]
print ' USING ORG', org, file
#check if there is an organism, must check due to files not in metaFile
if org == 'None':
print ' NO org (not in meta file)', file
continue
while True:
#submit job if there are less than ten
if clusterCheck.queryNumJobsQ('chrisgre') < 1000:
#subprocess.Popen(['qsub', '-q', 'xiao', '-l', 'mem=4G', '-V', '-o', mainConf.conf['outLog'], '-e', mainConf.conf['errorLog'], wrapperShell, file, org ])
subprocess.Popen(['qsub', '-V', '-o', mainConf.conf['outLog'], '-e', mainConf.conf['errorLog'], wrapperShell, file, org ])
#time.sleep(.5) #give it time to update qstat
break
else:#wait 10 secs...
time.sleep(20)
def splitRun(baseFN, memoryAmount, scriptName, *args):
if '/' in baseFN:
dirName = os.path.dirname(baseFN)
else:
dirName = os.environ['PWD']
basename = os.path.basename(baseFN)
#cat the SORTED range files.
rangeFiles = [
x for x in bioLibCG.recurseDir(dirName, start=basename, within='range')
if 'exitSignal' not in x
]
rangeFiles.sort(key=lambda x: int(x.split('.')[-2]))
for fN in rangeFiles:
#specify the correct qJob with correct memory
qJobX = '%s/exec/qJobX%s.sh' % (os.environ['HOME'], memoryAmount)
qDo = '%s/exec/qDo.sh' % (os.environ['HOME'])
#construct command to pass
if memoryAmount == 'LOCAL':
com = [qDo, scriptName]
else:
com = [qJobX, qDo, scriptName]
#append script arguments including SPLIT FN
for arg in args:
if arg == 'splitFN':
com.append(fN)
else:
com.append(arg)
#now append paraInfo for split run [splitRun, splitRun]
com.extend(['splitRun', 'splitRun'])
#run each job
subprocess.Popen(com).wait()
def loadWigDictFloat(wigDir):
'''Wig files in a directory must be a certain file format: NAME.chr.strand.wig'''
chr_strand_coord_expr = {}
for fN in bioLibCG.recurseDir(wigDir, end = '.wig'):
chrom, strand = fN.split('/')[-1].split('.')[1], fN.split('/')[-1].split('.')[2]
chr_strand_coord_expr.setdefault(chrom, {})[strand] = {}
f = open(fN, 'r')
f.readline() #header
for line in f:
ls = line.strip().split('\t')
start, end, expr = int(ls[1]) + 1, int(ls[2]), float(ls[3]) #1BASE
if float(expr) == 0.0: continue
for i in range(start, end + 1):
chr_strand_coord_expr[chrom][strand][i] = expr
f.close()
return chr_strand_coord_expr
def checkExit(fN, numPackets):
numPackets = int(numPackets)
dirName = os.path.dirname(fN)
baseName = os.path.basename(fN)
if os.environ['PWD'] not in dirName:
dirName = os.path.dirname(os.environ['PWD'] + '/' + fN)
sleepTime = 1
pbar = ProgressBar(
widgets=[' ', SimpleProgress(), ' ',
Timer(), ' ',
Bar()],
maxval=numPackets).start()
while True:
time.sleep(sleepTime)
exitSignals = bioLibCG.recurseDir(dirName,
start=baseName,
end='exitSignal')
pbar.update(len(exitSignals))
if len(exitSignals) == numPackets:
pbar.finish()
print 'Jobs all finished!'
break
def scanCoord(tcc, dirName):
fileNames = cg.recurseDir(dirName, end='.starts')
#get name of file for index
chrom, strand, start, end = cg.tccSplit(tcc)
nameCheck = '%s.%s' % (chrom, strand)
fN = 'None'
for fileName in fileNames:
if nameCheck in fileName: fN = fileName
if fN == 'None':
print 'No Index file for', nameCheck
return 0
fIndex = cgIndex.lineIndex(fN, header=False)
fIndex.passCheckFunction(cgIndex.mapStartCheckFunction)
fIndex.binarySearch(
tcc, skipEnd=True) #places file pointer at beginning of sequence line
#Check if you need to move down one line
checkLine = fIndex.getLineFromByte(fIndex.currentByte)
fIndex.passCheckFunction(
cgIndex.mapStartRangeCheckFunction
) #Note i'm passing now, but it is also used in extending
if fIndex.checkFunction(tcc, checkLine) != 0:
fIndex.file.readline()
fIndex.currentByte = fIndex.file.tell()
#Now extend up until in range, down until in range --> return reads.
fIndex.extendUp(tcc)
finalReads = []
for line in fIndex.file:
if fIndex.checkFunction(tcc, line) == 0:
finalReads.append(line.strip())
else:
return finalReads
def splitRun(baseFN, memoryAmount, scriptName, *args):
if "/" in baseFN:
dirName = os.path.dirname(baseFN)
else:
dirName = os.environ["PWD"]
basename = os.path.basename(baseFN)
# cat the SORTED range files.
rangeFiles = [x for x in bioLibCG.recurseDir(dirName, start=basename, within="range") if "exitSignal" not in x]
rangeFiles.sort(key=lambda x: int(x.split(".")[-2]))
for fN in rangeFiles:
# specify the correct qJob with correct memory
qJobX = "%s/exec/qJobX%s.sh" % (os.environ["HOME"], memoryAmount)
qDo = "%s/exec/qDo.sh" % (os.environ["HOME"])
# construct command to pass
if memoryAmount == "LOCAL":
com = [qDo, scriptName]
else:
com = [qJobX, qDo, scriptName]
# append script arguments including SPLIT FN
for arg in args:
if arg == "splitFN":
com.append(fN)
else:
com.append(arg)
# now append paraInfo for split run [splitRun, splitRun]
com.extend(["splitRun", "splitRun"])
# run each job
subprocess.Popen(com).wait()
def scanCoord(tcc, dirName):
fileNames = cg.recurseDir(dirName, end = '.starts')
#get name of file for index
chrom, strand, start, end = cg.tccSplit(tcc)
nameCheck = '%s.%s' % (chrom, strand)
fN = 'None'
for fileName in fileNames:
if nameCheck in fileName: fN = fileName
if fN == 'None':
print 'No Index file for', nameCheck
return 0
fIndex = cgIndex.lineIndex(fN, header = False)
fIndex.passCheckFunction(cgIndex.mapStartCheckFunction)
fIndex.binarySearch(tcc, skipEnd = True) #places file pointer at beginning of sequence line
#Check if you need to move down one line
checkLine = fIndex.getLineFromByte(fIndex.currentByte)
fIndex.passCheckFunction(cgIndex.mapStartRangeCheckFunction) #Note i'm passing now, but it is also used in extending
if fIndex.checkFunction(tcc, checkLine) != 0:
fIndex.file.readline()
fIndex.currentByte = fIndex.file.tell()
#Now extend up until in range, down until in range --> return reads.
fIndex.extendUp(tcc)
finalReads = []
for line in fIndex.file:
if fIndex.checkFunction(tcc, line) == 0:
finalReads.append(line.strip())
else:
return finalReads
def commit(self, id_obj):
#get schema
attName_field = getClassScheme(self.mappingClass)
#will update this functionality later
selectedAttNames = [x for x in attName_field]
#Get new values...Ideally this would only have the ones that have changed...but it doesn't...
id_att_newVals = {}
for id, obj in id_obj.items():
id_att_newVals[id] = {}
for attName, att in obj.__dict__.items():
id_att_newVals[id][attName] = att
#get old values
id_att_oldVals = {}
for fN in bioLibCG.recurseDir(self.aDir):
baseName = fN.strip().split('/')[-1]
if baseName.startswith('a.'):
#get atribute name/type
bs = baseName.split('.')
attName = bs[1]
attType = attName_field[attName].dataType
casteFxn = getCasteFunction(attType)
#now set the attributes
f = open(fN, 'r')
for line in f:
ls = line.strip().split('\t')
id = int(ls[0])
att = ls[1]
#caste them correctly
if attType in listTypes:
att = att.split(',')
att = [casteFxn(x) for x in att]
else:
att = casteFxn(att)
if id not in id_att_oldVals: id_att_oldVals[id] = {}
id_att_oldVals[id][attName] = att
f.close()
#consolidate
id_att_finalVals = {}
for id in id_att_oldVals:
for attName in id_att_oldVals[id]:
#check default...I don't think you have to do this because it is from file...no defaults...
#if id_att_oldVals[id][attName] == attName_field[attName].dataDefault:
#continue
#replace it
if id in id_att_newVals:
if attName in id_att_newVals[id]:
if id not in id_att_finalVals:
id_att_finalVals[id] = {}
id_att_finalVals[id][attName] = id_att_newVals[id][
attName]
else:
if id not in id_att_finalVals:
id_att_finalVals[id] = {}
id_att_finalVals[id][attName] = id_att_oldVals[id][
attName]
else:
if id not in id_att_finalVals: id_att_finalVals[id] = {}
id_att_finalVals[id][attName] = id_att_oldVals[id][attName]
for id in id_att_newVals:
for attName in id_att_newVals[id]:
#check default
#if id_att_newVals[id][attName] == attName_field[attName].dataDefault:
#continue
if id in id_att_oldVals:
if attName in id_att_oldVals[id]:
continue #already took care of this above
else:
if id not in id_att_finalVals:
id_att_finalVals[id] = {}
id_att_finalVals[id][attName] = id_att_newVals[id][
attName]
else:
if id not in id_att_finalVals: id_att_finalVals[id] = {}
id_att_finalVals[id][attName] = id_att_newVals[id][attName]
#write to files
for attName in selectedAttNames:
listFlag = False
if attName_field[attName].dataType in listTypes:
listFlag = True
f = open(self.aDir + '/a.' + attName, 'w')
for id in id_att_finalVals:
if attName in id_att_finalVals[id]:
finalVal = id_att_finalVals[id][attName]
if finalVal == attName_field[attName].dataDefault:
continue
if listFlag:
finalVal = ','.join([str(x) for x in finalVal])
f.write('%s\t%s\n' % (id, finalVal))
f.close()
listFlag = False
#using the continuos blocks from in the small RNA lib file, identify all the peaks...
import bioLibCG as cg
import cgConfig as c
mConf = c.cgConfig('Main.conf')
fileNames = cg.recurseDir(mConf.conf['wigMouse'], end='.wig')
for fN in fileNames:
file = open(fN, 'r')
file.readline() #header
#get all points in midpoint form
pointsDict = {}
for line in file:
start = int(cg.ss(line)[1])
end = int(cg.ss(line)[2])
point = start + (end - start) / 2 #midpoint
pointsDict[point] = int(cg.ss(line)[3].split('.')[0])
file.close()
#determine peaks based off of neighbors of each point
lowest = pointsDict.keys()
lowest.sort()
peaks = []
span = 2 #must be > 0
for i in range(span + 1, len(lowest) - span - 1):
val = pointsDict[lowest[i]]
if val < 5: #minimum
def updateReadDensity(tType):
#go through wig each chromosome and check the mature seqs
mainConf = cgConfig.cgConfig('Main.conf')
conf = cgConfig.cgConfig()
organism = conf.conf['organism']
wigFolder = mainConf.conf['wig%s' % organism]
newLines = []
if tType == 'E':
pFileName = conf.conf['resultsExons']
elif tType == 'I':
pFileName = conf.conf['resultsIntrons']
else:
print 'READ UPDATE FAIL'
print ' Updating Read Density:', tType
for wigFileN in cg.recurseDir(wigFolder, end='.wig'):
#init
chrom = wigFileN.strip().split('.')[-2]
strand = wigFileN.strip().split('.')[-4]
wigFile = open(wigFileN, 'r')
mirFile = open(pFileName, 'r')
print wigFileN
#get rid of header
wigFile.readline()
print ' populating hitmap'
#populate hitmap
wigMap = {}
for line in wigFile:
value = int(line.strip().split('\t')[3].split('.')[0])
if value > 0:
start = int(line.strip().split('\t')[1])
end = int(line.strip().split('\t')[2])
for i in range(start, end):
wigMap[i] = value
wigFile.close()
print ' calculating hits for mature seqs'
#calculate total hits per mature
for line in mirFile:
mTcc = line.strip().split('\t')[1]
mirID = line.strip().split('\t')[0]
if (mTcc.split(':')[0] == chrom) and (mTcc.split(':')[1]
== strand):
#if mirID == '26477.30.106643972': print 'Starting Total Count'
highestHit = 0
for i in range(int(mTcc.split(':')[2]),
int(mTcc.split(':')[3])):
#if mirID == '26477.30.106643972': print ' ', i
if i in wigMap:
if wigMap[i] > highestHit:
highestHit = wigMap[i]
#if mirID == '26477.30.106643972': print ' ', i, totalHits, wigMap[i]
newLines.append(cg.appendToLine(line, str(highestHit), 11))
mirFile.close()
print 'Writing New File'
#write new results file
outFile = open(pFileName, 'w')
for line in newLines:
outFile.write(line)
outFile.close()
####NOW UPDATE HIGHEST HIT PER CLUSTER####
clusterCount = {}
pFile = open(pFileName, 'r')
for line in pFile:
predictionCount = int(line.strip().split('\t')[11])
CID = line.strip().split('\t')[7]
if CID in clusterCount:
if clusterCount[CID] < predictionCount:
clusterCount[CID] = predictionCount
else:
clusterCount[CID] = predictionCount
pFile.close()
#update the file --> cluster small count
newLines = []
predFile = open(pFileName, 'r')
for line in predFile:
CID = line.strip().split('\t')[7]
numMax = clusterCount[CID]
newLines.append(cg.appendToLine(line, str(numMax), 12))
predFile.close()
#sort newLines by clusterID
sortDict = {}
CIDs = []
for line in newLines:
CID = int(line.strip().split('\t')[7])
if CID not in CIDs:
CIDs.append(CID)
if CID in sortDict:
sortDict[CID].append(line)
else:
sortDict[CID] = [line]
CIDs.sort()
newLines = []
for CID in CIDs:
for line in sortDict[CID]:
newLines.append(line)
#write new File
newFile = open(pFileName, 'w')
for line in newLines:
newFile.write(line)
newFile.close()
#using the continuos blocks from in the small RNA lib file, identify all the peaks...
import bioLibCG as cg
import cgConfig as c
mConf = c.cgConfig('Main.conf')
fileNames = cg.recurseDir(mConf.conf['wigMouse'], end = '.wig')
for fN in fileNames:
file = open(fN, 'r')
file.readline() #header
#get all points in midpoint form
pointsDict = {}
for line in file:
start = int(cg.ss(line)[1])
end = int(cg.ss(line)[2])
point = start + (end-start)/2 #midpoint
pointsDict[point] = int(cg.ss(line)[3].split('.')[0])
file.close()
#determine peaks based off of neighbors of each point
lowest = pointsDict.keys()
lowest.sort()
peaks = []
span = 2 #must be > 0
for i in range(span + 1,len(lowest) - span - 1):
val = pointsDict[lowest[i]]
import bioLibCG as cg
import os
direc = '/home/chrisgre/apps/projects/small.rna.libs'
metaFileName = direc + '/' + 'small.meta'
##make data of already made file so there aren't any duplicates:
fileDict = {} #filename...
metaFile = open(metaFileName, 'r')
#add new entries
countFiles = cg.recurseDir(direc, end='.fastq')
for file in countFiles:
fileName = file.strip().split('/')[-1]
if len(fileName.split('.')) > 2: #has to specifically end in fastq...
continue
fileName = cg.getBaseFileName(file, naked=True)
dir = file.strip().split('/')[-2]
org = 'NONE'
if 'human' in dir:
org = 'human'
if 'mouse' in dir:
org = 'mouse'
if 'pig' in dir:
org = 'pig'
if 'dog' in dir:
org = 'dog'
if 'rat' in dir:
org = 'rat'
###This will calculate the total number of hits per microRNA --> update
##the newResults file with them --> put the value of the highest mature --> CID
import cgConfig
import bioLibCG as cg
#go through wig each chromosome and check the mature seqs
mainConf = cgConfig.cgConfig('Main.conf')
conf = cgConfig.cgConfig()
newLines = []
pFileName = 'newResults.results'
for wigFileN in cg.recurseDir(mainConf.conf['wigMouse'], end = '.wig'):
#init
chrom = wigFileN.strip().split('.')[-2]
strand = wigFileN.strip().split('.')[-4]
wigFile = open(wigFileN, 'r')
mirFile = open(conf.conf['results'], 'r')
print wigFileN
#get rid of header
wigFile.readline()
print ' populating hitmap'
#populate hitmap
wigMap = {}
for line in wigFile:
value = int(line.strip().split('\t')[3].split('.')[0])
def commit(self, id_obj):
#get schema
attName_field = getClassScheme(self.mappingClass)
#will update this functionality later
selectedAttNames = [x for x in attName_field]
#Get new values...Ideally this would only have the ones that have changed...but it doesn't...
id_att_newVals = {}
for id, obj in id_obj.items():
id_att_newVals[id] = {}
for attName, att in obj.__dict__.items():
id_att_newVals[id][attName] = att
#get old values
id_att_oldVals = {}
for fN in bioLibCG.recurseDir(self.aDir):
baseName = fN.strip().split('/')[-1]
if baseName.startswith('a.'):
#get atribute name/type
bs = baseName.split('.')
attName = bs[1]
attType = attName_field[attName].dataType
casteFxn = getCasteFunction(attType)
#now set the attributes
f = open(fN, 'r')
for line in f:
ls = line.strip().split('\t')
id = int(ls[0])
att = ls[1]
#caste them correctly
if attType in listTypes:
att = att.split(',')
att = [casteFxn(x) for x in att]
else:
att = casteFxn(att)
if id not in id_att_oldVals: id_att_oldVals[id] = {}
id_att_oldVals[id][attName] = att
f.close()
#consolidate
id_att_finalVals = {}
for id in id_att_oldVals:
for attName in id_att_oldVals[id]:
#check default...I don't think you have to do this because it is from file...no defaults...
#if id_att_oldVals[id][attName] == attName_field[attName].dataDefault:
#continue
#replace it
if id in id_att_newVals:
if attName in id_att_newVals[id]:
if id not in id_att_finalVals: id_att_finalVals[id] = {}
id_att_finalVals[id][attName] = id_att_newVals[id][attName]
else:
if id not in id_att_finalVals: id_att_finalVals[id] = {}
id_att_finalVals[id][attName] = id_att_oldVals[id][attName]
else:
if id not in id_att_finalVals: id_att_finalVals[id] = {}
id_att_finalVals[id][attName] = id_att_oldVals[id][attName]
for id in id_att_newVals:
for attName in id_att_newVals[id]:
#check default
#if id_att_newVals[id][attName] == attName_field[attName].dataDefault:
#continue
if id in id_att_oldVals:
if attName in id_att_oldVals[id]:
continue #already took care of this above
else:
if id not in id_att_finalVals: id_att_finalVals[id] = {}
id_att_finalVals[id][attName] = id_att_newVals[id][attName]
else:
if id not in id_att_finalVals: id_att_finalVals[id] = {}
id_att_finalVals[id][attName] = id_att_newVals[id][attName]
#write to files
for attName in selectedAttNames:
listFlag = False
if attName_field[attName].dataType in listTypes:
listFlag = True
f = open(self.aDir + '/a.' + attName, 'w')
for id in id_att_finalVals:
if attName in id_att_finalVals[id]:
finalVal = id_att_finalVals[id][attName]
if finalVal == attName_field[attName].dataDefault:
continue
if listFlag:
finalVal = ','.join([str(x) for x in finalVal])
f.write('%s\t%s\n' % (id, finalVal))
f.close()
listFlag = False
def clipAdapterInDirQ(dirName): '''The Q if for doing it on a cluster using qsub Every Q function has a corresponding shell script''' for file in cg.recurseDir(dirName, end = '.fastq'): subprocess.Popen([wrapperShell, file])
import cgConfig as c
import wigValue
import compareData as compare
#init
mConf = c.cgConfig('Main.conf')
conf = c.cgConfig()
tccList = ['chr3:-1:96042576:96042685']
#tccList = compare.tccFileToList('mouseKnownMirs.tcc', 0)
timer = cg.cgTimer()
timer.start()
#put peaks in memory
print 'loading peak data'
peakFilesNames = cg.recurseDir(mConf.conf['wigMouse'], end = '.peaks')
peaks = {} # chr:peak:value
for pN in peakFilesNames:
chrom = pN.strip().split('.')[4]
strand = pN.strip().split('.')[2]
#init dictionary
if chrom not in peaks:
peaks[chrom] = {}
if strand not in peaks[chrom]:
peaks[chrom][strand] = {}
#get peaks and values and put in dictionary
pFile = open(pN, 'r')
for line in pFile:
#using the continuos blocks from in the small RNA lib file, identify all the peaks...
import bioLibCG as cg
import cgConfig as c
mConf = c.cgConfig('Main.conf')
fileNames = cg.recurseDir(mConf.conf['wigMouse'], start = 'cb.', end = '.tsv')
for fN in fileNames:
file = open(fN, 'r')
#get peak positions and average heights
lines = [str(int(x.split('\t')[0]) + int(x.split('\t')[2])/2) + '\t' + str(x.strip().split('\t')[3]) for x in file]
file.close()
#output new file with new ending
outFile = open(fN + '.peakdata', 'w')
for line in lines:
outFile.write('%s\n' % line)
def clipAdapterInDirQ(dirName):
'''The Q if for doing it on a cluster using qsub
Every Q function has a corresponding shell script'''
for file in cg.recurseDir(dirName, end='.fastq'):
subprocess.Popen([wrapperShell, file])
import cgConfig as c
import wigValue
import compareData as compare
#init
mConf = c.cgConfig('Main.conf')
conf = c.cgConfig()
tccList = ['chr3:-1:96042576:96042685']
#tccList = compare.tccFileToList('mouseKnownMirs.tcc', 0)
timer = cg.cgTimer()
timer.start()
#put peaks in memory
print 'loading peak data'
peakFilesNames = cg.recurseDir(mConf.conf['wigMouse'], end='.peaks')
peaks = {} # chr:peak:value
for pN in peakFilesNames:
chrom = pN.strip().split('.')[4]
strand = pN.strip().split('.')[2]
#init dictionary
if chrom not in peaks:
peaks[chrom] = {}
if strand not in peaks[chrom]:
peaks[chrom][strand] = {}
#get peaks and values and put in dictionary
pFile = open(pN, 'r')
for line in pFile:
import bioLibCG as cg
import os
direc = '/home/chrisgre/apps/projects/small.rna.libs'
metaFileName = direc + '/' + 'small.meta'
##make data of already made file so there aren't any duplicates:
fileDict = {} #filename...
metaFile = open(metaFileName, 'r')
#add new entries
countFiles = cg.recurseDir(direc, end = '.fastq')
for file in countFiles:
fileName = file.strip().split('/')[-1]
if len(fileName.split('.')) > 2: #has to specifically end in fastq...
continue
fileName = cg.getBaseFileName(file, naked = True)
dir = file.strip().split('/')[-2]
org = 'NONE'
if 'human' in dir:
org = 'human'
if 'mouse' in dir:
org = 'mouse'
if 'pig' in dir:
org = 'pig'
if 'dog' in dir:
org = 'dog'
if 'rat' in dir:
org = 'rat'
import cgConfig as c
import bioLibCG as cg
import cgSort
mConf = c.cgConfig('Main.conf')
smallPath = mConf.conf['smallPath']
smallPath = '/home/chrisgre/smallLibs/WIGS/zebrafish'
#grab everything - NOT WIG MERGES...
smallLibs = cg.recurseDir(smallPath, end = '.wig')
smallLibs.extend(cg.recurseDir(smallPath, end = '.wig'))
for lib in smallLibs:
print 'sorting', lib
cgSort.wigSort(lib)
def updateReadDensity(tType):
#go through wig each chromosome and check the mature seqs
mainConf = cgConfig.cgConfig('Main.conf')
conf = cgConfig.cgConfig()
organism = conf.conf['organism']
wigFolder = mainConf.conf['wig%s' % organism]
newLines = []
if tType == 'E':
pFileName = conf.conf['resultsExons']
elif tType == 'I':
pFileName = conf.conf['resultsIntrons']
else:
print 'READ UPDATE FAIL'
print ' Updating Read Density:', tType
for wigFileN in cg.recurseDir(wigFolder, end = '.wig'):
#init
chrom = wigFileN.strip().split('.')[-2]
strand = wigFileN.strip().split('.')[-4]
wigFile = open(wigFileN, 'r')
mirFile = open(pFileName, 'r')
print wigFileN
#get rid of header
wigFile.readline()
print ' populating hitmap'
#populate hitmap
wigMap = {}
for line in wigFile:
value = int(line.strip().split('\t')[3].split('.')[0])
if value > 0:
start = int(line.strip().split('\t')[1])
end = int(line.strip().split('\t')[2])
for i in range(start, end):
wigMap[i] = value
wigFile.close()
print ' calculating hits for mature seqs'
#calculate total hits per mature
for line in mirFile:
mTcc = line.strip().split('\t')[1]
mirID = line.strip().split('\t')[0]
if (mTcc.split(':')[0] == chrom) and (mTcc.split(':')[1] == strand):
#if mirID == '26477.30.106643972': print 'Starting Total Count'
highestHit = 0
for i in range(int(mTcc.split(':')[2]), int(mTcc.split(':')[3])):
#if mirID == '26477.30.106643972': print ' ', i
if i in wigMap:
if wigMap[i] > highestHit:
highestHit = wigMap[i]
#if mirID == '26477.30.106643972': print ' ', i, totalHits, wigMap[i]
newLines.append(cg.appendToLine(line, str(highestHit), 11))
mirFile.close()
print 'Writing New File'
#write new results file
outFile = open(pFileName, 'w')
for line in newLines:
outFile.write(line)
outFile.close()
####NOW UPDATE HIGHEST HIT PER CLUSTER####
clusterCount = {}
pFile = open(pFileName, 'r')
for line in pFile:
predictionCount = int(line.strip().split('\t')[11])
CID = line.strip().split('\t')[7]
if CID in clusterCount:
if clusterCount[CID] < predictionCount:
clusterCount[CID] = predictionCount
else:
clusterCount[CID] = predictionCount
pFile.close()
#update the file --> cluster small count
newLines = []
predFile = open(pFileName, 'r')
for line in predFile:
CID = line.strip().split('\t')[7]
numMax = clusterCount[CID]
newLines.append(cg.appendToLine(line, str(numMax), 12))
predFile.close()
#sort newLines by clusterID
sortDict = {}
CIDs = []
for line in newLines:
CID = int(line.strip().split('\t')[7])
if CID not in CIDs:
CIDs.append(CID)
if CID in sortDict:
sortDict[CID].append(line)
else:
sortDict[CID] = [line]
CIDs.sort()
newLines = []
for CID in CIDs:
for line in sortDict[CID]:
newLines.append(line)
#write new File
newFile = open(pFileName, 'w')
for line in newLines:
newFile.write(line)
newFile.close()
import bioLibCG as cg import cgRnaSeq dir = '/home/chrisgre/smallLibs/zebrafish.embryo.GSE22068.20620952' QNames = cg.recurseDir(dir, end = '.fastq') #fastqfile names slNames = cg.recurseDir(dir, end = '.txt') #single sequence per line file names fastNames = cg.recurseDir(dir, end = '.fa') fastNames.extend(cg.recurseDir(dir, end = '.fna')) for QFileName in QNames: print 'Creating counts for file', QFileName cgRnaSeq.createCountFileFastQ(QFileName) for slName in slNames: print 'Creating counts for file', slName cgRnaSeq.createCountFileSL(slName) for fastName in fastNames: print 'Creating counts for file', fastName cgRnaSeq.createCountFileFasta(fastName)
def createMultiTrack(dirName, organism):
'''merge all mapped tracks in directory and create a single wig file'''
mainConf = c.cgConfig('Main.conf')
metaFileName = mainConf.conf['metaFileName']
fileList = []
for file in cg.recurseDir(dirName, end = '.mapped'):
#check if mouse or human SHOULD PUT INTO A STD FUNCTION FOR META FILE
#check if mouse or human
baseFName = cg.getBaseFileName(file, naked= True)
metaDict = cg.getMetaFileDict(metaFileName)
org = 'None'
if baseFName in metaDict:
if metaDict[baseFName][1] == 'NONE':
print ' NO ORG KNOWN FOR', file
continue
elif not metaDict[baseFName][1] == organism:
print ' NOT ORGANISM RUNNING', file
continue
else:
org = metaDict[baseFName][1]
print ' USING ORG', org, file
#check if there is an organism, must check due to files not in metaFile
if org == 'None':
print ' NO org (not in meta file)', file
continue
#only make wig file for organism asked for
if not org == organism:
continue
#if it is right organism and has mapped file then add
fileList.append(file)
#make merged wig
if organism == 'human':
chroms = cg.humanChromosomes
assembly = 'hg19'
elif organism == 'mouse':
chroms = cg.mouseChromosomes
assembly = 'mm9'
elif organism == 'zebrafish':
chroms = cg.zebrafishChromosomes
assembly = 'danRer6'
print 'Making Bed File vectors'
cvg = HTSeq.GenomicArray(chroms, stranded=True, typecode='i')
for fName in fileList:
alignment_file = HTSeq.BowtieReader(fName)
for alngt in alignment_file:
if alngt.aligned:
cvg.add_value( 1, alngt.iv ) #iv is the genomic interval..
bedNamePos = dirName + '/Merge.' + organism + '.1.wig'
bedNameNeg = dirName + '/Merge.' + organism + '.-1.wig'
print 'Writing Bed File'
cvg.write_bedgraph_file(bedNamePos, "+" )
cvg.write_bedgraph_file(bedNameNeg, "-" )
#Now extend it
updateWigLength(bedNamePos, assembly)
updateWigLength(bedNameNeg, assembly)
#Now Sort it.
cgSort.wigSort(bedNamePos)
cgSort.wigSort(bedNameNeg)
