def run(config, job_uuid, genes, geneId, seedModels, wobble, cut, motifSizes, jobName, mirbase_species, bgModel, topRet=10, viral=False):
species = get_species_by_mirbase_id(mirbase_species)
if bgModel=='3p':
bgModel = species['weeder']
else:
bgModel = species['weeder'].rstrip('3P')
sequence_file = os.path.join(config.get('General', 'data_dir'),
"p3utrSeqs_" + species['ucsc_name'] + ".csv")
cut = float(cut)
curRunNum = randint(0,1000000)
# translate gene identifiers to entrez IDs
print "translating gene identifiers from %s to entrez IDs" % (geneId)
genes = map_genes_to_entrez_ids(job_uuid, geneId, mirbase_species)
print "genes = " + str(genes)
# 1. Read in sequences
seqFile = open(sequence_file,'r')
seqLines = seqFile.readlines()
ids = [i.strip().split(',')[0].upper() for i in seqLines]
sequences = [i.strip().split(',')[1] for i in seqLines]
seqs = dict(zip(ids,sequences))
seqFile.close()
# 2. Get sequences for each target
miRSeqs = {}
for gene in genes:
if gene in seqs:
miRSeqs[gene] = seqs[gene]
# if there are no matching sequences, bail out w/ a reasonable error message.
if (len(miRSeqs)==0):
print("no matching sequences found for genes in job " + str(job_uuid))
update_job_status(job_uuid, "error", "No sequences found for the genes entered.")
return False
# record whether a sequence was found for each gene
# previously stored when job was created (create_job_in_db)
set_genes_annotated(job_uuid, miRSeqs)
# 3. Make a FASTA file
fasta_dir = os.path.join(config.get('General', 'tmp_dir'), 'fasta')
if not os.path.exists(fasta_dir):
os.makedirs(fasta_dir)
fasta_fname = os.path.join(fasta_dir, 'tmp' + str(curRunNum) + '.fasta')
with open(fasta_fname, 'w') as fastaFile:
for seq in miRSeqs:
fastaFile.write('>'+str(seq)+'\n'+str(miRSeqs[seq])+'\n')
# 4. Run weeder
print 'Running weeder!'
update_job_status(job_uuid, "running weeder")
weederPSSMs1 = weeder(config,
seqFile=fasta_fname,
percTargets=50,
revComp=False,
bgModel=bgModel)
# 4a. Take only selected size motifs
weederPSSMsTmp = []
for pssm1 in weederPSSMs1:
png_path = os.path.join(config.get('General', 'pssm_images_dir'),
str(job_uuid) + '_' + pssm1.getName() + '.png')
if 6 in motifSizes and len(pssm1.getName())==6:
weederPSSMsTmp.append(deepcopy(pssm1))
plotPssm(pssm1, png_path)
if 8 in motifSizes and len(pssm1.getName())==8:
weederPSSMsTmp.append(deepcopy(pssm1))
plotPssm(pssm1, png_path)
print("pssm name = " + pssm1.getName())
weederPSSMs1 = deepcopy(weederPSSMsTmp)
del weederPSSMsTmp
# 5. Run miRvestigator HMM
update_job_status(job_uuid, "computing miRvestigator HMM")
mV = miRvestigator(config, weederPSSMs1, seqs.values(),
seedModel=seedModels,
minor=True,
p5=True, p3=True,
wobble=wobble, wobbleCut=cut,
textOut=False,
species=mirbase_species,
viral = viral)
# 6. Clean-up after yerself
os.remove(os.path.join(fasta_dir, 'tmp' + str(curRunNum) + '.fasta'))
os.remove(os.path.join(fasta_dir, 'tmp' + str(curRunNum) + '.fasta.wee'))
os.remove(os.path.join(fasta_dir, 'tmp' + str(curRunNum) + '.fasta.mix'))
os.remove(os.path.join(fasta_dir, 'tmp' + str(curRunNum) + '.fasta.html'))
# 7. write output to database
update_job_status(job_uuid, "compiling results")
for pssm in weederPSSMs1:
motif_id = store_motif(job_uuid, pssm)
scores = mV.getScoreList(pssm.getName())
store_mirvestigator_scores(motif_id, scores)
update_job_status(job_uuid, "done")
return True
def run(job_uuid, genes, geneId, seedModels, wobble, cut, motifSizes, jobName, mirbase_species, bgModel, topRet=10, viral=False):
species = get_species_by_mirbase_id(mirbase_species)
if bgModel=='3p':
bgModel = species['weeder']
else:
bgModel = species['weeder'].rstrip('3P')
sequence_file = conf.data_dir+"/p3utrSeqs_" + species['ucsc_name'] + ".csv"
cut = float(cut)
curRunNum = randint(0,1000000)
# translate gene identifiers to entrez IDs
print "translating gene identifiers from %s to entrez IDs" % (geneId)
genes = map_genes_to_entrez_ids(job_uuid, geneId, mirbase_species)
print "genes = " + str(genes)
# 1. Read in sequences
seqFile = open(sequence_file,'r')
seqLines = seqFile.readlines()
ids = [i.strip().split(',')[0].upper() for i in seqLines]
sequences = [i.strip().split(',')[1] for i in seqLines]
seqs = dict(zip(ids,sequences))
seqFile.close()
#update_job_status(job, "finished reading sequence file")
# 2. Get sequences for each target
miRSeqs = {}
for gene in genes:
if gene in seqs:
miRSeqs[gene] = seqs[gene]
# if there are no matching sequences, bail out w/ a reasonable error message.
if (len(miRSeqs)==0):
print("no matching sequences found for genes in job " + str(job_uuid))
update_job_status(job_uuid, "error", "No sequences found for the genes entered.")
return False
# record whether a sequence was found for each gene
# previously stored when job was created (create_job_in_db)
set_genes_annotated(job_uuid, miRSeqs)
# 3. Make a FASTA file
if not os.path.exists(conf.tmp_dir+'/fasta'):
os.makedirs(conf.tmp_dir+'/fasta')
fastaFile = open(conf.tmp_dir+'/fasta/tmp'+str(curRunNum)+'.fasta','w')
for seq in miRSeqs:
fastaFile.write('>'+str(seq)+'\n'+str(miRSeqs[seq])+'\n')
fastaFile.close()
# 4. Run weeder
print 'Running weeder!'
update_job_status(job_uuid, "running weeder")
weederPSSMs1 = weeder(seqFile=conf.tmp_dir+'/fasta/tmp'+str(curRunNum)+'.fasta', percTargets=50, revComp=False, bgModel=bgModel)
# 4a. Take only selected size motifs
weederPSSMsTmp = []
for pssm1 in weederPSSMs1:
if 6 in motifSizes and len(pssm1.getName())==6:
weederPSSMsTmp.append(deepcopy(pssm1))
plotPssm(pssm1,conf.pssm_images_dir+'/'+str(job_uuid)+'_'+pssm1.getName()+'.png')
if 8 in motifSizes and len(pssm1.getName())==8:
weederPSSMsTmp.append(deepcopy(pssm1))
plotPssm(pssm1,conf.pssm_images_dir+'/'+str(job_uuid)+'_'+pssm1.getName()+'.png')
print("pssm name = " + pssm1.getName())
weederPSSMs1 = deepcopy(weederPSSMsTmp)
del weederPSSMsTmp
# 5. Run miRvestigator HMM
update_job_status(job_uuid, "computing miRvestigator HMM")
mV = miRvestigator(weederPSSMs1, seqs.values(),
seedModel=seedModels,
minor=True,
p5=True, p3=True,
wobble=wobble, wobbleCut=cut,
textOut=False,
species=mirbase_species,
viral = viral)
# 6. Read in miRNAs to get mature miRNA ids
# import gzip
# miRNAFile = gzip.open('mature.fa.gz','r')
# miRNADict = {}
# while 1:
# miRNALine = miRNAFile.readline()
# seqLine = miRNAFile.readline()
# if not miRNALine:
# break
# # Get the miRNA name
# miRNAData = miRNALine.lstrip('>').split(' ')
# curMiRNA = miRNAData[0]
# if (curMiRNA.split('-'))[0]=='hsa':
# miRNADict[curMiRNA] = miRNAData[1]
# miRNAFile.close()
# 6. Clean-up after yerself
os.remove(conf.tmp_dir+'/fasta/tmp'+str(curRunNum)+'.fasta')
os.remove(conf.tmp_dir+'/fasta/tmp'+str(curRunNum)+'.fasta.wee')
os.remove(conf.tmp_dir+'/fasta/tmp'+str(curRunNum)+'.fasta.mix')
os.remove(conf.tmp_dir+'/fasta/tmp'+str(curRunNum)+'.fasta.html')
# 7. write output to database
update_job_status(job_uuid, "compiling results")
for pssm in weederPSSMs1:
motif_id = store_motif(job_uuid, pssm)
scores = mV.getScoreList(pssm.getName())
store_mirvestigator_scores(motif_id, scores)
update_job_status(job_uuid, "done")
return True
for seq in clusterSeqs:
fastaFile.write('>'+str(seq)+'\n'+str(clusterSeqs[seq])+'\n')
fastaFile.close()
# Run this using all cores available
weederPSSMs1 = mgr.list()
print 'Starting Weeder runs...'
cpus = cpu_count()
print 'There are', cpus,'CPUs avialable.'
pool = Pool(processes=cpus)
pool.map(runWeeder,range(len(fastaFiles)))
print 'Done with Weeder runs.\n'
# Compare to miRDB using my program
from miRvestigator import miRvestigator
m2m = miRvestigator(weederPSSMs1,seqs.values(),seedModel=[6,7,8],minor=True,p5=True,p3=True,wobble=False,wobbleCut=0.25)
outFile = open('m2m'+'_'+str(dataset)+'.pkl','wb')
cPickle.dump(m2m,outFile)
outFile.close()
# Now do PITA and TargetScan - iterate through both platforms
for db in ['TargetScan','PITA']:
# Get ready for multiprocessor goodness
mgr = Manager()
cpus = cpu_count()
# Load up db of miRNA ids
ls2 = [x for x in os.listdir('TargetPredictionDatabases/'+db) if '.csv' in x]
# Load the predicted target genes for each miRNA from the files
tmpDict = {}
def run_mirvestigator(seqs, refSeq2entrez, use_entrez, exp_dir='exp'):
"""
Run miRvestigator on all signatures
"""
global fastaFiles, weederPSSMs1
# Setup for multiprocessing
mgr = Manager()
fastaFiles = mgr.list()
# For each cluster file in exp from Goodarzi et al.
# Cluster files should have a header and be tab delimited to look like this:
# Gene\tGroup\n
# NM_000014\t52\n
# <RefSeq_ID>\t<signature_id>\n
# ...
files = os.listdir(exp_dir)
for file in files:
# 3. Read in cluster file and convert to entrez ids
with open(os.path.join(exp_dir, file), 'r') as inFile:
dataset = file.strip().split('.')[0]
inFile.readline()
lines = inFile.readlines()
clusters = defaultdict(list)
for line in lines:
gene, group = line.strip().split('\t')
group = int(group)
if use_entrez:
entrez = int(gene)
clusters[group].append(entrez)
else:
if gene in refSeq2entrez:
clusters[group].append(refSeq2entrez[gene])
# 5. Make a FASTA file & run weeder
for cluster in clusters:
print(cluster)
# Get seqeunces
clusterSeqs = {}
for target in clusters[cluster]:
if str(target) in seqs:
clusterSeqs[target] = seqs[str(target)]
else:
print("Did not find seq for '%s'" % target)
# Make FASTA file
print('Fasta output...')
fastaFiles.append('tmp/weeder/fasta/' + str(cluster) + '_' +
str(dataset) + '.fasta')
if not os.path.exists('tmp/weeder/fasta'):
os.makedirs('tmp/weeder/fasta')
with open(
'tmp/weeder/fasta/' + str(cluster) + '_' + str(dataset) +
'.fasta', 'w') as fastaFile:
for seq in clusterSeqs:
fastaFile.write('>' + str(seq) + '\n' +
str(clusterSeqs[seq]) + '\n')
# Run this using all cores available
weederPSSMs1 = mgr.list()
print('Starting Weeder runs...')
cpus = cpu_count()
print('There are %d CPUs available.' % cpus)
pool = Pool(processes=cpus)
pool.map(runWeeder, range(len(fastaFiles)))
print('Done with Weeder runs.')
# Compare to miRDB using my program
m2m = miRvestigator(weederPSSMs1,
seqs.values(),
seedModel=[6, 7, 8],
minor=True,
p5=True,
p3=True,
wobble=False,
wobbleCut=0.25)
with open('m2m' + '_' + str(dataset) + '.pkl', 'wb') as outFile:
pickle.dump(m2m, outFile)