def run(config, job_uuid, genes, geneId, seedModels, wobble, cut, motifSizes, jobName, mirbase_species, bgModel, topRet=10, viral=False): species = get_species_by_mirbase_id(mirbase_species) if bgModel=='3p': bgModel = species['weeder'] else: bgModel = species['weeder'].rstrip('3P') sequence_file = os.path.join(config.get('General', 'data_dir'), "p3utrSeqs_" + species['ucsc_name'] + ".csv") cut = float(cut) curRunNum = randint(0,1000000) # translate gene identifiers to entrez IDs print "translating gene identifiers from %s to entrez IDs" % (geneId) genes = map_genes_to_entrez_ids(job_uuid, geneId, mirbase_species) print "genes = " + str(genes) # 1. Read in sequences seqFile = open(sequence_file,'r') seqLines = seqFile.readlines() ids = [i.strip().split(',')[0].upper() for i in seqLines] sequences = [i.strip().split(',')[1] for i in seqLines] seqs = dict(zip(ids,sequences)) seqFile.close() # 2. Get sequences for each target miRSeqs = {} for gene in genes: if gene in seqs: miRSeqs[gene] = seqs[gene] # if there are no matching sequences, bail out w/ a reasonable error message. if (len(miRSeqs)==0): print("no matching sequences found for genes in job " + str(job_uuid)) update_job_status(job_uuid, "error", "No sequences found for the genes entered.") return False # record whether a sequence was found for each gene # previously stored when job was created (create_job_in_db) set_genes_annotated(job_uuid, miRSeqs) # 3. Make a FASTA file fasta_dir = os.path.join(config.get('General', 'tmp_dir'), 'fasta') if not os.path.exists(fasta_dir): os.makedirs(fasta_dir) fasta_fname = os.path.join(fasta_dir, 'tmp' + str(curRunNum) + '.fasta') with open(fasta_fname, 'w') as fastaFile: for seq in miRSeqs: fastaFile.write('>'+str(seq)+'\n'+str(miRSeqs[seq])+'\n') # 4. Run weeder print 'Running weeder!' update_job_status(job_uuid, "running weeder") weederPSSMs1 = weeder(config, seqFile=fasta_fname, percTargets=50, revComp=False, bgModel=bgModel) # 4a. Take only selected size motifs weederPSSMsTmp = [] for pssm1 in weederPSSMs1: png_path = os.path.join(config.get('General', 'pssm_images_dir'), str(job_uuid) + '_' + pssm1.getName() + '.png') if 6 in motifSizes and len(pssm1.getName())==6: weederPSSMsTmp.append(deepcopy(pssm1)) plotPssm(pssm1, png_path) if 8 in motifSizes and len(pssm1.getName())==8: weederPSSMsTmp.append(deepcopy(pssm1)) plotPssm(pssm1, png_path) print("pssm name = " + pssm1.getName()) weederPSSMs1 = deepcopy(weederPSSMsTmp) del weederPSSMsTmp # 5. Run miRvestigator HMM update_job_status(job_uuid, "computing miRvestigator HMM") mV = miRvestigator(config, weederPSSMs1, seqs.values(), seedModel=seedModels, minor=True, p5=True, p3=True, wobble=wobble, wobbleCut=cut, textOut=False, species=mirbase_species, viral = viral) # 6. Clean-up after yerself os.remove(os.path.join(fasta_dir, 'tmp' + str(curRunNum) + '.fasta')) os.remove(os.path.join(fasta_dir, 'tmp' + str(curRunNum) + '.fasta.wee')) os.remove(os.path.join(fasta_dir, 'tmp' + str(curRunNum) + '.fasta.mix')) os.remove(os.path.join(fasta_dir, 'tmp' + str(curRunNum) + '.fasta.html')) # 7. write output to database update_job_status(job_uuid, "compiling results") for pssm in weederPSSMs1: motif_id = store_motif(job_uuid, pssm) scores = mV.getScoreList(pssm.getName()) store_mirvestigator_scores(motif_id, scores) update_job_status(job_uuid, "done") return True
def run(job_uuid, genes, geneId, seedModels, wobble, cut, motifSizes, jobName, mirbase_species, bgModel, topRet=10, viral=False): species = get_species_by_mirbase_id(mirbase_species) if bgModel=='3p': bgModel = species['weeder'] else: bgModel = species['weeder'].rstrip('3P') sequence_file = conf.data_dir+"/p3utrSeqs_" + species['ucsc_name'] + ".csv" cut = float(cut) curRunNum = randint(0,1000000) # translate gene identifiers to entrez IDs print "translating gene identifiers from %s to entrez IDs" % (geneId) genes = map_genes_to_entrez_ids(job_uuid, geneId, mirbase_species) print "genes = " + str(genes) # 1. Read in sequences seqFile = open(sequence_file,'r') seqLines = seqFile.readlines() ids = [i.strip().split(',')[0].upper() for i in seqLines] sequences = [i.strip().split(',')[1] for i in seqLines] seqs = dict(zip(ids,sequences)) seqFile.close() #update_job_status(job, "finished reading sequence file") # 2. Get sequences for each target miRSeqs = {} for gene in genes: if gene in seqs: miRSeqs[gene] = seqs[gene] # if there are no matching sequences, bail out w/ a reasonable error message. if (len(miRSeqs)==0): print("no matching sequences found for genes in job " + str(job_uuid)) update_job_status(job_uuid, "error", "No sequences found for the genes entered.") return False # record whether a sequence was found for each gene # previously stored when job was created (create_job_in_db) set_genes_annotated(job_uuid, miRSeqs) # 3. Make a FASTA file if not os.path.exists(conf.tmp_dir+'/fasta'): os.makedirs(conf.tmp_dir+'/fasta') fastaFile = open(conf.tmp_dir+'/fasta/tmp'+str(curRunNum)+'.fasta','w') for seq in miRSeqs: fastaFile.write('>'+str(seq)+'\n'+str(miRSeqs[seq])+'\n') fastaFile.close() # 4. Run weeder print 'Running weeder!' update_job_status(job_uuid, "running weeder") weederPSSMs1 = weeder(seqFile=conf.tmp_dir+'/fasta/tmp'+str(curRunNum)+'.fasta', percTargets=50, revComp=False, bgModel=bgModel) # 4a. Take only selected size motifs weederPSSMsTmp = [] for pssm1 in weederPSSMs1: if 6 in motifSizes and len(pssm1.getName())==6: weederPSSMsTmp.append(deepcopy(pssm1)) plotPssm(pssm1,conf.pssm_images_dir+'/'+str(job_uuid)+'_'+pssm1.getName()+'.png') if 8 in motifSizes and len(pssm1.getName())==8: weederPSSMsTmp.append(deepcopy(pssm1)) plotPssm(pssm1,conf.pssm_images_dir+'/'+str(job_uuid)+'_'+pssm1.getName()+'.png') print("pssm name = " + pssm1.getName()) weederPSSMs1 = deepcopy(weederPSSMsTmp) del weederPSSMsTmp # 5. Run miRvestigator HMM update_job_status(job_uuid, "computing miRvestigator HMM") mV = miRvestigator(weederPSSMs1, seqs.values(), seedModel=seedModels, minor=True, p5=True, p3=True, wobble=wobble, wobbleCut=cut, textOut=False, species=mirbase_species, viral = viral) # 6. Read in miRNAs to get mature miRNA ids # import gzip # miRNAFile = gzip.open('mature.fa.gz','r') # miRNADict = {} # while 1: # miRNALine = miRNAFile.readline() # seqLine = miRNAFile.readline() # if not miRNALine: # break # # Get the miRNA name # miRNAData = miRNALine.lstrip('>').split(' ') # curMiRNA = miRNAData[0] # if (curMiRNA.split('-'))[0]=='hsa': # miRNADict[curMiRNA] = miRNAData[1] # miRNAFile.close() # 6. Clean-up after yerself os.remove(conf.tmp_dir+'/fasta/tmp'+str(curRunNum)+'.fasta') os.remove(conf.tmp_dir+'/fasta/tmp'+str(curRunNum)+'.fasta.wee') os.remove(conf.tmp_dir+'/fasta/tmp'+str(curRunNum)+'.fasta.mix') os.remove(conf.tmp_dir+'/fasta/tmp'+str(curRunNum)+'.fasta.html') # 7. write output to database update_job_status(job_uuid, "compiling results") for pssm in weederPSSMs1: motif_id = store_motif(job_uuid, pssm) scores = mV.getScoreList(pssm.getName()) store_mirvestigator_scores(motif_id, scores) update_job_status(job_uuid, "done") return True
for seq in clusterSeqs: fastaFile.write('>'+str(seq)+'\n'+str(clusterSeqs[seq])+'\n') fastaFile.close() # Run this using all cores available weederPSSMs1 = mgr.list() print 'Starting Weeder runs...' cpus = cpu_count() print 'There are', cpus,'CPUs avialable.' pool = Pool(processes=cpus) pool.map(runWeeder,range(len(fastaFiles))) print 'Done with Weeder runs.\n' # Compare to miRDB using my program from miRvestigator import miRvestigator m2m = miRvestigator(weederPSSMs1,seqs.values(),seedModel=[6,7,8],minor=True,p5=True,p3=True,wobble=False,wobbleCut=0.25) outFile = open('m2m'+'_'+str(dataset)+'.pkl','wb') cPickle.dump(m2m,outFile) outFile.close() # Now do PITA and TargetScan - iterate through both platforms for db in ['TargetScan','PITA']: # Get ready for multiprocessor goodness mgr = Manager() cpus = cpu_count() # Load up db of miRNA ids ls2 = [x for x in os.listdir('TargetPredictionDatabases/'+db) if '.csv' in x] # Load the predicted target genes for each miRNA from the files tmpDict = {}
def run_mirvestigator(seqs, refSeq2entrez, use_entrez, exp_dir='exp'): """ Run miRvestigator on all signatures """ global fastaFiles, weederPSSMs1 # Setup for multiprocessing mgr = Manager() fastaFiles = mgr.list() # For each cluster file in exp from Goodarzi et al. # Cluster files should have a header and be tab delimited to look like this: # Gene\tGroup\n # NM_000014\t52\n # <RefSeq_ID>\t<signature_id>\n # ... files = os.listdir(exp_dir) for file in files: # 3. Read in cluster file and convert to entrez ids with open(os.path.join(exp_dir, file), 'r') as inFile: dataset = file.strip().split('.')[0] inFile.readline() lines = inFile.readlines() clusters = defaultdict(list) for line in lines: gene, group = line.strip().split('\t') group = int(group) if use_entrez: entrez = int(gene) clusters[group].append(entrez) else: if gene in refSeq2entrez: clusters[group].append(refSeq2entrez[gene]) # 5. Make a FASTA file & run weeder for cluster in clusters: print(cluster) # Get seqeunces clusterSeqs = {} for target in clusters[cluster]: if str(target) in seqs: clusterSeqs[target] = seqs[str(target)] else: print("Did not find seq for '%s'" % target) # Make FASTA file print('Fasta output...') fastaFiles.append('tmp/weeder/fasta/' + str(cluster) + '_' + str(dataset) + '.fasta') if not os.path.exists('tmp/weeder/fasta'): os.makedirs('tmp/weeder/fasta') with open( 'tmp/weeder/fasta/' + str(cluster) + '_' + str(dataset) + '.fasta', 'w') as fastaFile: for seq in clusterSeqs: fastaFile.write('>' + str(seq) + '\n' + str(clusterSeqs[seq]) + '\n') # Run this using all cores available weederPSSMs1 = mgr.list() print('Starting Weeder runs...') cpus = cpu_count() print('There are %d CPUs available.' % cpus) pool = Pool(processes=cpus) pool.map(runWeeder, range(len(fastaFiles))) print('Done with Weeder runs.') # Compare to miRDB using my program m2m = miRvestigator(weederPSSMs1, seqs.values(), seedModel=[6, 7, 8], minor=True, p5=True, p3=True, wobble=False, wobbleCut=0.25) with open('m2m' + '_' + str(dataset) + '.pkl', 'wb') as outFile: pickle.dump(m2m, outFile)