def functional(data): pid = data["pid"] uid = data["uid"] sids = data["sid"] pipeline = data["pip"] rid = data["rid"] edges = [] FULL_MATRIX = [] #first see if the data set contains taxonomy, function or both annotations analysis = "function" #print rid # # os.system("mkdir " + rootvar.__ROOTPRO__ + "/" + pid + "/" + pipeline + "/RESULTS/ >> " + rootvar.log + " 2>&1") all_samples_tree_file = rootvar.__ROOTPRO__ + "/" + pid + "/" + pipeline + "/RESULTS/" + rid + ".all_samples_tree.pk" # if os.path.isfile(all_samples_tree_file): x = sql.SQL(all_samples_tree_file + ".db") val = x.exe("select distinct sample_name from full_matrix") stored_samples = len(val) # x = sql.SQL(main_db) # load all the samples that have been finalized to run sids = x.exe( 'select c.sample_id from (select * from samples a inner join (select * from sample_status where pip="' + str(pipeline) + '") b on a.sample_id==b.sid) c where c.project_id=="' + pid + '" and c.status="Done" and c.rid="' + rid + '"') x.close() if os.path.isfile(all_samples_tree_file) and stored_samples == len(sids): x = sql.SQL(all_samples_tree_file + ".db") FULL_MATRIX = x.exe("select * from full_matrix") return ["function", FULL_MATRIX, all_samples_tree_file + ".json"] nodes = {} x = sql.SQL(main_db) for sid in sids: samples = x.exe('select * from samples where project_id="' + pid + '" and sample_id="' + sid[0] + '"') xpath = x.project(pid)[0][4] sample = rootvar.samples(samples[0], xpath) #print sample.name rf = rootvar.result_files(pid, analysis, pipeline, sample.id, rid) view = rootvar.ViewSampleResults(sample, pipeline, rid, analysis, rf) FULL_MATRIX += view.all_func() #print sample.name x.close() #print FULL_MATRIX[0] #rootvar.full_matrix_function(all_samples_tree_file+".db", FULL_MATRIX) #print "matrix done" tree = ["none"] return ["function", FULL_MATRIX, all_samples_tree_file + ".json"]
def get_taxo_by_name(): data = request.get_json() pid = data["pid"] uid = data["uid"] sid = data["sid"] tid = data["tid"] lid = data["lid"] pip = data["pip"] dbname = data["rid"] pipeline = pip # dbname="isuezrouja" analysis = "taxonomy" x = sql.SQL(main_db) rf = rootvar.result_files(pid, analysis, pip, sid, dbname) samples = x.exe('select * from samples where project_id="' + pid+'" and sample_id="'+sid+'"') xpath = x.project(pid)[0][4] sample = rootvar.samples(samples[0], xpath) x.close() view = rootvar.ViewSampleResults(sample, pipeline, dbname, analysis, rf) try: # return jsonify(x=pip) return jsonify(matrix=view.level(lid, type="matches")) except Exception as inst: return "ERROR: "+str(inst)
def get_tree(): try: data = request.get_json() pid = data["pid"] uid = data["uid"] sid = data["sid"] selval = data["value"] pipeline = data["pip"] dbname = data["rid"] # dbname="isuezrouja" analysis = "taxonomy" rf = rootvar.result_files(pid, analysis, pipeline, sid, dbname) x = sql.SQL(main_db) samples = x.exe('select * from samples where project_id="' + pid+'" and sample_id="'+sid+'"') xpath = x.project(pid)[0][4] sample = rootvar.samples(samples[0], xpath) with open(rf.json) as data_file: data = json.load(data_file) x.close() ###print data view = rootvar.ViewSampleResults( sample, pipeline, dbname, analysis, rf) return jsonify(tree=[data], range=view.range(selval)[0], pip=pipeline) except Exception as inst: return "ERROR: "+str(inst)
def get_childs_of_taxonomy(): data = request.get_json() pid = data["pid"] uid = data["uid"] sid = data["sid"] tid = data["tid"] lid = data["lid"] cond = data["cond"] pip = data["pip"] rid = data["rid"] dbname = data["rid"] analysis = "taxonomy" ##print cond, pip, rid, pid, uid, sid, tid, lid if cond == "one": # this is if I am analyzing one or multiple samples rf = rootvar.result_files(pid, analysis, pip, sid, dbname) x = sql.SQL(main_db) samples = x.exe('select * from samples where project_id="' + pid+'" and sample_id="'+sid+'"') xpath = x.project(pid)[0][4] sample = rootvar.samples(samples[0], xpath) x.close() view = rootvar.ViewSampleResults(sample, pip, dbname, analysis, rf) matrix = view.childs_of(tid, 'rpkm') ###print matrix return jsonify(matrix=matrix) else: # load big tree all_samples_tree_file = rootvar.__ROOTPRO__+"/" + \ pid+"/"+pip+"/RESULTS/"+rid+".all_samples_tree.pk" G = nx.read_gpickle(all_samples_tree_file) tree = json_graph.tree_data(G, root='R') data = [] for i in G.successors(tid): data.append(i) x = sql.SQL(all_samples_tree_file+".db") matrix = rootvar.get_matrix_level_childs(x, data) samples_sel = list(set([str(i[0]) for i in matrix])) M, N = rootvar.v2m(matrix, samples_sel, 0, 0) ##print M heatmap = 'none' # heatmap=iclust.main(M,None) return jsonify(data=matrix[0], heatmap=heatmap)
def get_statistics(): data = request.get_json() pid = data["pid"] uid = data["uid"] sid = data["sid"] programs = data["value"] dbname = "isuezrouja" x = sql.SQL(main_db) samples = x.exe('select * from samples where project_id="' + pid+'" and sample_id="'+sid+'"') xpath = x.project(pid)[0][4] pname = str(x.project(pid)[0][1]) ##print pname sample = rootvar.samples(samples[0], xpath) x.close() ###print matrix data = [] pipeline = "matches" for program in programs: data.append(statsP(sample, program, sid, dbname, pipeline)) return jsonify(stats=data, sname=sample.name, pname=pname)
def get_functional_counts(): try: data = request.get_json() pid = data["pid"] uid = data["uid"] sid = data["sid"] pip = data["pip"] rid = data["rid"] pip = data['pip'] analysis = "function" # lid="assembly" # file=rootvar.__ROOTPRO__+"/"+pid+"/assembly/idba_ud/"+sid+"/pred.genes."+rid+".matches.function.abundance.results.sqlite3.db" x = sql.SQL(main_db) rf = rootvar.result_files(pid, "function", pip, sid, rid) samples = x.exe('select * from samples where project_id="' + pid+'" and sample_id="'+sid+'"') xpath = x.project(pid)[0][4] sample = rootvar.samples(samples[0], xpath) x.close() view = rootvar.ViewSampleResults(sample, pip, rid, analysis, rf) return jsonify(matrix=view.func_one_sample(), m2=view.func_structure(pid, pip, sid)) except Exception as inst: return "ERROR: "+str(inst)
def run(data): rid = data['rid'] x = sql.SQL(main_db) value = x.exe( 'select taxofile,functfile from reference where reference_id="' + rid + '"')[0] x.close() if rid == "MyTaxa" or rid == "abcdefghij": taxo = taxonomy(data) return taxo if value[0] != "none": #print 'taxonomy' taxo = taxonomy(data) if value[1] != "none": func = functional(data) if value[0] != "none" and value[1] != "none": return [taxo, func] elif value[0] != "none": return taxo elif value[1] != "none": return func
def run(data,main_db,DBS): project_id=str(data['pid']) pipeline=str(data['pip']) reads1=str(data['read1']) reads2=str(data['read2']) sample_id=str(data['sid']) user_id=str(data['uid']) refs=data["rids"] try: a=refs[0] except: return 'Error: No database has been selected' x=sql.SQL(main_db) x.exe('UPDATE project_status SET status="queue" WHERE project_id="'+project_id+'" AND sample_id="'+sample_id+'"') x.insert("sample_run",(sample_id,"created")) # let know that the files have been uploaded and the files created x.commit() #preprocessing FIXME: I need to get permisions to write rdir=rootvar.__ROOTPRO__+"/"+project_id+"/READS/" trim=pre.trimmomatic(rdir+reads1,rdir+reads2,rdir,sample_id) #if not rootvar.isdir(trim.outd+sample_id+'.trim.log'): trim.run() trim.run() #print "trimming" reads1=reads1.replace(".gz","") reads2=reads2.replace(".gz","") #print pipeline ################################################################################################## ##### This is very important, ##### First: Check if the GREENGENES database has been used for finding the 16s rRNAs, if not, ##### The greengenes database is force d to run this database will run first for normalization purposes. ################################################################################################## if pipeline=="assembly": greengenes_file=rootvar.__ROOTPRO__+"/"+project_id+"/assembly/idba_ud/"+sample_id+"/pred.genes.Gbfbquhild.matches" if pipeline=="matches": greengenes_file=rootvar.__ROOTPRO__+"/"+project_id+"/matches/"+sample_id+"/pred.genes.Gbfbquhild.matches" #_______________________________________________________________################################## # # Get the number of reads tfile=trim.outd+sample_id+'trim.log' for i in open(tfile): if "Input Read Pairs:" in i: i=i.split() raw_reads=float(i[3]) good_reads=float(i[6]) # the number of high quality reads after trimming and quality filter print good_reads if pipeline == "matches": log = my_logger(logfile=rootvar.__ROOTPRO__+"/"+project_id+"/matches/"+sample_id+"/arc_run.qsub.log") log.info('running read match pipeline') for ref in refs: log.info('Processing %s with reference id: %s'%(sample_id, ref) ) if not x.exe('select * from matches where sample_id="'+sample_id+'" and datasets="'+ref+'"'): x.c.execute('INSERT OR IGNORE INTO matches VALUES (?,?,?,?)', (sample_id,user_id,project_id,ref)) x.commit() try: val = MP(project_id, sample_id, DBS[ref], "matches", reads1, reads2, good_reads) except Exception as e: log.error(str(e)) log.error('Exception\t%s\t%s'%(sample_id, ref)) # #this is for the aseembly section # if pipeline=="assembly": log = my_logger(logfile=rootvar.__ROOTPRO__ + "/" + project_id + "/assembly/idba_ud/" + sample_id + "/arc_run.qsub.log") log.info('running assembly pipeline') for ref in refs: log.info('Processing %s with reference id: %s'%(sample_id, ref) ) if not x.exe('select * from assembly where sample_id="'+sample_id+'" and datasets="'+ref+'"'): x.c.execute('INSERT OR IGNORE INTO assembly VALUES (?,?,?,?)', (sample_id,user_id,project_id,ref)) x.commit() try: idba(project_id, sample_id, DBS[ref], "assembly", reads1, reads2, good_reads) except Exception as e: log.error(str(e)) log.error('Exception\t%s\t%s' % (sample_id, ref)) x.close() # in the end the fastq files need to be removed in order to save storage space; if not rootvar.isdir(rdir+reads1+".gz"): os.system('gzip '+rdir+reads1+' >> '+rootvar.log+" 2>&1") os.system('rm '+rdir+reads1+' >> '+rootvar.log+" 2>&1") else: os.system('rm '+rdir+reads1+' >> '+rootvar.log+" 2>&1") if not rootvar.isdir(rdir+reads2+".gz"): os.system('gzip '+rdir+reads2+' >> '+rootvar.log+" 2>&1") os.system('rm '+rdir+reads2+' >> '+rootvar.log+" 2>&1") else: os.system('rm '+rdir+reads2+' >> '+rootvar.log+" 2>&1") return 'success'
def RunMetaGen(): try: data = request.get_json() refs = data["rids"] sid = data['sid'] uid = data['uid'] pip = data['pip'] msg = data['msg'] if uid == "TesREPDooc73Ohw": return 'User not allowed' S = query_db('select * from user where user_id="'+uid+'"') T = query_db('select * from samples where sample_id="'+sid+'"') date = time.strftime("%m/%d/%Y") pid = T[0]['project_id'] arg = base64.b64encode(json.dumps( [data, refs, sid, uid, pip, rootvar.__FILEDB__, S, T])) SArc = bench2archu( 'python /groups/metastorm_cscee/MetaStorm/process.py ' + arg) if pip == "assembly": do = "/groups/metastorm_cscee/MetaStorm/Files/PROJECTS/"+data['pid']+"/assembly/idba_ud/"+sid+"/arc_run.qsub.init" else: do = "/groups/metastorm_cscee/MetaStorm/Files/PROJECTS/"+data['pid']+"/matches/"+sid+"/arc_run.qsub.init" try: check_job = bench2archu('cat {}'.format(do)) job_id, _, _ = [i for i in check_job['out'].split('\n')][0].split('.') SArc.update({'job_id': job_id}) except: SArc.update({'job_id': 'NULL'}) pass x = sql.SQL(rootvar.__FILEDB__) update_jobs(x, [uid, T[0]['project_id'], sid, pip, arg, 'queue', 'normal', date, SArc['out'].split(".")[0]]) val = x.exe('update samples set reads1="' + data['read1']+'" where project_id="'+data['pid']+'" and sample_id="'+sid+'"') val = x.exe('update samples set reads2="' + data['read2']+'" where project_id="'+data['pid']+'" and sample_id="'+sid+'"') for ref in refs: update_status(x, sid, ref, pip, "queue") if not x.exe('select * from '+pip+' where sample_id="'+sid+'" and datasets="'+ref+'"'): x.c.execute('INSERT OR IGNORE INTO '+pip + ' VALUES (?,?,?,?)', (sid, uid, pid, ref)) x.commit() x.close() try: os.system("mkdir -p "+rootvar.__ROOTPRO__+"/" + data['pid']+"/assembly/idba_ud/"+sid) os.system("mkdir -p "+rootvar.__ROOTPRO__ + "/"+data['pid']+"/assembly/RESULTS/") os.system("mkdir -p "+rootvar.__ROOTPRO__ + "/"+data['pid']+"/matches/"+sid) os.system("mkdir -p "+rootvar.__ROOTPRO__ + "/"+data['pid']+"/matches/RESULTS/") os.system("mkdir -p "+rootvar.__ROOTPRO__+"/"+data['pid']+"/READS") except: pass x = email.send_email(S[0]['user_name'], S[0]['user_affiliation'], 'Processing sample: '+T[0]['sample_name'], "Dear MetaStorm User, <br><br><br> the sample <b>" + T[0]['sample_name'] + '</b> has been submitted into the MetaStorm server. It will run the <b>' + pip+'</b> pipeline <br><br>' 'The time for making the analysis depends on the current web traffic and availability of the web server. Once the ' + 'analysis is done you will receive a notification via email. <br><br><br><br>' + 'Thank you<br><b>MetaStorm Team</b>') return jsonify(SArc) except Exception as inst: return "ERROR: "+str(inst)
def idbaud(projectid,sampleid,db,protocol,reads1, reads2, good_reads): #db=root.dataset(db) #1 get project path x=sql.SQL(root.filedb()) xpath=x.project(projectid)[0][4] #print 'here------------' ########################################################################### #2 update the reads on the sql dataset, doit anyway, so if the sample is re run just take the new input, it could be modified. ########################################################################### val=x.exe('update samples set reads1="'+reads1+'" where project_id="'+projectid+'" and sample_id="'+sampleid+'"') val=x.exe('update samples set reads2="'+reads2+'" where project_id="'+projectid+'" and sample_id="'+sampleid+'"') ########################################################################### #3 get the sample full information - load the class samples ########################################################################### samples=x.exe('select * from samples where project_id="'+projectid+'" and sample_id="'+sampleid+'"') sample=samples[0] sample=root.samples(sample,xpath) root.mkdir(sample.assemblyDir) ########################################################################### # 4.1 Run fq2fa - this is used by udba_ud program ########################################################################### idba_ud=root.program('idba_ud',sample,db) update_status(x,sampleid,db.id,protocol,"Preprocessing") fq2fa=root.program('fq2fa', sample,db) if not root.isdir(idba_ud.out): fq2fa.run() #make sure that there is a scaffold.fa file. If not, it computes again the fastq to fasta and the assembly ########################################################################### # 4.2 Run idba_ud - assembly the samples ########################################################################### update_status(x,sampleid,db.id,protocol,"Assembling") idba_ud=root.program('idba_ud',sample,db) if not root.isdir(idba_ud.out): idba_ud.run(); os.system(' cd ' + idba_ud.path + ' && rm kmer contig-* align-* graph-* local-contig-* reads.fa') ########################################################################### # 4.2 Run gene finder - look at the genes over the scaffolds ########################################################################### prodigal=root.program("prodigal", sample,db) update_status(x,sampleid,db.id,protocol,"Finding Genes") if not root.isdir(prodigal.output+".gff"): prodigal.run() if db.name=="abcdefghij": print "MetaPlAn2" update_status(x,sampleid,db.id,protocol,"Processing") metaphlan=root.program('MetaPhlAn',sample,db) if not root.isdir(metaphlan.out): metaphlan.run() #print "Here 2" G=txp.metaphlan_taxonomy_tree(metaphlan.out) abn=root.SampleResults(sample,G,protocol, db.name, "taxonomy", metaphlan.out) abn.start() update_status(x,sampleid,db.id,protocol,"Done") if db.name=='MyTaxa': #print "MyTaxa" taxa=root.mytaxa(sample,db) update_status(x,sampleid,db.id,protocol,"Screening") if not root.isdir(taxa.output+".prot.mytaxa.fa"): taxa.pre() if not root.isdir(taxa.output+".MyTaxa.matches.daa"): taxa.align() if not root.isdir(taxa.output+".MyTaxa.align"): taxa.postd() if not root.isdir(taxa.output+".MyTaxa.input"): taxa.mpre() if not root.isdir(taxa.output+".MyTaxa.out"): taxa.run() update_status(x,sampleid,db.id,protocol,"Quantification") data=taxa.postM() G=txp.mytaxa_taxonomy_tree(data,taxa.output+".MyTaxa.matches.taxonomy.abundance") abn=root.SampleResults(sample,G,protocol, "MyTaxa", "taxonomy", taxa.output+".MyTaxa.matches") abn.start() update_status(x,sampleid,db.id,protocol,"Done") if not db.taxo=="none": print "taxonomy" ########################################################################### # 4.3 Run bowtie to find matches ########################################################################### update_status(x,sampleid,db.id,protocol,"Screening") if db.name=="ryaetguxun": blastn=root.program('diamond_blastp',sample,db) else: blastn=root.program('blastn',sample,db) blastn.run() #blastn.run() ########################################################################### # 4.4 taxonomy abundance ########################################################################### update_status(x,sampleid,db.id,protocol,"Quantification") abundance=pb(blastn.out, db.taxo, db.len, db.taxodb, "taxonomy", db.name, "none",good_reads) ########################################################################### # 4.5 processing Visualization ########################################################################### G=txp.taxonomy_tree(abundance,blastn.out, protocol, "taxonomy", db.name ) abn=root.SampleResults(sample,G,protocol, db.name, "taxonomy", blastn.out) abn.start() root.updateStatus(x,projectid,sampleid,"done") update_status(x,sampleid,db.id,protocol,"Done") if not db.func=="none": print "functional annotation" update_status(x,sampleid,db.id,protocol,"Screening") fileso=root.result_files(projectid, "function", protocol, sampleid, db.name) ########################################################################### # 4.3 Run bowtie to find matches ########################################################################### root.updateStatus(x,projectid,sampleid,"functional annotation") blastn=root.program('diamond_blastp',sample,db) blastn.run() ########################################################################### # 4.4 taxonomy abundance ########################################################################### update_status(x,sampleid,db.id,protocol,"Quantification") abundance=pb(blastn.out, db.func, db.len, db.funcdb, "function", db.name, fileso.GGenes+".rpkm", good_reads) ########################################################################### # 4.5 processing Visualization ########################################################################### abn=root.SampleResults(sample,'none',protocol, db.name, "function", blastn.out) abn.createFuncDb(abundance) update_status(x,sampleid,db.id,protocol,"Done")
def process(projectid, sampleid, db, protocol, reads1, reads2, good_reads): #db=root.dataset(db) x = sql.SQL(root.filedb()) xpath = x.project(projectid)[0][4] val = x.exe('update samples set reads1="' + reads1 + '" where project_id="' + projectid + '" and sample_id="' + sampleid + '"') val = x.exe('update samples set reads2="' + reads2 + '" where project_id="' + projectid + '" and sample_id="' + sampleid + '"') samples = x.exe('select * from samples where project_id="' + projectid + '" and sample_id="' + sampleid + '"') sample = samples[0] sample = root.samples(sample, xpath) root.mkdir(sample.matchesDir) rdir = root.__ROOTPRO__ + "/" + projectid + "/READS/" if db.name == "abcdefghij": #print "MetaPhlAnn" update_status(x, sampleid, db.id, protocol, "Processing") metaphlan = root.program('MetaPhlAnR', sample, db) if not root.isdir(metaphlan.out): metaphlan.run() G = txp.metaphlan_taxonomy_tree(metaphlan.out) abn = root.SampleResults(sample, G, protocol, db.name, "taxonomy", metaphlan.out) abn.start() update_status(x, sampleid, db.id, protocol, "Done") if not db.taxo == "none": # run bowtie using the paired end reads update_status(x, sampleid, db.id, 'matches', "Screening") cmd = " ".join([ root.__ROOTEXEDIR__ + 'bowtie2', '--very-fast-local -p ' + p + ' --no-unal --no-hd --no-sq -x', db.bowtie, '-1', sample.reads1, '-2', sample.reads2, '-S', sample.matchesDir + '/alignment.' + db.id + '.matches >>', root.log, '2>&1' ]) if not root.isdir(sample.matchesDir + '/alignment.' + db.id + '.matches'): os.system(cmd) #process output in sam format to get genes and number of reads per gene. update_status(x, sampleid, db.id, protocol, "Quantification") if not root.isdir(sample.matchesDir + '/alignment.' + db.id + '.matches.taxonomy.abundance.results.sqlite3.db'): abundance = parse_sam( sample.matchesDir + '/alignment.' + db.id + '.matches', db, good_reads) G = txp.taxonomy_tree( abundance, sample.matchesDir + '/alignment.' + db.id + '.matches', protocol, "taxonomy", db.id) abn = root.SampleResults(sample, G, protocol, db.id, "taxonomy", sample.matchesDir + '/alignment.' + db.id + '.matches') # Store data in the sql TABLE abn.start() update_status(x, sampleid, db.id, protocol, "Done") return 'success' if not db.func == "none": fileso = root.result_files(projectid, "function", protocol, sampleid, db.name) #Merge paired ends update_status(x, sampleid, db.id, protocol, "Merge") cmd = " ".join([ 'python', root.__ROOTEXEDIR__ + "pairend_join.py -s -p " + p + " -m 8 -o ", sample.matchesDir + '/merged.reads.fastq', sample.reads1, sample.reads2 ]) #print cmd root.flog(cmd) #print cmd if not root.isdir(sample.matchesDir + '/merged.reads.fastq'): os.system(cmd) #Get fasta files cmd = ' '.join([ root.__ROOTEXEDIR__ + '/seqtk seq -a', sample.matchesDir + '/merged.reads.fastq >', sample.matchesDir + '/merged.reads.fasta' ]) if not root.isdir(sample.matchesDir + '/merged.reads.fasta'): os.system(cmd) #BlastX from diamond update_status(x, sampleid, db.id, protocol, "Screening") dout = sample.matchesDir + 'alignment.' + db.id din = sample.matchesDir + '/merged.reads.fasta' cmd = ' '.join([ root.__ROOTEXEDIR__ + '/diamond blastx --id 60 -p ' + p + ' -k 1 -e 1e-5 -d', db.diamond, '-a', dout + '.pre', '-q', din, '>>', root.log, "2>&1" ]) if not root.isdir(dout + '.daa'): os.system(cmd) cmd = ' '.join([ root.__ROOTEXEDIR__ + '/diamond view -a', dout + '.pre.daa', '-o', dout + '.matches -f tab', ">>", root.log, "2>&1" ]) if not root.isdir(dout + '.matches'): os.system(cmd) # parse diamond output update_status(x, sampleid, db.id, protocol, "Quantification") if not root.isdir(sample.matchesDir + '/alignment.' + db.id + '.matches.function.abundance.results.sqlite3.db'): abundance = pb(dout + '.matches', db.func, db.len, db.funcdb, "function", db.name, fileso.GGenes + ".rpkm", good_reads) #abundance=pdx(dout+'.matches', db, good_reads) abn = root.SampleResults(sample, 'none', protocol, db.name, "function", dout + '.matches') abn.createFuncDb(abundance) update_status(x, sampleid, db.id, protocol, "Done") os.system('rm ' + sample.matchesDir + '/merged.reads.fastq >> ' + root.log + " 2>&1") os.system('rm ' + sample.matchesDir + '/merged.reads.fasta >> ' + root.log + " 2>&1") return 'success'
def taxonomy(data): pid = data["pid"] uid = data["uid"] sids = data["sid"] pipeline = data["pip"] rid = data["rid"] edges = [] FULL_MATRIX = [] #first see if the data set contains taxonomy, function or both annotations analysis = "taxonomy" #print '\n\n\n here we goo!!!\n\n\n', pipeline rootvar.mkdir(rootvar.__ROOTPRO__ + "/" + pid + "/" + pipeline + "/RESULTS/") all_samples_tree_file = rootvar.__ROOTPRO__ + "/" + pid + "/" + pipeline + "/RESULTS/" + rid + ".all_samples_tree.pk" # stored_samples = 0 if os.path.isfile(all_samples_tree_file): #print '\n\n\n if the X file has been created' x = sql.SQL(all_samples_tree_file + ".db") val = x.exe("select distinct sample_name from full_matrix") stored_samples = len(val) # x = sql.SQL(main_db) sids = x.exe( 'select c.sample_id from (select * from samples a inner join (select * from sample_status where pip="' + str(pipeline) + '") b on a.sample_id==b.sid) c where c.project_id=="' + pid + '" and c.status="Done" and c.rid="' + rid + '"') # # #print '\n\n\n\n super important \n', sids #print '\n\n\n get all the samples from the X file' # if os.path.isfile(all_samples_tree_file) and stored_samples == len(sids): #print 'both have the same length' G = nx.read_gpickle(all_samples_tree_file) tree = json_graph.tree_data(G, root='R') return ["taxonomy", tree] #section #print '\n\n\n SO the file has not been created, because the matrix of abundances has not been fetched for those samples' nodes = {} #print '\n\n\n So for each sample I get all the information from the sql tables. And create the tree \n\n\n\n\n\n' # for sid in sids: samples = x.exe('select * from samples where project_id="' + pid + '" and sample_id="' + sid[0] + '"') xpath = x.project(pid)[0][4] sample = rootvar.samples(samples[0], xpath) rf = rootvar.result_files(pid, analysis, pipeline, sample.id, rid) view = rootvar.ViewSampleResults(sample, pipeline, rid, analysis, rf) FULL_MATRIX += view.all() Gp = nx.read_gpickle(rf.pk) edges += Gp.edges() for node in Gp.nodes(): if not node in nodes: nodes[node] = Gp.node[node]['level'] x.close() rootvar.full_matrix_sql(all_samples_tree_file + ".db", FULL_MATRIX) G = nx.DiGraph() for i in edges: if not i in G.edges(): if not i[1] in G.nodes(): G.add_node(i[1], samples=1, level=nodes[i[1]]) else: G.node[i[1]]['samples'] += 1 if not i[0] in G.nodes(): G.add_node(i[0], samples=1, level=nodes[i[0]]) else: G.node[i[0]]['samples'] += 1 if not G.predecessors(i[1]): G.add_edge(i[0], i[1]) else: G.node[i[0]]['samples'] += 1 G.node[i[1]]['samples'] += 1 tree = json_graph.tree_data(G, root='R') nx.write_gpickle(G, all_samples_tree_file) return ["taxonomy", tree, all_samples_tree_file + ".json"]
from app.lib.common import rootvar from app.lib.common.sqlite3_commands import update_jobs, update_status from app.lib.create_project import insert_new_project as sql import time import os import base64 import json from app.lib.common.arc_connect import bench2archu, arcon from app.lib.email import Email as email import datetime database = sql.SQL(rootvar.__FILEDB__) import re import logging import datetime def get_results(job='', status='', message=[]): log = logging.getLogger() try: inp = json.loads(base64.b64decode(job)) data = inp[0] refs = inp[1] sid = inp[2] uid = inp[3] pip = inp[4] USER = inp[6] SAMPLE = inp[7] if pip == "assembly":