def plotNormalProbability(vals=None, RISet='', title=None, showstrains=0, specialStrains=[None], size=(750,500)): dataXZ = vals[:] dataXZ.sort(webqtlUtil.cmpOrder) dataLabel = [] dataX = map(lambda X: X[1], dataXZ) showLabel = showstrains if len(dataXZ) > 50: showLabel = 0 for item in dataXZ: strainName = webqtlUtil.genShortStrainName(RISet=RISet, input_strainName=item[0]) dataLabel.append(strainName) dataY=Plot.U(len(dataX)) dataZ=map(Plot.inverseCumul,dataY) c = pid.PILCanvas(size=(750,500)) Plot.plotXY(c, dataZ, dataX, dataLabel = dataLabel, XLabel='Expected Z score', connectdot=0, YLabel='Trait value', title=title, specialCases=specialStrains, showLabel = showLabel) filename= webqtlUtil.genRandStr("nP_") c.save(webqtlConfig.IMGDIR+filename, format='gif') img=HT.Image('/image/'+filename+'.gif',border=0) return img
def run_plink(self): plink_output_filename = webqtlUtil.genRandStr( "%s_%s_" % (self.dataset.group.name, self.this_trait.name)) self.gen_pheno_txt_file_plink(pheno_filename=plink_output_filename) plink_command = PLINK_COMMAND + ' --noweb --bed %s/%s.bed --bim %s/%s.bim --fam %s/%s.fam --no-fid --no-parents --no-sex --no-pheno --pheno %s%s.txt --pheno-name %s --maf %s --missing-phenotype -9999 --out %s%s --assoc ' % ( PLINK_PATH, self.dataset.group.name, PLINK_PATH, self.dataset.group.name, PLINK_PATH, self.dataset.group.name, webqtlConfig.TMPDIR, plink_output_filename, self.this_trait.name, self.maf, webqtlConfig.TMPDIR, plink_output_filename) #print("plink_command:", plink_command) os.system(plink_command) count, p_values = self.parse_plink_output(plink_output_filename) #for marker in self.dataset.group.markers.markers: # if marker['name'] not in included_markers: # print("marker:", marker) # self.dataset.group.markers.markers.remove(marker) # #del self.dataset.group.markers.markers[marker] print("p_values:", pf(p_values)) self.dataset.group.markers.add_pvalues(p_values) return self.dataset.group.markers.markers
def addToTable(self, traitNames, strainNames,strainIds, traitValues, SE, NStrain, fd): self.cursor.execute('delete Temp, TempData from Temp, TempData where Temp.DataId = TempData.Id and UNIX_TIMESTAMP()-UNIX_TIMESTAMP(CreateTime)>%d;' % webqtlConfig.MAXLIFE) i = 0 for trait in traitNames: ct0 = time.localtime(time.time()) ct = time.strftime("%B/%d %H:%M:%S",ct0) if trait == '': trait = "Unnamed Trait" user_ip = fd.remote_ip newDescription = '%s entered at %s from IP %s' % (trait,ct,user_ip) newProbeSetID = webqtlUtil.genRandStr('Usr_TMP_') self.cursor.execute('SelecT max(id) from TempData') try: DataId = self.cursor.fetchall()[0][0] + 1 except: DataId = 1 self.cursor.execute('Select Id from InbredSet where Name = "%s"' % fd.RISet) InbredSetId = self.cursor.fetchall()[0][0] self.cursor.execute('insert into Temp(Name,description, createtime,DataId,InbredSetId,IP) values(%s,%s,Now(),%s,%s,%s)' ,(newProbeSetID, newDescription, DataId,InbredSetId,user_ip)) for k in range(len(traitValues[i])): if traitValues[i][k] != None: self.cursor.execute('insert into TempData(Id, StrainId, value, SE, NStrain) values(%s, %s, %s, %s, %s)' , (DataId, strainIds[k], traitValues[i][k],SE[i][k],NStrain[i][k])) self.searchResult.append('Temp::%s' % newProbeSetID) i += 1
def plotBoxPlot(vals): valsOnly = [] dataXZ = vals[:] for i in range(len(dataXZ)): valsOnly.append(dataXZ[i][1]) plotHeight = 320 plotWidth = 220 xLeftOffset = 60 xRightOffset = 40 yTopOffset = 40 yBottomOffset = 60 canvasHeight = plotHeight + yTopOffset + yBottomOffset canvasWidth = plotWidth + xLeftOffset + xRightOffset canvas = pid.PILCanvas(size=(canvasWidth,canvasHeight)) XXX = [('', valsOnly[:])] Plot.plotBoxPlot(canvas, XXX, offset=(xLeftOffset, xRightOffset, yTopOffset, yBottomOffset), XLabel= "Trait") filename= webqtlUtil.genRandStr("Box_") canvas.save(webqtlConfig.IMGDIR+filename, format='gif') img=HT.Image('/image/'+filename+'.gif',border=0) plotLink = HT.Span("More about ", HT.Href(text="Box Plots", url="http://davidmlane.com/hyperstat/A37797.html", target="_blank", Class="fs13")) return img, plotLink
def run_plink(this_trait, dataset, species, vals, maf): plink_output_filename = webqtlUtil.genRandStr("%s_%s_"%(dataset.group.name, this_trait.name)) gen_pheno_txt_file_plink(this_trait, dataset, vals, pheno_filename = plink_output_filename) plink_command = PLINK_COMMAND + ' --noweb --ped %s/%s.ped --no-fid --no-parents --no-sex --no-pheno --map %s/%s.map --pheno %s%s.txt --pheno-name %s --maf %s --missing-phenotype -9999 --out %s%s --assoc ' % ( PLINK_PATH, dataset.group.name, PLINK_PATH, dataset.group.name, TMPDIR, plink_output_filename, this_trait.name, maf, TMPDIR, plink_output_filename) logger.debug("plink_command:", plink_command) os.system(plink_command) count, p_values = parse_plink_output(plink_output_filename, species) #for marker in self.dataset.group.markers.markers: # if marker['name'] not in included_markers: # logger.debug("marker:", marker) # self.dataset.group.markers.markers.remove(marker) # #del self.dataset.group.markers.markers[marker] logger.debug("p_values:", pf(p_values)) dataset.group.markers.add_pvalues(p_values) return dataset.group.markers.markers
def screePlot(self, NNN=0, pearsonEigenValue=None): c1 = pid.PILCanvas(size=(700,500)) Plot.plotXY(canvas=c1, dataX=range(1,NNN+1), dataY=pearsonEigenValue, rank=0, labelColor=pid.blue,plotColor=pid.red, symbolColor=pid.blue, XLabel='Factor Number', connectdot=1,YLabel='Percent of Total Variance %', title='Pearson\'s R Scree Plot') filename= webqtlUtil.genRandStr("Scree_") c1.save(webqtlConfig.IMGDIR+filename, format='gif') img=HT.Image('/image/'+filename+'.gif',border=0) return img
def factorLoadingsPlot(self, pearsonEigenVectors=None, traitList=None): traitname = map(lambda X:str(X.name), traitList) c2 = pid.PILCanvas(size=(700,500)) Plot.plotXY(c2, pearsonEigenVectors[0],pearsonEigenVectors[1], 0, dataLabel = traitname, labelColor=pid.blue, plotColor=pid.red, symbolColor=pid.blue,XLabel='Factor (1)', connectdot=1, YLabel='Factor (2)', title='Factor Loadings Plot (Pearson)', loadingPlot=1) filename= webqtlUtil.genRandStr("FacL_") c2.save(webqtlConfig.IMGDIR+filename, format='gif') img = HT.Image('/image/'+filename+'.gif',border=0) return img
def __init__(self, start_vars, temp_uuid): #Currently only getting trait data for one trait, but will need #to change this to accept multiple traits once the collection page is implemented helper_functions.get_species_dataset_trait(self, start_vars) tempdata = temp_data.TempData(temp_uuid) self.samples = [] # Want only ones with values self.vals = [] for sample in self.dataset.group.samplelist: value = start_vars['value:' + sample] self.samples.append(str(sample)) self.vals.append(value) print("start_vars:", start_vars) self.set_options(start_vars) self.json_data = {} #if self.method == "qtl_reaper": self.json_data['lodnames'] = ['lod.hk'] self.gen_reaper_results(tempdata) #else: # self.gen_pylmm_results(tempdata) #self.gen_qtl_results(tempdata) #Get chromosome lengths for drawing the interval map plot chromosome_mb_lengths = {} self.json_data['chrnames'] = [] for key in self.species.chromosomes.chromosomes.keys(): self.json_data['chrnames'].append([self.species.chromosomes.chromosomes[key].name, self.species.chromosomes.chromosomes[key].mb_length]) chromosome_mb_lengths[key] = self.species.chromosomes.chromosomes[key].mb_length #print("self.qtl_results:", self.qtl_results) print("JSON DATA:", self.json_data) #os.chdir(webqtlConfig.TMPDIR) json_filename = webqtlUtil.genRandStr(prefix="intmap_") json.dumps(self.json_data, webqtlConfig.TMPDIR + json_filename) self.js_data = dict( manhattan_plot = self.manhattan_plot, additive = self.additive, chromosomes = chromosome_mb_lengths, qtl_results = self.qtl_results, json_data = self.json_data #lrs_lod = self.lrs_lod, )
def plotBarGraph(identification='', RISet='', vals=None, type="name"): this_identification = "unnamed trait" if identification: this_identification = identification if type=="rank": dataXZ = vals[:] dataXZ.sort(webqtlUtil.cmpOrder) title='%s' % this_identification else: dataXZ = vals[:] title='%s' % this_identification tvals = [] tnames = [] tvars = [] for i in range(len(dataXZ)): tvals.append(dataXZ[i][1]) tnames.append(webqtlUtil.genShortStrainName(RISet=RISet, input_strainName=dataXZ[i][0])) tvars.append(dataXZ[i][2]) nnStrain = len(tnames) sLabel = 1 ###determine bar width and space width if nnStrain < 20: sw = 4 elif nnStrain < 40: sw = 3 else: sw = 2 ### 700 is the default plot width minus Xoffsets for 40 strains defaultWidth = 650 if nnStrain > 40: defaultWidth += (nnStrain-40)*10 defaultOffset = 100 bw = int(0.5+(defaultWidth - (nnStrain-1.0)*sw)/nnStrain) if bw < 10: bw = 10 plotWidth = (nnStrain-1)*sw + nnStrain*bw + defaultOffset plotHeight = 500 #print [plotWidth, plotHeight, bw, sw, nnStrain] c = pid.PILCanvas(size=(plotWidth,plotHeight)) Plot.plotBarText(c, tvals, tnames, variance=tvars, YLabel='Value', title=title, sLabel = sLabel, barSpace = sw) filename= webqtlUtil.genRandStr("Bar_") c.save(webqtlConfig.IMGDIR+filename, format='gif') img=HT.Image('/image/'+filename+'.gif',border=0) return img
def run_rqtl_plink(self): os.chdir("/home/zas1024/plink") output_filename = webqtlUtil.genRandStr("%s_%s_"%(self.dataset.group.name, self.this_trait.name)) self.gen_pheno_txt_file_plink(pheno_filename = output_filename) rqtl_command = './plink --noweb --ped %s.ped --no-fid --no-parents --no-sex --no-pheno --map %s.map --pheno %s/%s.txt --pheno-name %s --maf %s --missing-phenotype -9999 --out %s%s --assoc ' % (self.dataset.group.name, self.dataset.group.name, webqtlConfig.TMPDIR, plink_output_filename, self.this_trait.name, self.maf, webqtlConfig.TMPDIR, plink_output_filename) os.system(rqtl_command) count, p_values = self.parse_rqtl_output(plink_output_filename)
def run_rqtl_plink(self): # os.chdir("") never do this inside a webserver!! output_filename = webqtlUtil.genRandStr("%s_%s_"%(self.dataset.group.name, self.this_trait.name)) self.gen_pheno_txt_file_plink(pheno_filename = output_filename) rqtl_command = './plink --noweb --ped %s.ped --no-fid --no-parents --no-sex --no-pheno --map %s.map --pheno %s/%s.txt --pheno-name %s --maf %s --missing-phenotype -9999 --out %s%s --assoc ' % (self.dataset.group.name, self.dataset.group.name, TMPDIR, plink_output_filename, self.this_trait.name, self.maf, TMPDIR, plink_output_filename) os.system(rqtl_command) count, p_values = self.parse_rqtl_output(plink_output_filename)
def run_rqtl_plink(self): # os.chdir("") never do this inside a webserver!! output_filename = webqtlUtil.genRandStr("%s_%s_"%(self.dataset.group.name, self.this_trait.name)) plink_mapping.gen_pheno_txt_file_plink(self.this_trait, self.dataset, self.vals, pheno_filename = output_filename) rqtl_command = './plink --noweb --ped %s.ped --no-fid --no-parents --no-sex --no-pheno --map %s.map --pheno %s/%s.txt --pheno-name %s --maf %s --missing-phenotype -9999 --out %s%s --assoc ' % (self.dataset.group.name, self.dataset.group.name, TMPDIR, plink_output_filename, self.this_trait.name, self.maf, TMPDIR, plink_output_filename) os.system(rqtl_command) count, p_values = self.parse_rqtl_output(plink_output_filename)
def __init__(self, start_vars, temp_uuid): # Currently only getting trait data for one trait, but will need # to change this to accept multiple traits once the collection page is implemented helper_functions.get_species_dataset_trait(self, start_vars) tempdata = temp_data.TempData(temp_uuid) self.samples = [] # Want only ones with values self.vals = [] for sample in self.dataset.group.samplelist: value = start_vars["value:" + sample] self.samples.append(str(sample)) self.vals.append(value) print("start_vars:", start_vars) self.set_options(start_vars) self.score_type = "LRS" self.cutoff = 3 self.json_data = {} self.json_data["lodnames"] = ["lod.hk"] self.gen_reaper_results(tempdata) # Get chromosome lengths for drawing the interval map plot chromosome_mb_lengths = {} self.json_data["chrnames"] = [] for key in self.species.chromosomes.chromosomes.keys(): self.json_data["chrnames"].append( [self.species.chromosomes.chromosomes[key].name, self.species.chromosomes.chromosomes[key].mb_length] ) chromosome_mb_lengths[key] = self.species.chromosomes.chromosomes[key].mb_length print("JSON DATA:", self.json_data) json_filename = webqtlUtil.genRandStr(prefix="intmap_") json.dumps(self.json_data, webqtlConfig.TMPDIR + json_filename) self.js_data = dict( result_score_type=self.score_type, manhattan_plot=self.manhattan_plot, chromosomes=chromosome_mb_lengths, qtl_results=self.qtl_results, json_data=self.json_data, )
def run_plink(this_trait, dataset, species, vals, maf): plink_output_filename = webqtlUtil.genRandStr("%s_%s_"%(dataset.group.name, this_trait.name)) gen_pheno_txt_file(dataset, vals) plink_command = PLINK_COMMAND + ' --noweb --bfile %s/%s --no-pheno --no-fid --no-parents --no-sex --maf %s --out %s%s --assoc ' % ( flat_files('mapping'), dataset.group.name, maf, TMPDIR, plink_output_filename) logger.debug("plink_command:", plink_command) os.system(plink_command) count, p_values = parse_plink_output(plink_output_filename, species) logger.debug("p_values:", p_values) dataset.group.markers.add_pvalues(p_values) return dataset.group.markers.markers
def run_plink(this_trait, dataset, species, vals, maf): plink_output_filename = webqtlUtil.genRandStr( f"{dataset.group.name}_{this_trait.name}_") gen_pheno_txt_file(dataset, vals) plink_command = f"{PLINK_COMMAND} --noweb --bfile {flat_files('mapping')}/{dataset.group.name} --no-pheno --no-fid --no-parents --no-sex --maf {maf} --out { TMPDIR}{plink_output_filename} --assoc " logger.debug("plink_command:", plink_command) os.system(plink_command) count, p_values = parse_plink_output(plink_output_filename, species) logger.debug("p_values:", p_values) dataset.group.markers.add_pvalues(p_values) return dataset.group.markers.markers
def factorLoadingsPlot(self, pearsonEigenVectors=None, traitList=None): traitname = map(lambda X:str(X.name), traitList) c2 = pid.PILCanvas(size=(700,500)) if type(pearsonEigenVectors[0][0]).__name__ == 'complex': pearsonEigenVectors_0 = self.removeimag_array(values=pearsonEigenVectors[0]) else: pearsonEigenVectors_0 = pearsonEigenVectors[0] if type(pearsonEigenVectors[1][0]).__name__ == 'complex': pearsonEigenVectors_1 = self.removeimag_array(values=pearsonEigenVectors[1]) else: pearsonEigenVectors_1 = pearsonEigenVectors[1] Plot.plotXY(c2, pearsonEigenVectors_0,pearsonEigenVectors_1, 0, dataLabel = traitname, labelColor=pid.blue, plotColor=pid.red, symbolColor=pid.blue,XLabel='Factor (1)', connectdot=1, YLabel='Factor (2)', title='Factor Loadings Plot (Pearson)', loadingPlot=1) filename= webqtlUtil.genRandStr("FacL_") c2.save(webqtlConfig.IMGDIR+filename, format='gif') img = HT.Image('/image/'+filename+'.gif',border=0) return img
def run_plink(this_trait, dataset, species, vals, maf): plink_output_filename = webqtlUtil.genRandStr( "%s_%s_" % (dataset.group.name, this_trait.name)) gen_pheno_txt_file(dataset, vals) plink_command = PLINK_COMMAND + ' --noweb --bfile %s/%s --no-pheno --no-fid --no-parents --no-sex --maf %s --out %s%s --assoc ' % ( flat_files('mapping'), dataset.group.name, maf, TMPDIR, plink_output_filename) logger.debug("plink_command:", plink_command) os.system(plink_command) count, p_values = parse_plink_output(plink_output_filename, species) logger.debug("p_values:", p_values) dataset.group.markers.add_pvalues(p_values) return dataset.group.markers.markers
def run_plink(self): plink_output_filename = webqtlUtil.genRandStr("%s_%s_" % (self.dataset.group.name, self.this_trait.name)) self.gen_pheno_txt_file_plink(pheno_filename=plink_output_filename) plink_command = ( PLINK_COMMAND + " --noweb --ped %s/%s.ped --no-fid --no-parents --no-sex --no-pheno --map %s/%s.map --pheno %s%s.txt --pheno-name %s --maf %s --missing-phenotype -9999 --out %s%s --assoc " % ( PLINK_PATH, self.dataset.group.name, PLINK_PATH, self.dataset.group.name, webqtlConfig.TMPDIR, plink_output_filename, self.this_trait.name, self.maf, webqtlConfig.TMPDIR, plink_output_filename, ) ) print("plink_command:", plink_command) os.system(plink_command) count, p_values = self.parse_plink_output(plink_output_filename) # for marker in self.dataset.group.markers.markers: # if marker['name'] not in included_markers: # print("marker:", marker) # self.dataset.group.markers.markers.remove(marker) # #del self.dataset.group.markers.markers[marker] print("p_values:", pf(p_values)) self.dataset.group.markers.add_pvalues(p_values) return self.dataset.group.markers.markers
def run_plink(this_trait, dataset, species, vals, maf): plink_output_filename = webqtlUtil.genRandStr("%s_%s_"%(dataset.group.name, this_trait.name)) gen_pheno_txt_file(dataset, vals) #gen_pheno_txt_file_plink(this_trait, dataset, vals, pheno_filename = plink_output_filename) plink_command = PLINK_COMMAND + ' --noweb --bfile %s/%s --no-fid --no-parents --no-sex --maf %s --missing-phenotype -9 --out %s/%s --assoc ' % ( flat_files('mapping'), dataset.group.name, maf, TMPDIR, plink_output_filename) logger.debug("plink_command:", plink_command) os.system(plink_command) count, p_values = parse_plink_output(plink_output_filename, species) #for marker in self.dataset.group.markers.markers: # if marker['name'] not in included_markers: # logger.debug("marker:", marker) # self.dataset.group.markers.markers.remove(marker) # #del self.dataset.group.markers.markers[marker] logger.debug("p_values:", p_values) dataset.group.markers.add_pvalues(p_values) return dataset.group.markers.markers
def run_plink(self): os.chdir("/home/zas1024/plink") plink_output_filename = webqtlUtil.genRandStr("%s_%s_"%(self.dataset.group.name, self.this_trait.name)) self.gen_pheno_txt_file_plink(pheno_filename = plink_output_filename) plink_command = './plink --noweb --ped %s.ped --no-fid --no-parents --no-sex --no-pheno --map %s.map --pheno %s/%s.txt --pheno-name %s --maf %s --missing-phenotype -9999 --out %s%s --assoc ' % (self.dataset.group.name, self.dataset.group.name, webqtlConfig.TMPDIR, plink_output_filename, self.this_trait.name, self.maf, webqtlConfig.TMPDIR, plink_output_filename) os.system(plink_command) count, p_values = self.parse_plink_output(plink_output_filename) #gemma_command = './gemma -bfile %s -k output_%s.cXX.txt -lmm 1 -o %s_output' % ( # self.dataset.group.name, # self.dataset.group.name, # self.dataset.group.name) #print("gemma_command:" + gemma_command) # #os.system(gemma_command) # #included_markers, p_values = self.parse_gemma_output() # #self.dataset.group.get_specified_markers(markers = included_markers) #for marker in self.dataset.group.markers.markers: # if marker['name'] not in included_markers: # print("marker:", marker) # self.dataset.group.markers.markers.remove(marker) # #del self.dataset.group.markers.markers[marker] print("p_values:", pf(p_values)) self.dataset.group.markers.add_pvalues(p_values) return self.dataset.group.markers.markers
def __init__(self, fd): templatePage.__init__(self, fd) if not self.openMysql(): return fd.readGenotype() TD_LR = HT.TD(height=200,width="100%",bgColor='#eeeeee') self.database = fd.formdata.getfirst('database') self.ProbeSetID = fd.formdata.getfirst('ProbeSetID') self.CellID = fd.formdata.getfirst('CellID') self.db = webqtlDataset(self.database, self.cursor) thisTrait = webqtlTrait(db= self.db, cursor=self.cursor, name=self.ProbeSetID) #, cellid=CellID) thisTrait.retrieveInfo() try: self.cursor.execute('SELECT ProbeFreeze.Name FROM ProbeFreeze,ProbeSetFreeze WHERE ProbeFreeze.Id = ProbeSetFreeze.ProbeFreezeId and ProbeSetFreeze.Name = "%s"' % self.db.name) self.probeDatabase = self.cursor.fetchall()[0][0] self.probeInfoDatabase = 'Probe' except: heading = 'Probe Information' intro = ['Trying to retrieve the probe information for ProbeSet ',HT.Span('%s' % self.ProbeSetID, Class="fwb cdg"),' in Database ',HT.Href(text='%s' % self.db.fullname,url=webqtlConfig.infopagehref % self.database)] detail = ['The information you just requested is not available at this time.'] self.error(heading=heading,intro=intro,detail=detail) return form = HT.Form(cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name='showDatabase', submit=HT.Input(type='hidden')) hddn = {'FormID':'showDatabase','ProbeSetID':'_','database':'_','CellID':'_','RISet':fd.RISet, 'incparentsf1':'on'} if fd.RISet == 'BXD': hddn['parentsf1']='ON' for key in hddn.keys(): form.append(HT.Input(name=key, value=hddn[key], type='hidden')) #Buttons on search page linkinfo ="%s/probeInfo.html" % webqtlConfig.PORTADDR mintmap = "" probeinfo = HT.Input(type='button' ,name='mintmap',value='Info', onClick="openNewWin('%s');" % linkinfo, Class="button") cormatrix = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('showDatabase')[0], 'corMatrix');") cormatrix_img = HT.Image("/images/correlation_matrix1_final.jpg", alt="Correlation Matrix and PCA", title="Correlation Matrix and PCA", style="border:none;") cormatrix.append(cormatrix_img) heatmap = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('showDatabase')[0], 'heatmap');") heatmap_img = HT.Image("/images/heatmap2_final.jpg", name='mintmap', alt="QTL Heat Map and Clustering", title="QTL Heatmap and Clustering", style="border:none;") heatmap.append(heatmap_img) if self.ProbeSetID[-2:] in ('_A', '_B'): thisProbeSetID = self.ProbeSetID[:-2] else: thisProbeSetID = self.ProbeSetID thisurl = 'http://www.ensembl.org/Mus_musculus/featureview?type=AffyProbe&id=%s' % thisProbeSetID verifyButton = HT.Input(type="button",value="Verify Ensembl",onClick= "openNewWin('%s')" % thisurl, Class="button") addselect = HT.Input(type='button' ,name='addselect',value='Add to Collection', onClick="addRmvSelection('%s', this.form, 'addToSelection');" % fd.RISet,Class="button") selectall = HT.Input(type='button' ,name='selectall',value='Select All', onClick="checkAll(this.form);",Class="button") selectpm = HT.Input(type='button' ,name='selectall',value='Select PM', onClick="checkPM(this.form);",Class="button") selectmm = HT.Input(type='button' ,name='selectall',value='Select MM', onClick="checkMM(this.form);",Class="button") selectinvert = HT.Input(type='button' ,name='selectinvert',value='Select Invert', onClick="checkInvert(this.form);",Class="button") reset = HT.Input(type='reset',name='',value='Select None',Class="button") chrMenu = HT.Input(type='hidden',name='chromosomes',value='all') probedata = HT.Input(type='hidden',name='probedata',value='all') url_rudi_track = self.getProbeTrackURL(self.probeDatabase, self.ProbeSetID) if url_rudi_track: rudi_track = HT.Input(type='button', name='ruditrack', value='Probe Track', onClick="openNewWin('%s')"%url_rudi_track, Class="button") else: rudi_track = None pinfopage = "/probeInfo.html" #updated by NL: 07-22-2011 get chosenStrains _f1, _f12, _mat, _pat = webqtlUtil.ParInfo[fd.RISet] chosenStrains="%s,%s"%(_mat,_pat) tblobj = {} tblobj['header']=[] tblobj['header'].append([ THCell(HT.TD("", Class="cbrb cw fwb fs13 b1", rowspan=2,nowrap='ON'), sort=0), THCell(HT.TD(HT.Href(target="_PROBEINFO", url=pinfopage+"#probe", text=HT.Span('Probe', Class="cw fwb fs13")), HT.Sup(HT.Italic('1')), Class="cbrb cw fwb fs13 b1",align='center',rowspan=2,nowrap='ON'), text="probe", idx=1), THCell(HT.TD(HT.Href(text=HT.Span('Sequence', Class="cw fwb fs13"), target="_PROBEINFO", url=pinfopage+"#Sequence"),HT.Sup(HT.Italic('2')), Class="cbrb cw fwb fs13 b1", align='center',rowspan=2,nowrap='ON'), text="seq", idx=2), THCell(HT.TD(HT.Href(text=HT.Span('bl2seq', Class="cw fwb fs13"), target="_PROBEINFO", url=pinfopage+"#bl2seq"),HT.Sup(HT.Italic('3')), Class="cbrb cw fwb fs13 b1", align='center',rowspan=2,nowrap='ON'), sort=0), THCell(HT.TD(HT.Href(text=HT.Span('Exons', Class="cw fwb fs13"), target="_PROBEINFO", url=pinfopage+"#Exon"),HT.Sup(HT.Italic('4')), Class="cbrb cw fwb fs13 b1",align='center',rowspan=2,nowrap='ON'), sort=0), THCell(HT.TD(HT.Href(text=HT.Span('Tm °C', Class="cw fwb fs13"), target="_PROBEINFO", url=pinfopage+"#Tm"),HT.Sup(HT.Italic('5')), Class="cbrb cw fwb fs13 b1",align='center',rowspan=2,nowrap='ON'), text="tm", idx=5), THCell(HT.TD(HT.Href(text=HT.Span('Stacking Energy K', HT.Sub('B'),'T', Class="cw fwb fs13"), target="_PROBEINFO", url=pinfopage+"#KBT"),HT.Sup(HT.Italic('6')), Class="cbrb cw fwb fs13 b1",align='center',colspan=2,NOWRAP="yes",nowrap='ON'), sort=0), THCell(HT.TD(HT.Href(text=HT.Span('Mean', Class="cw fwb fs13"), target="_PROBEINFO", url=pinfopage+"#Mean"),HT.Sup(HT.Italic('7')), Class="cbrb cw fwb fs13 b1",align='center',rowspan=2,nowrap='ON'), text="mean", idx=8), THCell(HT.TD(HT.Href(text=HT.Span('Stdev', Class="cw fwb fs13"), target="_PROBEINFO", url=pinfopage+"#Stdev"),HT.Sup(HT.Italic('8')), Class="cbrb cw fwb fs13 b1",align='center',rowspan=2,nowrap='ON'), text="std", idx=9), THCell(HT.TD(HT.Href(text=HT.Span('Probe h2', Class="cw fwb fs13"), target="_PROBEINFO", url=pinfopage+"#h2"),HT.Sup(HT.Italic('9')), Class="cbrb cw fwb fs13 b1",align='center',rowspan=2,NOWRAP="yes"), text="h2", idx=10), THCell(HT.TD(HT.Href(text=HT.Span('Probe Location', Class="cw fwb fs13"), target="_PROBEINFO", url=pinfopage+"#location"), HT.Sup(HT.Italic('10')),Class="cbrb cw fwb fs13 b1",align='center',colspan=3)), THCell(HT.TD(HT.Href(text=HT.Span('SNPs', HT.BR(), '(Across all strains)', Class="cw fwb fs13"), target="_PROBEINFO", url=pinfopage+"#snps"), HT.Sup(HT.Italic('11')),Class="cbrb cw fwb fs13 b1",align='center',rowspan=2,NOWRAP="yes")), THCell(HT.TD(HT.Href(text=HT.Span('SNPs', HT.BR(),'(Different alleles only between %s and %s)'%(_mat,_pat), Class="cw fwb fs13"), target="_PROBEINFO", url=pinfopage+"#snps"), HT.Sup(HT.Italic('11')),Class="cbrb cw fwb fs13 b1",align='center',rowspan=2,NOWRAP="yes")) ]) tblobj['header'].append([ THCell(HT.TD(HT.Span('GSB', Class="cw fwb fs13"),align='center', Class="cbrb ffl fwb fs13 b1",), text="gsb", idx=6), THCell(HT.TD(HT.Span('NSB', Class="cw fwb fs13"),align='center', Class="cbrb ffl fwb fs13 b1",), text="nsb", idx=7), THCell(HT.TD(HT.Span('Chr', Class="cw fwb fs13"), align='center', Class="cbrb ffl2 fwb fs13 b1",)), THCell(HT.TD(HT.Span('Start', Class="cw fwb fs13"),align='center', Class="cbrb ffl fwb fs13 b1",)), THCell(HT.TD(HT.Span('End', Class="cw fwb fs13"),align='center', Class="cbrb ffl fwb fs13 b1",)), ]) tblobj['body'] = [] blatbutton = '' fetchField = ['Probe.Name','Probe.Sequence','Probe.ExonNo','Probe.Tm', 'Probe.E_GSB','Probe.E_NSB', 'ProbeH2.h2', 'ProbeH2.weight'] query = "SELECT %s FROM (Probe, ProbeSet, ProbeFreeze) left join ProbeH2 on ProbeH2.ProbeId = Probe.Id and ProbeH2.ProbeFreezeId = ProbeFreeze.Id WHERE ProbeSet.Name = '%s' and Probe.ProbeSetId = ProbeSet.Id and ProbeFreeze.Name = '%s' order by Probe.SerialOrder" % (string.join(fetchField,','), self.ProbeSetID, self.probeDatabase) self.cursor.execute(query) results = self.cursor.fetchall() blatsequence = "" # add by NL: get strains' name in SnpPattern database table strainsInSnpPatternDBtable=self.getStrainNameIndexPair() # after snpBrowserPage.py change to MVC, this function can be removed in this class and called from other class; allStrainNameList=[v[0] for v in strainsInSnpPatternDBtable] speciesid = webqtlDatabaseFunction.retrieveSpeciesId(cursor=self.cursor,RISet=fd.RISet) for result in results: """ ProbeId, CellID,Sequence,ExonNo,Tm, E_GSB,E_NSB = map(self.nullRecord,result) h2 = '' query = "SELECT h2 FROM ProbeH2 WHERE ProbeFreezeId = '%s' and ProbeId=%s" % (self.probeDatabase, ProbeId) self.cursor.execute(query) results = self.cursor.fetchall() """ CellID,Sequence,ExonNo,Tm, E_GSB,E_NSB,h2, weight = map(self.nullRecord,result) Average = '' STDEV = '' mean = -10000.0 stdev = -10000.0 try: thisTrait.cellid = CellID thisTrait.retrieveData() mean, median, var, stdev, sem, N = reaper.anova(thisTrait.exportInformative()[1]) if mean: Average = '%2.2f' % mean if stdev: STDEV = '%2.2f' % stdev except: pass if CellID == self.CellID: bkColor = "cbrdull fs11 b1" else: bkColor = "fs11 b1" seqcolor= '' if thisTrait.blatseq: blatsequence = thisTrait.blatseq if int(CellID[-1]) % 2 == 1: seqcolor= 'cdg' else: if int(CellID[-1]) % 2 == 1: seqcolor= 'cdg' blatsequence += string.strip(Sequence) if thisTrait.genbankid and (int(CellID[-1]) % 2 == 1): probeurl = 'http://www.ncbi.nlm.nih.gov/blast/bl2seq/wblast2.cgi?one=%s&sseq=%s' % (thisTrait.genbankid, Sequence) probefy1 = HT.Input(type="button",value="Blast",onClick= "openNewWin('%s')" % probeurl, Class="buttonsmaller") else: probefy1 = '' traitName = str(thisTrait) #XZ, Aug 08, 2011: Note that probesets on some affy chips are not name as "xxx_at" (i.e., Affy Mouse Gene 1.0 ST (GPL6246)). #EnsemblProbeSetID = self.ProbeSetID[0:self.ProbeSetID.index('_at')+3] EnsemblProbeSetID = self.ProbeSetID if '_at' in self.ProbeSetID: EnsemblProbeSetID = self.ProbeSetID[0:self.ProbeSetID.index('_at')+3] self.cursor.execute(''' SELECT EnsemblProbeLocation.* FROM EnsemblProbeLocation, EnsemblProbe, EnsemblChip, GeneChipEnsemblXRef, ProbeFreeze WHERE EnsemblProbeLocation.ProbeId=EnsemblProbe.Id and EnsemblProbe.ChipId=GeneChipEnsemblXRef.EnsemblChipId and GeneChipEnsemblXRef.GeneChipId=ProbeFreeze.ChipId and EnsemblProbe.Name=%s and EnsemblProbe.ProbeSet=%s and ProbeFreeze.Name=%s group by Chr, Start, End''' ,(CellID, EnsemblProbeSetID, self.probeDatabase)) LocationFields = self.cursor.fetchall() Chr='' Start='' End='' if (len(LocationFields)>=1): Chr,Start,End,Strand,MisMatch,ProbeId = map(self.nullRecord,LocationFields[0]) Start /= 1000000.0 End /= 1000000.0 if (len(LocationFields)>1): self.cursor.execute(''' SELECT ProbeSet.Chr, ProbeSet.Mb FROM ProbeSet, ProbeFreeze WHERE ProbeSet.ChipId=ProbeFreeze.ChipId and ProbeSet.Name=%s and ProbeFreeze.Name=%s''' ,(self.ProbeSetID, self.probeDatabase)) ProbeSetChr, ProbeSetMb = map(self.nullRecord,self.cursor.fetchall()[0]) self.cursor.execute(''' SELECT EnsemblProbeLocation.*, ABS(EnsemblProbeLocation.Start/1000000-%s) as Mb FROM EnsemblProbeLocation, EnsemblProbe, EnsemblChip, GeneChipEnsemblXRef, ProbeFreeze WHERE EnsemblProbeLocation.ProbeId=EnsemblProbe.Id and EnsemblProbe.ChipId=GeneChipEnsemblXRef.EnsemblChipId and GeneChipEnsemblXRef.GeneChipId=ProbeFreeze.ChipId and EnsemblProbe.Name=%s and EnsemblProbe.ProbeSet=%s and EnsemblProbeLocation.Chr=%s and ProbeFreeze.Name=%s order by Mb limit 1''' ,(ProbeSetMb, CellID, EnsemblProbeSetID, ProbeSetChr, self.probeDatabase)) NewLocationFields = self.cursor.fetchall() if (len(NewLocationFields)>0): Chr,Start,End,Strand,MisMatch,ProbeId,Mb = map(self.nullRecord,NewLocationFields[0]) Start /= 1000000.0 End /= 1000000.0 snp_collection = [] snpDiff_collection=[] startIndex=3 if Chr != '' and Start != '' and End != '' and speciesid != None: self.cursor.execute(''' SELECT a.SnpName, a.Id, b.* FROM SnpAll a, SnpPattern b WHERE a.Chromosome=%s and a.Position>=%s and a.Position<=%s and a.SpeciesId=%s and a.Id=b.SnpId''' ,(Chr, Start, End, speciesid)) #chr,Start, End, 1)) snpresults = self.cursor.fetchall() index1=allStrainNameList.index(_mat) #_mat index in results index2=allStrainNameList.index(_pat) #_pat index in results for v in snpresults: #updated by NL: 07-22-2011 check 'limit to' to get snpBrowser snpresults snp_collection.append(HT.Href(text=v[0], url=os.path.join(webqtlConfig.CGIDIR, "main.py?FormID=SnpBrowserResultPage&submitStatus=1&customStrain=1")+ "&geneName=%s" % v[0], Class="fs12 fwn", target="_blank")) snp_collection.append(HT.BR()) #updated by NL: 07-27-2011 link snp info for different allele only strain1_allele=v[startIndex+index1] strain2_allele=v[startIndex+index2] if strain1_allele!=strain2_allele: snpDiff_collection.append(HT.Href(text=v[0], url=os.path.join(webqtlConfig.CGIDIR, "main.py?FormID=SnpBrowserResultPage&submitStatus=1&customStrain=1&diffAlleles=1&chosenStrains=%s"%chosenStrains)+ "&geneName=%s" % v[0], Class="fs12 fwn", target="_blank")) snpDiff_collection.append(HT.BR()) tr = [] tr.append(TDCell(HT.TD(HT.Input(type="checkbox", Class='checkbox', name="searchResult",value=traitName, onClick="highlight(this)"), align="right", Class=bkColor, nowrap="on"), text=traitName)) tr.append(TDCell(HT.TD(HT.Href(text=CellID, url = "javascript:showDatabase2('%s','%s','%s');" % (self.database,self.ProbeSetID,CellID),Class="fs12 fwn"),Class=bkColor), traitName, traitName.upper())) tr.append(TDCell(HT.TD(Sequence, Class=bkColor + " %s ffmono fs14" % seqcolor),Sequence,Sequence.upper())) tr.append(TDCell(HT.TD(probefy1,align='center',Class=bkColor))) tr.append(TDCell(HT.TD(ExonNo,align='center',Class=bkColor))) try: TmValue = float(Tm) except: TmValue = 0.0 tr.append(TDCell(HT.TD(Tm,align='center',Class=bkColor), Tm, TmValue)) try: E_GSBValue = float(E_GSB) except: E_GSBValue = -10000.0 tr.append(TDCell(HT.TD(E_GSB,align='center',Class=bkColor), E_GSB, E_GSBValue)) try: E_NSBValue = float(E_NSB) except: E_NSBValue = -10000.0 tr.append(TDCell(HT.TD(E_NSB,align='center',Class=bkColor), E_NSB, E_NSBValue)) tr.append(TDCell(HT.TD(Average,align='center',Class=bkColor), Average, mean)) tr.append(TDCell(HT.TD(STDEV,align='center',Class=bkColor), STDEV, stdev)) try: h2Value = float(h2) except: h2Value = -10000.0 tr.append(TDCell(HT.TD(h2,align='center',Class=bkColor), h2, h2Value)) tr.append(TDCell(HT.TD(Chr,align='left',Class=bkColor))) tr.append(TDCell(HT.TD(Start,align='left',Class=bkColor))) tr.append(TDCell(HT.TD(End,align='left',Class=bkColor))) snp_td = HT.TD(align='left',Class=bkColor) for one_snp_href in snp_collection: snp_td.append(one_snp_href) tr.append(TDCell(snp_td)) #07-27-2011:add by NL: show SNP results for different allele only snpDiff_td= HT.TD(align='left', valign='top', Class=bkColor) for one_snpDiff_href in snpDiff_collection: snpDiff_td.append(one_snpDiff_href) tr.append(TDCell(snpDiff_td)) tblobj['body'].append(tr) # import cPickle filename = webqtlUtil.genRandStr("Probe_") objfile = open('%s.obj' % (webqtlConfig.TMPDIR+filename), 'wb') cPickle.dump(tblobj, objfile) objfile.close() # NL, 07/27/2010. genTableObj function has been moved from templatePage.py to webqtlUtil.py; div = HT.Div(webqtlUtil.genTableObj(tblobj=tblobj, file=filename, sortby=("", ""), tableID = "sortable", addIndex = "1"), Id="sortable") #UCSC _Species = webqtlDatabaseFunction.retrieveSpecies(cursor=self.cursor, RISet=fd.RISet) if _Species == "rat": thisurl = webqtlConfig.UCSC_BLAT % ('rat', 'rn3', blatsequence) elif _Species == "mouse": thisurl = webqtlConfig.UCSC_BLAT % ('mouse', 'mm9', blatsequence) else: thisurl = "" if thisurl: blatbutton = HT.Input(type='button' ,name='blatPM',value='Verify UCSC', onClick="window.open('%s','_blank')" % thisurl,Class="button") else: blatbutton = "" #GenBank genbankSeq = "" if thisTrait.genbankid: self.cursor.execute("SELECT Sequence FROM Genbank WHERE Id = '%s'" % thisTrait.genbankid ) genbankSeq = self.cursor.fetchone() if genbankSeq: genbankSeq = genbankSeq[0] if genbankSeq: if _Species == "rat": thisurl2 = webqtlConfig.UCSC_BLAT % ('rat', 'rn3', genbankSeq) if _Species == "mouse": thisurl2 = webqtlConfig.UCSC_BLAT % ('mouse', 'mm9', genbankSeq) else: thisurl2 = '' if thisurl2: blatbutton2 = HT.Input(type='button' ,name='blatPM',value='Verify GenBank', onClick="window.open('%s','_blank')" % thisurl2,Class="button") else: blatbutton2 = "" #Snp snpBrowser = "" if thisTrait.symbol and _Species == 'mouse': self.cursor.execute("select geneSymbol from GeneList where geneSymbol = %s", thisTrait.symbol) geneName = self.cursor.fetchone() if geneName: snpurl = os.path.join(webqtlConfig.CGIDIR, "main.py?FormID=snpBrowser") + "&geneName=%s" % geneName[0] else: if thisTrait.chr and thisTrait.mb: snpurl = os.path.join(webqtlConfig.CGIDIR, "main.py?FormID=snpBrowser") + \ "&chr=%s&start=%2.6f&end=%2.6f" % (thisTrait.chr, thisTrait.mb-0.002, thisTrait.mb+0.002) else: snpurl = "" if snpurl: snpBrowser = HT.Input(type="button",value="SNP Browser",onClick= \ "openNewWin('%s')" % snpurl, Class="button") else: snpBrowser = "" #end if heading = HT.Paragraph('Probe Information', Class="title") intro = HT.Paragraph('The table below lists information of all probes of probe set ',HT.Span(self.ProbeSetID, Class="fwb fs13"),' from database ', HT.Span(self.probeDatabase, Class="fwb fs13"), ".") buttons = HT.Paragraph(probedata,probeinfo,heatmap,cormatrix,blatbutton,blatbutton2,verifyButton,snpBrowser, HT.P(),selectall,selectpm,selectmm,selectinvert,reset,addselect) if rudi_track: buttons.append(rudi_track) form.append(buttons,div,HT.P()) TD_LR.append(heading,intro,form, HT.P()) self.dict['basehref'] = '' self.dict['body'] = str(TD_LR) self.dict['title'] = self.db.shortname + ' : ' + self.ProbeSetID +' / Probe Information' # updated by NL, javascript function xmlhttpPost(strURL, div, querystring) and function updatepage(Id, str) # have been moved to dhtml.js self.dict['js1'] = ''
def run_rqtl_geno(vals, samples, dataset, method, model, permCheck, num_perm, perm_strata_list, do_control, control_marker, manhattan_plot, pair_scan, cofactors): ## Get pointers to some common R functions r_library = ro.r["library"] # Map the library function r_c = ro.r["c"] # Map the c function plot = ro.r["plot"] # Map the plot function png = ro.r["png"] # Map the png function dev_off = ro.r["dev.off"] # Map the device off function print(r_library("qtl")) # Load R/qtl ## Get pointers to some R/qtl functions scanone = ro.r["scanone"] # Map the scanone function scantwo = ro.r["scantwo"] # Map the scantwo function calc_genoprob = ro.r["calc.genoprob"] # Map the calc.genoprob function crossname = dataset.group.name #try: # generate_cross_from_rdata(dataset) # read_cross_from_rdata = ro.r["generate_cross_from_rdata"] # Map the local read_cross_from_rdata function # genofilelocation = locate(crossname + ".RData", "genotype/rdata") # cross_object = read_cross_from_rdata(genofilelocation) # Map the local GENOtoCSVR function #except: generate_cross_from_geno(dataset) GENOtoCSVR = ro.r["GENOtoCSVR"] # Map the local GENOtoCSVR function crossfilelocation = TMPDIR + crossname + ".cross" if dataset.group.genofile: genofilelocation = locate(dataset.group.genofile, "genotype") else: genofilelocation = locate(dataset.group.name + ".geno", "genotype") cross_object = GENOtoCSVR(genofilelocation, crossfilelocation) # TODO: Add the SEX if that is available if manhattan_plot: cross_object = calc_genoprob(cross_object) else: cross_object = calc_genoprob(cross_object, step=1, stepwidth="max") pheno_string = sanitize_rqtl_phenotype(vals) cross_object = add_phenotype(cross_object, pheno_string, "the_pheno") # Add the phenotype # Scan for QTLs marker_covars = create_marker_covariates(control_marker, cross_object) # Create the additive covariate markers if cofactors != "": cross_object, trait_covars = add_cofactors(cross_object, dataset, cofactors, samples) # Create the covariates from selected traits ro.r('all_covars <- cbind(marker_covars, trait_covars)') else: ro.r('all_covars <- marker_covars') covars = ro.r['all_covars'] if pair_scan: if do_control == "true": logger.info("Using covariate"); result_data_frame = scantwo(cross_object, pheno = "the_pheno", addcovar = covars, model=model, method=method, n_cluster = 16) else: logger.info("No covariates"); result_data_frame = scantwo(cross_object, pheno = "the_pheno", model=model, method=method, n_cluster = 16) pair_scan_filename = webqtlUtil.genRandStr("scantwo_") + ".png" png(file=TEMPDIR+pair_scan_filename) plot(result_data_frame) dev_off() return process_pair_scan_results(result_data_frame) else: if do_control == "true" or cofactors != "": logger.info("Using covariate"); result_data_frame = scanone(cross_object, pheno = "the_pheno", addcovar = covars, model=model, method=method) else: logger.info("No covariates"); result_data_frame = scanone(cross_object, pheno = "the_pheno", model=model, method=method) if num_perm > 0 and permCheck == "ON": # Do permutation (if requested by user) if len(perm_strata_list) > 0: #ZS: The strata list would only be populated if "Stratified" was checked on before mapping cross_object, strata_ob = add_perm_strata(cross_object, perm_strata_list) if do_control == "true" or cofactors != "": perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", addcovar = covars, n_perm = int(num_perm), perm_strata = strata_ob, model=model, method=method) else: perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", n_perm = num_perm, perm_strata = strata_ob, model=model, method=method) else: if do_control == "true" or cofactors != "": perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", addcovar = covars, n_perm = int(num_perm), model=model, method=method) else: perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", n_perm = num_perm, model=model, method=method) perm_output, suggestive, significant = process_rqtl_perm_results(num_perm, perm_data_frame) # Functions that sets the thresholds for the webinterface the_scale = check_mapping_scale(genofilelocation) return perm_output, suggestive, significant, process_rqtl_results(result_data_frame, dataset.group.species), the_scale else: the_scale = check_mapping_scale(genofilelocation) return process_rqtl_results(result_data_frame, dataset.group.species), the_scale
def run_analysis(self, requestform): logger.info("Starting PheWAS analysis on dataset") genofilelocation = locate( "BXD.geno", "genotype") # Get the location of the BXD genotypes precompfile = locate_phewas( "PheWAS_pval_EMMA_norm.RData", "auwerx") # Get the location of the pre-computed EMMA results # Get user parameters, trait_id and dataset, and store/update them in self self.trait_id = requestform["trait_id"] self.datasetname = requestform["dataset"] self.dataset = data_set.create_dataset(self.datasetname) self.region = int(requestform["num_region"]) self.mtadjust = str(requestform["sel_mtadjust"]) # Logger.Info some debug logger.info("self.trait_id:" + self.trait_id + "\n") logger.info("self.datasetname:" + self.datasetname + "\n") logger.info("self.dataset.type:" + self.dataset.type + "\n") # GN Magic ? self.this_trait = GeneralTrait(dataset=self.dataset, name=self.trait_id, get_qtl_info=False, get_sample_info=False) logger.info(vars(self.this_trait)) # Set the values we need self.chr = str(self.this_trait.chr) self.mb = int(self.this_trait.mb) # logger.info some debug logger.info("location:" + self.chr + ":" + str(self.mb) + "+/-" + str(self.region) + "\n") # Load in the genotypes file *sigh* to make the markermap parser = genofile_parser.ConvertGenoFile(genofilelocation) parser.process_csv() snpinfo = [] for marker in parser.markers: snpinfo.append(marker["name"]) snpinfo.append(marker["chr"]) snpinfo.append(marker["Mb"]) rnames = r_seq(1, len(parser.markers)) # Create the snp aligner object out of the BXD genotypes snpaligner = ro.r.matrix(snpinfo, nrow=len(parser.markers), dimnames=r_list(rnames, r_c("SNP", "Chr", "Pos")), ncol=3, byrow=True) # Create the phenotype aligner object using R phenoaligner = self.r_create_Pheno_aligner() self.results = {} self.results['imgurl1'] = webqtlUtil.genRandStr("phewas_") + ".png" self.results['imgloc1'] = GENERATED_IMAGE_DIR + self.results['imgurl1'] self.results['mtadjust'] = self.mtadjust logger.info("IMAGE AT:", self.results['imgurl1']) logger.info("IMAGE AT:", self.results['imgloc1']) # Create the PheWAS plot (The gene/probe name, chromosome and gene/probe positions should come from the user input) # TODO: generate the PDF in the temp folder, with a unique name assert (precompfile) assert (phenoaligner) assert (snpaligner) phewasres = self.r_PheWASManhattan("Test", precompfile, phenoaligner, snpaligner, "None", self.chr, self.mb, self.region, self.results['imgloc1'], self.mtadjust) self.results['phewas1'] = phewasres[0] self.results['phewas2'] = phewasres[1] self.results['tabulardata'] = phewasres[2] self.results['R_debuglog'] = phewasres[3] #self.r_PheWASManhattan(allpvalues) #self.r_Stop() logger.info("Initialization of PheWAS done !")
def run_analysis(self, requestform): print("Starting WGCNA analysis on dataset") self.r_enableWGCNAThreads() # Enable multi threading self.trait_db_list = [trait.strip() for trait in requestform['trait_list'].split(',')] print("Retrieved phenotype data from database", requestform['trait_list']) helper_functions.get_trait_db_obs(self, self.trait_db_list) self.input = {} # self.input contains the phenotype values we need to send to R strains = [] # All the strains we have data for (contains duplicates) traits = [] # All the traits we have data for (should not contain duplicates) for trait in self.trait_list: traits.append(trait[0].name) self.input[trait[0].name] = {} for strain in trait[0].data: strains.append(strain) self.input[trait[0].name][strain] = trait[0].data[strain].value # Transfer the load data from python to R uStrainsR = r_unique(ro.Vector(strains)) # Unique strains in R vector uTraitsR = r_unique(ro.Vector(traits)) # Unique traits in R vector r_cat("The number of unique strains:", r_length(uStrainsR), "\n") r_cat("The number of unique traits:", r_length(uTraitsR), "\n") # rM is the datamatrix holding all the data in R /rows = strains columns = traits rM = ro.r.matrix(ri.NA_Real, nrow=r_length(uStrainsR), ncol=r_length(uTraitsR), dimnames = r_list(uStrainsR, uTraitsR)) for t in uTraitsR: trait = t[0] # R uses vectors every single element is a vector for s in uStrainsR: strain = s[0] # R uses vectors every single element is a vector #DEBUG: print(trait, strain, " in python: ", self.input[trait].get(strain), "in R:", rM.rx(strain,trait)[0]) rM.rx[strain, trait] = self.input[trait].get(strain) # Update the matrix location sys.stdout.flush() self.results = {} self.results['nphe'] = r_length(uTraitsR)[0] # Number of phenotypes/traits self.results['nstr'] = r_length(uStrainsR)[0] # Number of strains self.results['phenotypes'] = uTraitsR # Traits used self.results['strains'] = uStrainsR # Strains used in the analysis self.results['requestform'] = requestform # Store the user specified parameters for the output page # Calculate soft threshold if the user specified the SoftThreshold variable if requestform.get('SoftThresholds') is not None: powers = [int(threshold.strip()) for threshold in requestform['SoftThresholds'].rstrip().split(",")] rpow = r_unlist(r_c(powers)) print "SoftThresholds: {} == {}".format(powers, rpow) self.sft = self.r_pickSoftThreshold(rM, powerVector = rpow, verbose = 5) print "PowerEstimate: {}".format(self.sft[0]) self.results['PowerEstimate'] = self.sft[0] if self.sft[0][0] is ri.NA_Integer: print "No power is suitable for the analysis, just use 1" self.results['Power'] = 1 # No power could be estimated else: self.results['Power'] = self.sft[0][0] # Use the estimated power else: # The user clicked a button, so no soft threshold selection self.results['Power'] = requestform.get('Power') # Use the power value the user gives # Create the block wise modules using WGCNA network = self.r_blockwiseModules(rM, power = self.results['Power'], TOMType = requestform['TOMtype'], minModuleSize = requestform['MinModuleSize'], verbose = 3) # Save the network for the GUI self.results['network'] = network # How many modules and how many gene per module ? print "WGCNA found {} modules".format(r_table(network[1])) self.results['nmod'] = r_length(r_table(network[1]))[0] # The iconic WCGNA plot of the modules in the hanging tree self.results['imgurl'] = webqtlUtil.genRandStr("WGCNAoutput_") + ".png" self.results['imgloc'] = webqtlConfig.TMPDIR + self.results['imgurl'] r_png(self.results['imgloc'], width=1000, height=600) mergedColors = self.r_labels2colors(network[1]) self.r_plotDendroAndColors(network[5][0], mergedColors, "Module colors", dendroLabels = False, hang = 0.03, addGuide = True, guideHang = 0.05) r_dev_off() sys.stdout.flush()
def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control, control_marker, manhattan_plot, pair_scan): geno_to_rqtl_function(dataset) ## Get pointers to some common R functions r_library = ro.r["library"] # Map the library function r_c = ro.r["c"] # Map the c function plot = ro.r["plot"] # Map the plot function png = ro.r["png"] # Map the png function dev_off = ro.r["dev.off"] # Map the device off function print(r_library("qtl")) # Load R/qtl ## Get pointers to some R/qtl functions scanone = ro.r["scanone"] # Map the scanone function scantwo = ro.r["scantwo"] # Map the scantwo function calc_genoprob = ro.r["calc.genoprob"] # Map the calc.genoprob function GENOtoCSVR = ro.r["GENOtoCSVR"] # Map the local GENOtoCSVR function crossname = dataset.group.name genofilelocation = locate(crossname + ".geno", "genotype") crossfilelocation = TMPDIR + crossname + ".cross" cross_object = GENOtoCSVR(genofilelocation, crossfilelocation) # TODO: Add the SEX if that is available if manhattan_plot: cross_object = calc_genoprob(cross_object) else: cross_object = calc_genoprob(cross_object, step=1, stepwidth="max") cross_object = add_phenotype(cross_object, sanitize_rqtl_phenotype(vals)) # Add the phenotype # Scan for QTLs covar = create_covariates(control_marker, cross_object) # Create the additive covariate matrix if pair_scan: if do_control == "true": logger.info("Using covariate"); result_data_frame = scantwo(cross_object, pheno = "the_pheno", addcovar = covar, model=model, method=method, n_cluster = 16) else: logger.info("No covariates"); result_data_frame = scantwo(cross_object, pheno = "the_pheno", model=model, method=method, n_cluster = 16) pair_scan_filename = webqtlUtil.genRandStr("scantwo_") + ".png" png(file=TEMPDIR+pair_scan_filename) plot(result_data_frame) dev_off() return process_pair_scan_results(result_data_frame) else: if do_control == "true": logger.info("Using covariate"); result_data_frame = scanone(cross_object, pheno = "the_pheno", addcovar = covar, model=model, method=method) else: logger.info("No covariates"); result_data_frame = scanone(cross_object, pheno = "the_pheno", model=model, method=method) if num_perm > 0 and permCheck == "ON": # Do permutation (if requested by user) if do_control == "true": perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", addcovar = covar, n_perm = num_perm, model=model, method=method) else: perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", n_perm = num_perm, model=model, method=method) perm_output, suggestive, significant = process_rqtl_perm_results(num_perm, perm_data_frame) # Functions that sets the thresholds for the webinterface return perm_output, suggestive, significant, process_rqtl_results(result_data_frame) else: return process_rqtl_results(result_data_frame)
def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control, control_marker, manhattan_plot, pair_scan): geno_to_rqtl_function(dataset) ## Get pointers to some common R functions r_library = ro.r["library"] # Map the library function r_c = ro.r["c"] # Map the c function r_sum = ro.r["sum"] # Map the sum function plot = ro.r["plot"] # Map the plot function postscript = ro.r["postscript"] # Map the postscript function png = ro.r["png"] # Map the png function dev_off = ro.r["dev.off"] # Map the device off function print(r_library("qtl")) # Load R/qtl ## Get pointers to some R/qtl functions scanone = ro.r["scanone"] # Map the scanone function scantwo = ro.r["scantwo"] # Map the scantwo function calc_genoprob = ro.r["calc.genoprob"] # Map the calc.genoprob function read_cross = ro.r["read.cross"] # Map the read.cross function write_cross = ro.r["write.cross"] # Map the write.cross function GENOtoCSVR = ro.r["GENOtoCSVR"] # Map the local GENOtoCSVR function crossname = dataset.group.name genofilelocation = locate(crossname + ".geno", "genotype") crossfilelocation = TMPDIR + crossname + ".cross" #print("Conversion of geno to cross at location:", genofilelocation, " to ", crossfilelocation) cross_object = GENOtoCSVR( genofilelocation, crossfilelocation) # TODO: Add the SEX if that is available if manhattan_plot: cross_object = calc_genoprob(cross_object) else: cross_object = calc_genoprob(cross_object, step=1, stepwidth="max") cross_object = add_phenotype( cross_object, sanitize_rqtl_phenotype(vals)) # Add the phenotype # for debug: write_cross(cross_object, "csvr", "test.csvr") # Scan for QTLs covar = create_covariates( control_marker, cross_object) # Create the additive covariate matrix if pair_scan: if do_control == "true": # If sum(covar) > 0 we have a covariate matrix print("Using covariate") result_data_frame = scantwo(cross_object, pheno="the_pheno", addcovar=covar, model=model, method=method, n_cluster=16) else: print("No covariates") result_data_frame = scantwo(cross_object, pheno="the_pheno", model=model, method=method, n_cluster=16) #print("Pair scan results:", result_data_frame) pair_scan_filename = webqtlUtil.genRandStr("scantwo_") + ".png" png(file=TEMPDIR + pair_scan_filename) plot(result_data_frame) dev_off() return process_pair_scan_results(result_data_frame) else: if do_control == "true": print("Using covariate") result_data_frame = scanone(cross_object, pheno="the_pheno", addcovar=covar, model=model, method=method) else: print("No covariates") result_data_frame = scanone(cross_object, pheno="the_pheno", model=model, method=method) if num_perm > 0 and permCheck == "ON": # Do permutation (if requested by user) if do_control == "true": perm_data_frame = scanone(cross_object, pheno_col="the_pheno", addcovar=covar, n_perm=num_perm, model=model, method=method) else: perm_data_frame = scanone(cross_object, pheno_col="the_pheno", n_perm=num_perm, model=model, method=method) perm_output, suggestive, significant = process_rqtl_perm_results( num_perm, perm_data_frame ) # Functions that sets the thresholds for the webinterface return perm_output, suggestive, significant, process_rqtl_results( result_data_frame) else: return process_rqtl_results(result_data_frame)
def __init__(self, fd): templatePage.__init__(self, fd) if not self.openMysql(): return searchResult = fd.formdata.getvalue("searchResult") if not searchResult: heading = "Partial Correlation" detail = ["You need to select at least three traits in order to calculate partial correlation."] self.error(heading=heading, detail=detail) return ## Adds the Trait instance for each trait name from the collection traits = [] for item in searchResult: traits.append(webqtlTrait(fullname=item, cursor=self.cursor)) RISet = fd.formdata.getvalue("RISet") species = webqtlDatabaseFunction.retrieveSpecies(cursor=self.cursor, RISet=RISet) # XZ: HTML part TD_LR = HT.TD(colspan=2, height=200, width="100%", bgColor="#eeeeee") TD_LR.append( "Please select one primary trait, one to three control traits, and at least one target trait.", HT.P() ) mainFormName = "showDatabase" mainForm = HT.Form( cgi=os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), name=mainFormName, submit=HT.Input(type="hidden"), ) # XZ: Add hidden form values hddn = { "FormID": "calPartialCorrTrait", "database": "", "ProbeSetID": "", "CellID": "", # XZ: These four parameters are required by javascript function showDatabase2. "controlTraits": "", "primaryTrait": "", "targetTraits": "", "pcMethod": "", "RISet": RISet, } for key in hddn.keys(): mainForm.append(HT.Input(type="hidden", name=key, value=hddn[key])) radioNames = [] for thisTrait in traits: oneRadioName = thisTrait.getName() radioNames.append(oneRadioName) radioNamesString = ",".join(radioNames) # Creates the image href that runs the javascript setting all traits as target or ignored setAllTarget = HT.Href( url="#redirect", onClick="setAllAsTarget(document.getElementsByName('showDatabase')[0], '%s');" % radioNamesString, ) setAllTargetImg = HT.Image("/images/select_all.gif", alt="Select All", title="Select All", style="border:none;") setAllTarget.append(setAllTargetImg) setAllIgnore = HT.Href( url="#redirect", onClick="setAllAsIgnore(document.getElementsByName('showDatabase')[0], '%s');" % radioNamesString, ) setAllIgnoreImg = HT.Image("/images/select_all.gif", alt="Select All", title="Select All", style="border:none;") setAllIgnore.append(setAllIgnoreImg) tblobj = {} tblobj["header"] = self.getCollectionTableHeader() sortby = self.getSortByValue() tblobj["body"] = self.getCollectionTableBody(traitList=traits, formName=mainFormName, species=species) filename = webqtlUtil.genRandStr("Search_") objfile = open("%s.obj" % (webqtlConfig.TMPDIR + filename), "wb") cPickle.dump(tblobj, objfile) objfile.close() div = HT.Div( webqtlUtil.genTableObj(tblobj=tblobj, file=filename, sortby=sortby, tableID="sortable", addIndex="1"), Id="sortable", ) mainForm.append(div) # XZ: Add button radioNamesString = ",".join(radioNames) jsCommand_1 = "validateTrait(this.form, '" + radioNamesString + "', 0, 1);" jsCommand_2 = "validateTrait(this.form, '" + radioNamesString + "', 0, 2);" partialCorrTraitButton_1 = HT.Input( type="button", name="submitPartialCorrTrait_1", value="Pearson's r", onClick="%s" % jsCommand_1, Class="button", ) partialCorrTraitButton_2 = HT.Input( type="button", name="submitPartialCorrTrait_2", value="Spearman's rho", onClick="%s" % jsCommand_2, Class="button", ) mainForm.append( HT.BR(), "Compute partial correlation for target selected above:", HT.BR(), partialCorrTraitButton_1, partialCorrTraitButton_2, HT.BR(), HT.BR(), HT.HR(color="gray", size=3), ) jsCommand = "validateTrait(this.form, '" + radioNamesString + "', 1);" partialCorrDBButton = HT.Input( type="button", name="submitPartialCorrDB", value="Calculate", onClick="%s" % jsCommand, Class="button" ) methodText = HT.Span("Calculate:", Class="ffl fwb fs12") methodMenu = HT.Select(name="method") methodMenu.append(("Genetic Correlation, Pearson's r", "1")) methodMenu.append(("Genetic Correlation, Spearman's rho", "2")) methodMenu.append(("SGO Literature Correlation", "3")) methodMenu.append(("Tissue Correlation, Pearson's r", "4")) methodMenu.append(("Tissue Correlation, Spearman's rho", "5")) databaseText = HT.Span("Choose Database:", Class="ffl fwb fs12") databaseMenu = HT.Select(name="database2") nmenu = 0 self.cursor.execute( 'SELECT PublishFreeze.FullName,PublishFreeze.Name FROM \ PublishFreeze,InbredSet WHERE PublishFreeze.InbredSetId = InbredSet.Id \ and InbredSet.Name = "%s" and PublishFreeze.public > %d' % (RISet, webqtlConfig.PUBLICTHRESH) ) for item in self.cursor.fetchall(): databaseMenu.append(item) nmenu += 1 self.cursor.execute( 'SELECT GenoFreeze.FullName,GenoFreeze.Name FROM GenoFreeze,\ InbredSet WHERE GenoFreeze.InbredSetId = InbredSet.Id and InbredSet.Name = \ "%s" and GenoFreeze.public > %d' % (RISet, webqtlConfig.PUBLICTHRESH) ) for item in self.cursor.fetchall(): databaseMenu.append(item) nmenu += 1 # 03/09/2009: Xiaodong changed the SQL query to order by Name as requested by Rob. self.cursor.execute("SELECT Id, Name FROM Tissue order by Name") for item in self.cursor.fetchall(): TId, TName = item databaseMenuSub = HT.Optgroup(label="%s ------" % TName) self.cursor.execute( 'SELECT ProbeSetFreeze.FullName,ProbeSetFreeze.Name FROM ProbeSetFreeze, ProbeFreeze, \ InbredSet WHERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id and ProbeFreeze.TissueId = %d and \ ProbeSetFreeze.public > %d and ProbeFreeze.InbredSetId = InbredSet.Id and InbredSet.Name like "%s%%" \ order by ProbeSetFreeze.CreateTime desc, ProbeSetFreeze.AvgId ' % (TId, webqtlConfig.PUBLICTHRESH, RISet) ) for item2 in self.cursor.fetchall(): databaseMenuSub.append(item2) nmenu += 1 databaseMenu.append(databaseMenuSub) if nmenu: criteriaText = HT.Span("Return:", Class="ffl fwb fs12") criteriaMenu = HT.Select(name="criteria", selected="500") criteriaMenu.append(("top 100", "100")) criteriaMenu.append(("top 200", "200")) criteriaMenu.append(("top 500", "500")) criteriaMenu.append(("top 1000", "1000")) criteriaMenu.append(("top 2000", "2000")) criteriaMenu.append(("top 5000", "5000")) criteriaMenu.append(("top 10000", "10000")) criteriaMenu.append(("top 15000", "15000")) criteriaMenu.append(("top 20000", "20000")) self.MPDCell = HT.TD() correlationMenus = HT.TableLite( HT.TR(HT.TD(databaseText, HT.BR(), databaseMenu, colspan=4)), HT.TR(HT.TD(methodText, HT.BR(), methodMenu), self.MPDCell, HT.TD(criteriaText, HT.BR(), criteriaMenu)), border=0, cellspacing=4, cellpadding=0, ) else: correlationMenus = "" mainForm.append( HT.Font("or", color="red", size=4), HT.BR(), HT.BR(), "Compute partial correlation for each trait in the database selected below:", HT.BR(), ) mainForm.append(partialCorrDBButton, HT.BR(), HT.BR(), correlationMenus) TD_LR.append(mainForm) self.dict["body"] = str(TD_LR) self.dict["js1"] = "" self.dict["title"] = "Partial Correlation Input"
def __init__(self, fd): templatePage.__init__(self, fd) self.initializeDisplayParameters(fd) if not fd.genotype: fd.readGenotype() mdpchoice = None strainlist = fd.strainlist fd.readData() if not self.openMysql(): return isSampleCorr = 1 isTissueCorr = 0 TD_LR = HT.TD(colspan=2, height=200, width="100%", bgColor="#eeeeee", align="left", wrap="off") dataX = [] dataY = [] dataZ = [] # shortname fullTissueName = [] xlabel = "" ylabel = "" if isSampleCorr: plotHeading = HT.Paragraph("Sample Correlation Scatterplot") plotHeading.__setattr__("class", "title") # XZ: retrieve trait 1 info, Y axis trait1_data = [] # trait 1 data trait1Url = "" try: Trait1 = webqtlTrait(db=self.database, name=self.ProbeSetID, cellid=self.CellID, cursor=self.cursor) Trait1.retrieveInfo() Trait1.retrieveData() except: heading = "Retrieve Data" detail = ["The database you just requested has not been established yet."] self.error(heading=heading, detail=detail) return trait1_data = Trait1.exportData(strainlist) trait1Url = Trait1.genHTML(dispFromDatabase=1) ylabel = "%s : %s" % (Trait1.db.shortname, Trait1.name) if Trait1.cellid: ylabel += " : " + Trait1.cellid # XZ, retrieve trait 2 info, X axis trait2_data = [] # trait 2 data trait2Url = "" try: Trait2 = webqtlTrait(db=self.database2, name=self.ProbeSetID2, cellid=self.CellID2, cursor=self.cursor) Trait2.retrieveInfo() Trait2.retrieveData() except: heading = "Retrieve Data" detail = ["The database you just requested has not been established yet."] self.error(heading=heading, detail=detail) return trait2_data = Trait2.exportData(strainlist) trait2Url = Trait2.genHTML(dispFromDatabase=1) xlabel = "%s : %s" % (Trait2.db.shortname, Trait2.name) if Trait2.cellid: xlabel += " : " + Trait2.cellid for strain in Trait1.data.keys(): if Trait2.data.has_key(strain): dataX.append(Trait2.data[strain].val) dataY.append(Trait1.data[strain].val) if self.showstrains: dataZ.append(webqtlUtil.genShortStrainName(RISet=fd.RISet, input_strainName=strain)) # XZ: We have gotten all data for both traits. if len(dataX) >= self.corrMinInformative: if self.rankOrder == 0: rankPrimary = 0 rankSecondary = 1 else: rankPrimary = 1 rankSecondary = 0 lineColor = self.setLineColor() symbolColor = self.setSymbolColor() idColor = self.setIdColor() c = pid.PILCanvas(size=(self.plotSize, self.plotSize * 0.90)) data_coordinate = Plot.plotXY( canvas=c, dataX=dataX, dataY=dataY, rank=rankPrimary, dataLabel=dataZ, labelColor=pid.black, lineSize=self.lineSize, lineColor=lineColor, idColor=idColor, idFont=self.idFont, idSize=self.idSize, symbolColor=symbolColor, symbolType=self.symbol, filled=self.filled, symbolSize=self.symbolSize, XLabel=xlabel, connectdot=0, YLabel=ylabel, title="", fitcurve=self.showline, displayR=1, offset=(90, self.plotSize / 20, self.plotSize / 10, 90), showLabel=self.showIdentifiers, ) if rankPrimary == 1: dataXlabel, dataYlabel = webqtlUtil.calRank(xVals=dataX, yVals=dataY, N=len(dataX)) else: dataXlabel, dataYlabel = dataX, dataY gifmap1 = HT.Map(name="CorrelationPlotImageMap1") for i, item in enumerate(data_coordinate): one_rect_coordinate = "%d, %d, %d, %d" % (item[0] - 5, item[1] - 5, item[0] + 5, item[1] + 5) if isTissueCorr: one_rect_title = "%s (%s, %s)" % (fullTissueName[i], dataXlabel[i], dataYlabel[i]) else: one_rect_title = "%s (%s, %s)" % (dataZ[i], dataXlabel[i], dataYlabel[i]) gifmap1.areas.append(HT.Area(shape="rect", coords=one_rect_coordinate, title=one_rect_title)) filename = webqtlUtil.genRandStr("XY_") c.save(webqtlConfig.IMGDIR + filename, format="gif") img1 = HT.Image("/image/" + filename + ".gif", border=0, usemap="#CorrelationPlotImageMap1") mainForm_1 = HT.Form( cgi=os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype="multipart/form-data", name="showDatabase", submit=HT.Input(type="hidden"), ) hddn = { "FormID": "showDatabase", "ProbeSetID": "_", "database": "_", "CellID": "_", "RISet": fd.RISet, "ProbeSetID2": "_", "database2": "_", "CellID2": "_", "allstrainlist": string.join(fd.strainlist, " "), "traitList": fd.formdata.getvalue("traitList"), } if fd.incparentsf1: hddn["incparentsf1"] = "ON" for key in hddn.keys(): mainForm_1.append(HT.Input(name=key, value=hddn[key], type="hidden")) if isSampleCorr: mainForm_1.append( HT.P(), HT.Blockquote( HT.Strong("X axis:"), HT.Blockquote(trait2Url), HT.Strong("Y axis:"), HT.Blockquote(trait1Url), style="width: %spx;" % self.plotSize, wrap="hard", ), ) graphForm = HT.Form( cgi=os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), name="MDP_Form", submit=HT.Input(type="hidden"), ) graph_hddn = self.setHiddenParameters(fd, rankPrimary) webqtlUtil.exportData( graph_hddn, fd.allTraitData ) # XZ: This is necessary to replot with different groups of strains for key in graph_hddn.keys(): graphForm.append(HT.Input(name=key, value=graph_hddn[key], type="hidden")) options = self.createOptionsMenu(fd, mdpchoice) if self.showOptions == "0": showOptionsButton = HT.Input( type="button", name="optionsButton", value="Hide Options", onClick="showHideOptions();", Class="button", ) else: showOptionsButton = HT.Input( type="button", name="optionsButton", value="Show Options", onClick="showHideOptions();", Class="button", ) # updated by NL: 12-07-2011 add variables for tissue abbreviation page if isTissueCorr: graphForm.append(HT.Input(name="shortTissueName", value="", type="hidden")) graphForm.append(HT.Input(name="fullTissueName", value="", type="hidden")) shortTissueNameStr = string.join(dataZ, ",") fullTissueNameStr = string.join(fullTissueName, ",") tissueAbbrButton = HT.Input( type="button", name="tissueAbbrButton", value="Show Abbreviations", onClick="showTissueAbbr('MDP_Form','%s','%s')" % (shortTissueNameStr, fullTissueNameStr), Class="button", ) graphForm.append(showOptionsButton, " ", tissueAbbrButton, HT.BR(), HT.BR()) else: graphForm.append(showOptionsButton, HT.BR(), HT.BR()) graphForm.append(options, HT.BR()) graphForm.append(HT.HR(), HT.BR(), HT.P()) TD_LR.append(plotHeading, HT.BR(), graphForm, HT.BR(), gifmap1, HT.P(), img1, HT.P(), mainForm_1) TD_LR.append(HT.BR(), HT.HR(color="grey", size=5, width="100%")) c = pid.PILCanvas(size=(self.plotSize, self.plotSize * 0.90)) data_coordinate = Plot.plotXY( canvas=c, dataX=dataX, dataY=dataY, rank=rankSecondary, dataLabel=dataZ, labelColor=pid.black, lineColor=lineColor, lineSize=self.lineSize, idColor=idColor, idFont=self.idFont, idSize=self.idSize, symbolColor=symbolColor, symbolType=self.symbol, filled=self.filled, symbolSize=self.symbolSize, XLabel=xlabel, connectdot=0, YLabel=ylabel, title="", fitcurve=self.showline, displayR=1, offset=(90, self.plotSize / 20, self.plotSize / 10, 90), showLabel=self.showIdentifiers, ) if rankSecondary == 1: dataXlabel, dataYlabel = webqtlUtil.calRank(xVals=dataX, yVals=dataY, N=len(dataX)) else: dataXlabel, dataYlabel = dataX, dataY gifmap2 = HT.Map(name="CorrelationPlotImageMap2") for i, item in enumerate(data_coordinate): one_rect_coordinate = "%d, %d, %d, %d" % (item[0] - 6, item[1] - 6, item[0] + 6, item[1] + 6) if isTissueCorr: one_rect_title = "%s (%s, %s)" % (fullTissueName[i], dataXlabel[i], dataYlabel[i]) else: one_rect_title = "%s (%s, %s)" % (dataZ[i], dataXlabel[i], dataYlabel[i]) gifmap2.areas.append(HT.Area(shape="rect", coords=one_rect_coordinate, title=one_rect_title)) filename = webqtlUtil.genRandStr("XY_") c.save(webqtlConfig.IMGDIR + filename, format="gif") img2 = HT.Image("/image/" + filename + ".gif", border=0, usemap="#CorrelationPlotImageMap2") mainForm_2 = HT.Form( cgi=os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype="multipart/form-data", name="showDatabase2", submit=HT.Input(type="hidden"), ) hddn = { "FormID": "showDatabase2", "ProbeSetID": "_", "database": "_", "CellID": "_", "RISet": fd.RISet, "ProbeSetID2": "_", "database2": "_", "CellID2": "_", "allstrainlist": string.join(fd.strainlist, " "), "traitList": fd.formdata.getvalue("traitList"), } if fd.incparentsf1: hddn["incparentsf1"] = "ON" for key in hddn.keys(): mainForm_2.append(HT.Input(name=key, value=hddn[key], type="hidden")) if isSampleCorr: mainForm_2.append( HT.P(), HT.Blockquote( HT.Strong("X axis:"), HT.Blockquote(trait2Url), HT.Strong("Y axis:"), HT.Blockquote(trait1Url), style="width:%spx;" % self.plotSize, ), ) TD_LR.append(HT.BR(), HT.P()) TD_LR.append("\n", gifmap2, HT.P(), HT.P(), img2, HT.P(), mainForm_2) self.dict["body"] = str(TD_LR) else: heading = "Correlation Plot" detail = [ "Fewer than %d strain data were entered for %s data set. No statitical analysis has been attempted." % (self.corrMinInformative, fd.RISet) ] self.error(heading=heading, detail=detail) return
def displayMultiSymbolsResultPage(self, geneSymbolLst=None, symbolCount=None, tProbeSetFreezeId=None,TissueCorrMatrixObject=None,note=None,TissueCount =None): formName = webqtlUtil.genRandStr("fm_") form = HT.Form(cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data',name= formName, submit=HT.Input(type='hidden')) # the following hidden elements are required parameter in Class(PlotCorrelationPage). So we need to define them here. form.append(HT.Input(type="hidden", name="action", value="disp")) form.append(HT.Input(type="hidden", name="FormID", value="dispMultiTissueCorrelation")) form.append(HT.Input(type="hidden", name="X_geneSymbol", value="")) form.append(HT.Input(type="hidden", name="Y_geneSymbol", value="")) form.append(HT.Input(type="hidden", name="ProbeSetID", value="")) # RISet is not using in Tissue correlation, but is a required parameter in Class(PlotCorrelationPage). So we set dummy value(BXD). form.append(HT.Input(type="hidden", name="RISet", value="BXD")) form.append(HT.Input(type="hidden", name="ShowLine", value="1")) form.append(HT.Input(type="hidden", name="TissueProbeSetFreezeId", value=tProbeSetFreezeId)) form.append(HT.Input(type="hidden", name="rankOrder", value=0)) # updated by NL, 2011-01-06, build multi list for later use to descrease access to db again symbolList,geneIdDict,dataIdDict,ChrDict,MbDict,descDict,pTargetDescDict = TissueCorrMatrixObject.getTissueProbeSetXRefInfo(GeneNameLst=geneSymbolLst) # In case, upper case and lower case issue of symbol, mappedByTargetList function will update input geneSymbolLst based on database search result geneSymbolLst =self.mappedByTargetList(primaryList=symbolList,targetList=geneSymbolLst) # Added by NL, 2011-01-06, get all shortNames, verboseNames, verboseNames2, verboseNames3, exportArray # for Short Label, Long Label, Export functions geneIdLst,shortNames, verboseNames, verboseNames2, verboseNames3, exportArray = self.getAllLabelsInfo(geneSymbolList =geneSymbolLst, geneIdDict=geneIdDict,ChrDict=ChrDict, MbDict=MbDict, descDict=descDict, pTargetDescDict=pTargetDescDict) heading = "Tissue Correlation Matrix" #get correlation value and p value based on Gene Symbols list, and return the values in corrArray and pvArray seperately corrArray,pvArray = TissueCorrMatrixObject.getTissueCorrPvArray(geneNameLst=geneSymbolLst,dataIdDict=dataIdDict) # in the matrix table, top right corner displays Spearman Rank Correlation's Values and P-Values for each pair of geneSymbols; # left bottom displays Pearson Correlation values and P-Vlues for each pair of geneSymbols. tissueCorrMatrixHeading = HT.Paragraph(heading,Class="title") tcmTable = HT.TableLite(Class="collap", border=0, cellspacing=1, cellpadding=5, width='100%') row1 = HT.TR(HT.TD(Class="fs14 fwb ffl b1 cw cbrb"),HT.TD('Spearman Rank Correlation (rho)' , Class="fs14 fwb ffl b1 cw cbrb", colspan= symbolCount+2,align="center")) col1 = HT.TR(HT.TD("P e a r s o n r", rowspan= symbolCount+1,Class="fs14 fwb ffl b1 cw cbrb", width=10,align="center"),HT.TD("Gene Symbol",Class="fs13 fwb cb b1", width=300)) for i in range(symbolCount): GeneSymbol=geneSymbolLst[i].strip() geneId = geneIdLst[i] if geneId!=0: _url="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=%s" % geneId curURL = HT.Href(text=GeneSymbol,url=_url,Class='fs13',target="_blank") else: curURL = GeneSymbol col1.append(HT.TD(curURL,Class="b1", align="center")) tcmTable.append(row1,col1) # to decide to whether to show note for "*" or not flag = 0 for i in range(symbolCount): GeneSymbol=geneSymbolLst[i].strip() geneId = geneIdLst[i] newrow = HT.TR() newrow.append(HT.Input(name="Symbol", value=GeneSymbol, type='hidden')) if geneId!=0: _url="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=%s" %geneId geneIdURL = HT.Href(text="%s "%GeneSymbol,url=_url,Class="b1",target="_blank") else: # flag =1 will show note for "*" flag = 1 geneIdURL =HT.Italic("%s"%GeneSymbol,HT.Font('*', color='red')) newrow.append(HT.TD(geneIdURL,shortNames[i],verboseNames[i],verboseNames2[i],verboseNames3[i], Class="b1", align="left",NOWRAP="ON")) for j in range(symbolCount): GeneSymbol2=geneSymbolLst[j].strip() corr = corrArray[i][j] pValue = pvArray[i][j] Color='' if j==i: newrow.append(HT.TD(HT.Font(HT.Italic("n"),HT.BR(),str(TissueCount),Class="fs11 fwn b1",align="center", color="000000"), bgColor='#cccccc', align="center", Class="b1", NOWRAP="ON")) exportArray[i+1][j+1] = '%d/%d' % (TissueCount,TissueCount) else: if corr: corr = float(corr) tCorr = "%2.3f" % corr pValue = float(pValue) tPV = "%2.3f" % pValue # updated by NL, based on Rob's requirement: delete p value, 2010-02-14 # set color for cells by correlationValue if corr > 0.7: fontcolor="red" elif corr > 0.5: fontcolor="#FF6600" elif corr < -0.7: fontcolor="blue" elif corr < -0.5: fontcolor="#009900" else: fontcolor ="#000000" # set label for cells # if rank is equal to 0, pearson correlation plot will be the first one; # if rank is equal to 1, spearman ran correlation plot will be the first one. if j>i: exportArray[i+1][j+1] =tCorr+"/"+tPV rank =1 elif j<i: exportArray[i+1][j+1] =tCorr+"/"+tPV rank =0 tCorrStr= tCorr tPVStr = tPV tCorrPlotURL = "javascript:showTissueCorrPlot('%s','%s','%s',%d)" %(formName,GeneSymbol, GeneSymbol2,rank) corrURL= HT.Href(text=HT.Font(tCorrStr,HT.BR(),color=fontcolor, Class="fs11 fwn"), url = tCorrPlotURL) else: corr = 'N/A' corrURL= HT.Font(corr) exportArray[i+1][j+1] ="---/---" newrow.append(HT.TD(corrURL,bgColor=Color,Class="b1",NOWRAP="ON",align="middle")) tcmTable.append(newrow) Intro = HT.Blockquote('Lower left cells list Pearson ',HT.EM('r'),' values; upper right cells list Spearman rho values. Each cell also contains the n samples of tissues and organs. Values higher than 0.7 are displayed in ',HT.Font('red', color='red'),'; those between 0.5 and 0.7 in ',HT.Font('orange', color='#FF6600'),'; Values lower than -0.7 are in ',HT.Font('blue', color='blue'),'; between -0.5 and -0.7 in ',HT.Font('green', color='#009900'),'.', HT.BR(),HT.BR(), HT.Strong('Make scatter plots by clicking on cell values '),'(', HT.EM('r'),' or rho). ', Class="fs13 fwn") shortButton = HT.Input(type='button' ,name='dispShort',value=' Short Labels ', onClick="displayTissueShortName();",Class="button") verboseButton = HT.Input(type='button' ,name='dispVerbose',value=' Long Labels ', onClick="displayTissueVerboseName();", Class="button") exportbutton = HT.Input(type='button', name='export', value='Export', onClick="exportTissueText(allCorrelations);",Class="button") lableNote = HT.Blockquote(HT.Italic(HT.Font('*', color='red',Class="fs9 fwn"), ' Symbol(s) can not be found in database.')) # flag =1 will show note for "*", which means there's unidentified symbol. if flag==1: form.append(HT.Blockquote(tcmTable,lableNote,HT.P(),shortButton,verboseButton,exportbutton)) else: form.append(HT.Blockquote(tcmTable,HT.P(),shortButton,verboseButton,exportbutton)) exportScript = """ <SCRIPT language=JavaScript> var allCorrelations = %s; </SCRIPT> """ exportScript = exportScript % str(exportArray) self.dict['js1'] = exportScript+'<SCRIPT SRC="/javascript/correlationMatrix.js"></SCRIPT><BR>' TD_LR = HT.TD(colspan=2,width="100%",bgcolor="#eeeeee") TD_LR.append(tissueCorrMatrixHeading,note,Intro,form,HT.P()) self.dict['body'] = str(TD_LR) self.dict['title'] = 'Tissue Correlation Result' return
def displaySingleSymbolResultPage(self,primaryGeneSymbol=None, datasetFullName=None,tProbeSetFreezeId=None, TissueCorrMatrixObject =None,recordReturnNum=None,method=None,note=None,TissueCount =None): formName = webqtlUtil.genRandStr("fm_") form = HT.Form(cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data',name= formName, submit=HT.Input(type='hidden')) # the following hidden elements are required parameter in Class(PlotCorrelationPage). So we need to define them here. form.append(HT.Input(type="hidden", name="action", value="disp")) form.append(HT.Input(type="hidden", name="FormID", value="dispSingleTissueCorrelation")) form.append(HT.Input(type="hidden", name="X_geneSymbol", value="")) form.append(HT.Input(type="hidden", name="Y_geneSymbol", value="")) form.append(HT.Input(type="hidden", name="ProbeSetID", value="")) # RISet is not using in Tissue correlation, but is a required parameter in Class(PlotCorrelationPage). So we set dummy value(BXD). form.append(HT.Input(type="hidden", name="RISet", value="BXD")) form.append(HT.Input(type="hidden", name="ShowLine", value="1")) form.append(HT.Input(type="hidden", name="TissueProbeSetFreezeId", value=tProbeSetFreezeId)) form.append(HT.Input(type="hidden", name="rankOrder", value=0)) traitList =[] try: symbolCorrDict, symbolPvalueDict = TissueCorrMatrixObject.calculateCorrOfAllTissueTrait(primaryTraitSymbol=primaryGeneSymbol,method=method) except: heading = "Tissue Correlation" detail = ['Please use the official NCBI gene symbol.' ] self.error(heading=heading,detail=detail) return symbolList0,geneIdDict,dataIdDict,ChrDict,MbDict,descDict,pTargetDescDict=TissueCorrMatrixObject.getTissueProbeSetXRefInfo(GeneNameLst=[]) # In case, upper case and lower case issue of symbol, mappedByTargetList function will update input geneSymbolLst based on database search result tempPrimaryGeneSymbol =self.mappedByTargetList(primaryList=symbolList0,targetList=[primaryGeneSymbol]) primaryGeneSymbol =tempPrimaryGeneSymbol[0] returnNum = self.getReturnNum(recordReturnNum) symbolListSorted=[] symbolList=[] # get key(list) of symbolCorrDict(dict) based on sorting symbolCorrDict(dict) by its' value in desc order symbolListSorted=sorted(symbolCorrDict, key=symbolCorrDict.get, reverse=True) symbolList = self.mappedByTargetList(primaryList=symbolList0,targetList=symbolListSorted) if returnNum==None: returnNum =len(symbolList0) IntroReturnNum ="All %d "%returnNum else: IntroReturnNum ="The Top %d" %returnNum symbolList = symbolList[:returnNum] pageTable = HT.TableLite(cellSpacing=0,cellPadding=0,width="100%", border=0, align="Left") ############## # Excel file # ############## filename= webqtlUtil.genRandStr("Corr_") xlsUrl = HT.Input(type='button', value = 'Download Table', onClick= "location.href='/tmp/%s.xls'" % filename, Class='button') # Create a new Excel workbook workbook = xl.Writer('%s.xls' % (webqtlConfig.TMPDIR+filename)) headingStyle = workbook.add_format(align = 'center', bold = 1, border = 1, size=13, fg_color = 0x1E, color="white") #There are 6 lines of header in this file. worksheet = self.createExcelFileWithTitleAndFooter(workbook=workbook, datasetName=datasetFullName, returnNumber=returnNum) newrow = 6 pageTable.append(HT.TR(HT.TD(xlsUrl,height=40))) # get header part of result table and export excel file tblobj = {} tblobj['header'], worksheet = self.getTableHeader( method=method, worksheet=worksheet, newrow=newrow, headingStyle=headingStyle) newrow += 1 # get body part of result table and export excel file tblobj['body'], worksheet = self.getTableBody(symbolCorrDict=symbolCorrDict, symbolPvalueDict=symbolPvalueDict,symbolList=symbolList,geneIdDict=geneIdDict,ChrDict=ChrDict,MbDict=MbDict,descDict=descDict,pTargetDescDict=pTargetDescDict,primarySymbol=primaryGeneSymbol,TissueCount=TissueCount, formName=formName, worksheet=worksheet, newrow=newrow,method=method) workbook.close() # creat object for result table for sort function objfile = open('%s.obj' % (webqtlConfig.TMPDIR+filename), 'wb') cPickle.dump(tblobj, objfile) objfile.close() sortby = ("tissuecorr", "down") div = HT.Div(webqtlUtil.genTableObj(tblobj=tblobj, file=filename, sortby=sortby, tableID = "sortable", addIndex = "1"), Id="sortable") if method =="0": IntroMethod="Pearson\'s r " else: IntroMethod="Spearman\'s rho " Intro = HT.Blockquote('%s correlations ranked by the %s are displayed.' % (IntroReturnNum,IntroMethod), ' You can resort this list using the small arrowheads in the top row.') Intro.append(HT.BR(),' Click the correlation values to generate scatter plots. Select the symbol to open NCBI Entrez.') pageTable.append(HT.TR(HT.TD(div))) form.append(HT.P(), HT.P(),pageTable) corrHeading = HT.Paragraph('Tissue Correlation Table', Class="title") TD_LR = HT.TD(height=200,width="100%",bgcolor='#eeeeee',align="left") TD_LR.append(corrHeading,note,Intro, form, HT.P()) self.dict['body'] = str(TD_LR) self.dict['js1'] = '<SCRIPT SRC="/javascript/correlationMatrix.js"></SCRIPT><BR>' self.dict['title'] = 'Tissue Correlation Result' return
def marker_regression_page(): initial_start_vars = request.form logger.debug("Marker regression called with initial_start_vars:", initial_start_vars.items()) logger.error(request.url) temp_uuid = initial_start_vars['temp_uuid'] wanted = ( 'trait_id', 'dataset', 'method', 'mapping_results_path', 'trimmed_markers', 'selected_chr', 'chromosomes', 'mapping_scale', 'plotScale', 'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'num_bootstrap', 'bootCheck', 'bootstrap_results', 'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'control_marker', 'control_marker_db', 'do_control', 'genofile', 'pair_scan', 'startMb', 'endMb', 'graphWidth', 'lrsMax', 'additiveCheck', 'showSNP', 'showGenes', 'viewLegend', 'haplotypeAnalystCheck', 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno' ) start_vars = {} for key, value in initial_start_vars.iteritems(): if key in wanted or key.startswith(('value:')): start_vars[key] = value logger.debug("Marker regression called with start_vars:", start_vars) version = "v3" key = "marker_regression:{}:".format(version) + json.dumps(start_vars, sort_keys=True) logger.info("key is:", pf(key)) with Bench("Loading cache"): result = None # Just for testing #result = Redis.get(key) #logger.info("************************ Starting result *****************") #logger.info("result is [{}]: {}".format(type(result), result)) #logger.info("************************ Ending result ********************") if result: logger.info("Cache hit!!!") with Bench("Loading results"): result = pickle.loads(result) else: logger.info("Cache miss!!!") with Bench("Total time in MarkerRegression"): template_vars = marker_regression.MarkerRegression(start_vars, temp_uuid) if template_vars.mapping_method != "gemma" and template_vars.mapping_method != "plink": template_vars.js_data = json.dumps(template_vars.js_data, default=json_default_handler, indent=" ") json_filename = webqtlUtil.genRandStr("") + ".json" with open(GENERATED_TEXT_DIR + "/" + json_filename, "wb") as json_file: json_file.write(template_vars.js_data) csv_filename = webqtlUtil.genRandStr("") + ".csv" with open(GENERATED_TEXT_DIR + "/" + csv_filename, "wb") as csv_file: writer = csv.writer(csv_file) writer.writerow(("Locus", "Chr", "Mb", "LOD")) for (row) in qtl_results: score = row["lod_score"] if "lod_score" in row else row["lrs_value"] writer.writerow((row["name"], row["chr"], row["Mb"], score)) result = template_vars.__dict__ if result['pair_scan']: with Bench("Rendering template"): img_path = result['pair_scan_filename'] logger.info("img_path:", img_path) initial_start_vars = request.form logger.info("initial_start_vars:", initial_start_vars) imgfile = open(TEMPDIR + img_path, 'rb') imgdata = imgfile.read() imgB64 = imgdata.encode("base64") bytesarray = array.array('B', imgB64) result['pair_scan_array'] = bytesarray rendered_template = render_template("pair_scan_results.html", **result) else: #for item in template_vars.__dict__.keys(): # logger.info(" ---**--- {}: {}".format(type(template_vars.__dict__[item]), item)) gn1_template_vars = marker_regression_gn1.MarkerRegression(result).__dict__ #pickled_result = pickle.dumps(result, pickle.HIGHEST_PROTOCOL) #logger.info("pickled result length:", len(pickled_result)) #Redis.set(key, pickled_result) #Redis.expire(key, 1*60) with Bench("Rendering template"): if (gn1_template_vars['mapping_method'] == "gemma") or (gn1_template_vars['mapping_method'] == "plink"): gn1_template_vars.pop('qtlresults', None) print("TEMPLATE KEYS:", list(gn1_template_vars.keys())) rendered_template = render_template("marker_regression_gn1.html", **gn1_template_vars) return rendered_template
def buildCanvas(self, colorScheme='', targetDescriptionChecked='', clusterChecked='', sessionfile='', genotype=None, strainlist=None, ppolar=None, mpolar=None, traitList=None, traitDataList=None, userPrivilege=None, userName=None): labelFont = pid.Font(ttf="tahoma",size=14,bold=0) topHeight = 0 NNN = len(traitList) #XZ: It's necessory to define canvas here canvas = pid.PILCanvas(size=(80+NNN*20,880)) names = map(webqtlTrait.displayName, traitList) #XZ, 7/29/2009: create trait display and find max strWidth strWidth = 0 for j in range(len(names)): thisTrait = traitList[j] if targetDescriptionChecked: if thisTrait.db.type == 'ProbeSet': if thisTrait.probe_target_description: names[j] += ' [%s at Chr %s @ %2.3fMB, %s]' % (thisTrait.symbol, thisTrait.chr, thisTrait.mb, thisTrait.probe_target_description) else: names[j] += ' [%s at Chr %s @ %2.3fMB]' % (thisTrait.symbol, thisTrait.chr, thisTrait.mb) elif thisTrait.db.type == 'Geno': names[j] += ' [Chr %s @ %2.3fMB]' % (thisTrait.chr, thisTrait.mb) elif thisTrait.db.type == 'Publish': if thisTrait.confidential: if webqtlUtil.hasAccessToConfidentialPhenotypeTrait(privilege=userPrivilege, userName=userName, authorized_users=thisTrait.authorized_users): if thisTrait.post_publication_abbreviation: names[j] += ' [%s]' % (thisTrait.post_publication_abbreviation) else: if thisTrait.pre_publication_abbreviation: names[j] += ' [%s]' % (thisTrait.pre_publication_abbreviation) else: if thisTrait.post_publication_abbreviation: names[j] += ' [%s]' % (thisTrait.post_publication_abbreviation) else: pass i = canvas.stringWidth(names[j], font=labelFont) if i > strWidth: strWidth = i width = NNN*20 xoffset = 40 yoffset = 40 cellHeight = 3 nLoci = reduce(lambda x,y: x+y, map(lambda x: len(x),genotype),0) if nLoci > 2000: cellHeight = 1 elif nLoci > 1000: cellHeight = 2 elif nLoci < 200: cellHeight = 10 else: pass pos = range(NNN) neworder = [] BWs = Plot.BWSpectrum() colors100 = Plot.colorSpectrum() colors = Plot.colorSpectrum(130) finecolors = Plot.colorSpectrum(250) colors100.reverse() colors.reverse() finecolors.reverse() scaleFont=pid.Font(ttf="tahoma",size=10,bold=0) if not clusterChecked: #XZ: this part is for original order for i in range(len(names)): neworder.append((xoffset+20*(i+1), i)) canvas = pid.PILCanvas(size=(80+NNN*20+240,80+ topHeight +5+5+strWidth+nLoci*cellHeight+80+20*cellHeight)) self.drawTraitNameBottom(canvas,names,yoffset,neworder,strWidth,topHeight,labelFont) else: #XZ: this part is to cluster traits topHeight = 400 canvas = pid.PILCanvas(size=(80+NNN*20+240,80+ topHeight +5+5+strWidth+nLoci*cellHeight+80+20*cellHeight)) corArray = [([0] * (NNN))[:] for i in range(NNN)] nnCorr = len(strainlist) #XZ, 08/04/2009: I commented out pearsonArray, spearmanArray for i, thisTrait in enumerate(traitList): names1 = [thisTrait.db.name, thisTrait.name, thisTrait.cellid] for j, thisTrait2 in enumerate(traitList): names2 = [thisTrait2.db.name, thisTrait2.name, thisTrait2.cellid] if j < i: corr,nOverlap = webqtlUtil.calCorrelation(traitDataList[i], traitDataList[j],nnCorr) if (1-corr) < 0: distance = 0.0 else: distance = 1-corr corArray[i][j] = distance corArray[j][i] = distance elif j == i: corArray[i][j] = 0.0 else: pass #XZ, 7/29/2009: The parameter d has info of cluster (group member and distance). The format of d is tricky. Print it out to see it's format. d = slink.slink(corArray) #XZ, 7/29/2009: Attention: The 'neworder' is changed by the 'draw' function #XZ, 7/30/2009: Only toppos[1][0] and top[1][1] are used later. Then what toppos[0] is used for? toppos = self.draw(canvas,names,d,xoffset,yoffset,neworder,topHeight) self.drawTraitNameTop(canvas,names,yoffset,neworder,strWidth,topHeight,labelFont) #XZ, 7/29/2009: draw the top vertical line canvas.drawLine(toppos[1][0],toppos[1][1],toppos[1][0],yoffset) #XZ: draw string 'distance = 1-r' canvas.drawString('distance = 1-r',neworder[-1][0] + 50, topHeight*3/4,font=labelFont,angle=90) #draw Scale scaleFont=pid.Font(ttf="tahoma",size=10,bold=0) x = neworder[-1][0] canvas.drawLine(x+5, topHeight+yoffset, x+5, yoffset, color=pid.black) y = 0 while y <=2: canvas.drawLine(x+5, topHeight*y/2.0+yoffset, x+10, topHeight*y/2.0+yoffset) canvas.drawString('%2.1f' % (2-y), x+12, topHeight*y/2.0+yoffset, font=scaleFont) y += 0.5 chrname = 0 chrnameFont=pid.Font(ttf="tahoma",size=24,bold=0) Ncol = 0 nearestMarkers = self.getNearestMarker(traitList, genotype) # import cPickle if sessionfile: fp = open(os.path.join(webqtlConfig.TMPDIR, sessionfile + '.session'), 'rb') permData = cPickle.load(fp) fp.close() else: permData = {} areas = [] #XZ, 7/31/2009: This for loop is to generate the heatmap #XZ: draw trait by trait instead of marker by marker for order in neworder: #startHeight = 40+400+5+5+strWidth startHeight = topHeight + 40+5+5+strWidth startWidth = order[0]-5 if Ncol and Ncol % 5 == 0: drawStartPixel = 8 else: drawStartPixel = 9 tempVal = traitDataList[order[1]] _vals = [] _strains = [] for i in range(len(strainlist)): if tempVal[i] != None: _strains.append(strainlist[i]) _vals.append(tempVal[i]) qtlresult = genotype.regression(strains = _strains, trait = _vals) if sessionfile: LRSArray = permData[str(traitList[order[1]])] else: LRSArray = genotype.permutation(strains = _strains, trait = _vals, nperm = 1000) permData[str(traitList[order[1]])] = LRSArray sugLRS = LRSArray[369] sigLRS = LRSArray[949] prechr = 0 chrstart = 0 nearest = nearestMarkers[order[1]] midpoint = [] for item in qtlresult: if item.lrs > webqtlConfig.MAXLRS: adjustlrs = webqtlConfig.MAXLRS else: adjustlrs = item.lrs if item.locus.chr != prechr: if prechr: canvas.drawRect(startWidth-drawStartPixel, startHeight, startWidth+10, startHeight+3,edgeColor=pid.white, edgeWidth=0, fillColor=pid.white) startHeight+= 3 if not chrname: canvas.drawString(prechr,xoffset-20,(chrstart+startHeight)/2,font = chrnameFont,color=pid.dimgray) prechr = item.locus.chr chrstart = startHeight if colorScheme == '0': if adjustlrs <= sugLRS: colorIndex = int(65*adjustlrs/sugLRS) else: colorIndex = int(65 + 35*(adjustlrs-sugLRS)/(sigLRS-sugLRS)) if colorIndex > 99: colorIndex = 99 colorIndex = colors100[colorIndex] elif colorScheme == '1': sugLRS = LRSArray[369]/2.0 if adjustlrs <= sugLRS: colorIndex = BWs[20+int(50*adjustlrs/sugLRS)] else: if item.additive > 0: colorIndex = int(80 + 50*(adjustlrs-sugLRS)/(sigLRS-sugLRS)) else: colorIndex = int(50 - 50*(adjustlrs-sugLRS)/(sigLRS-sugLRS)) if colorIndex > 129: colorIndex = 129 if colorIndex < 0: colorIndex = 0 colorIndex = colors[colorIndex] elif colorScheme == '2': if item.additive > 0: colorIndex = int(150 + 100*(adjustlrs/sigLRS)) else: colorIndex = int(100 - 100*(adjustlrs/sigLRS)) if colorIndex > 249: colorIndex = 249 if colorIndex < 0: colorIndex = 0 colorIndex = finecolors[colorIndex] else: colorIndex = pid.white if startHeight > 1: canvas.drawRect(startWidth-drawStartPixel, startHeight, startWidth+10, startHeight+cellHeight,edgeColor=colorIndex, edgeWidth=0, fillColor=colorIndex) else: canvas.drawLine(startWidth-drawStartPixel, startHeight, startWidth+10, startHeight, Color=colorIndex) if item.locus.name == nearest: midpoint = [startWidth,startHeight-5] startHeight+=cellHeight #XZ, map link to trait name and band COORDS = "%d,%d,%d,%d" %(startWidth-drawStartPixel,topHeight+40,startWidth+10,startHeight) HREF = "javascript:showDatabase2('%s','%s','%s');" % (traitList[order[1]].db.name, traitList[order[1]].name, traitList[order[1]].cellid) area = (COORDS, HREF, '%s' % names[order[1]]) areas.append(area) if midpoint: traitPixel = ((midpoint[0],midpoint[1]),(midpoint[0]-6,midpoint[1]+12),(midpoint[0]+6,midpoint[1]+12)) canvas.drawPolygon(traitPixel,edgeColor=pid.black,fillColor=pid.orange,closed=1) if not chrname: canvas.drawString(prechr,xoffset-20,(chrstart+startHeight)/2,font = chrnameFont,color=pid.dimgray) chrname = 1 Ncol += 1 #draw Spectrum startSpect = neworder[-1][0] + 30 startHeight = topHeight + 40+5+5+strWidth if colorScheme == '0': for i in range(100): canvas.drawLine(startSpect+i,startHeight+20,startSpect+i,startHeight+40,color=colors100[i]) scaleFont=pid.Font(ttf="tahoma",size=10,bold=0) canvas.drawLine(startSpect,startHeight+45,startSpect,startHeight+39,color=pid.black) canvas.drawString('LRS = 0',startSpect,startHeight+55,font=scaleFont) canvas.drawLine(startSpect+64,startHeight+45,startSpect+64,startHeight+39,color=pid.black) canvas.drawString('Suggestive LRS',startSpect+64,startHeight+55,font=scaleFont) canvas.drawLine(startSpect+99,startHeight+45,startSpect+99,startHeight+39,color=pid.black) canvas.drawString('Significant LRS',startSpect+105,startHeight+40,font=scaleFont) elif colorScheme == '1': for i in range(50): canvas.drawLine(startSpect+i,startHeight,startSpect+i,startHeight+40,color=BWs[20+i]) for i in range(50,100): canvas.drawLine(startSpect+i,startHeight,startSpect+i,startHeight+20,color=colors[100-i]) canvas.drawLine(startSpect+i,startHeight+20,startSpect+i,startHeight+40,color=colors[30+i]) canvas.drawLine(startSpect,startHeight+45,startSpect,startHeight+39,color=pid.black) canvas.drawString('LRS = 0',startSpect,startHeight+60,font=scaleFont) canvas.drawLine(startSpect+50,startHeight+45,startSpect+50,startHeight+39,color=pid.black) canvas.drawString('0.5*Suggestive LRS',startSpect+50,startHeight+ 60,font=scaleFont) canvas.drawLine(startSpect+99,startHeight+45,startSpect+99,startHeight+39,color=pid.black) canvas.drawString('Significant LRS',startSpect+105,startHeight+50,font=scaleFont) textFont=pid.Font(ttf="verdana",size=18,bold=0) canvas.drawString('%s +' % ppolar,startSpect+120,startHeight+ 35,font=textFont,color=pid.red) canvas.drawString('%s +' % mpolar,startSpect+120,startHeight+ 15,font=textFont,color=pid.blue) elif colorScheme == '2': for i in range(100): canvas.drawLine(startSpect+i,startHeight,startSpect+i,startHeight+20,color=finecolors[100-i]) canvas.drawLine(startSpect+i,startHeight+20,startSpect+i,startHeight+40,color=finecolors[150+i]) canvas.drawLine(startSpect,startHeight+45,startSpect,startHeight+39,color=pid.black) canvas.drawString('LRS = 0',startSpect,startHeight+60,font=scaleFont) canvas.drawLine(startSpect+99,startHeight+45,startSpect+99,startHeight+39,color=pid.black) canvas.drawString('Significant LRS',startSpect+105,startHeight+50,font=scaleFont) textFont=pid.Font(ttf="verdana",size=18,bold=0) canvas.drawString('%s +' % ppolar,startSpect+120,startHeight+ 35,font=textFont,color=pid.red) canvas.drawString('%s +' % mpolar,startSpect+120,startHeight+ 15,font=textFont,color=pid.blue) filename= webqtlUtil.genRandStr("Heatmap_") canvas.save(webqtlConfig.IMGDIR+filename, format='png') if not sessionfile: sessionfile = webqtlUtil.generate_session() webqtlUtil.dump_session(permData, os.path.join(webqtlConfig.TMPDIR, sessionfile +'.session')) self.filename=filename self.areas=areas self.sessionfile=sessionfile
def __init__(self, fd): templatePage.__init__(self, fd) if not self.openMysql(): return if not fd.genotype: fd.readData(incf1=1) self.strainlist = [] self.vals = [] for i, strain in enumerate(fd.f1list + fd.strainlist): if fd.allTraitData.has_key(strain) and fd.allTraitData[strain].val != None: self.strainlist.append(strain) self.vals.append([fd.allTraitData[strain].val, fd.allTraitData[strain].var]) if len(self.strainlist) > webqtlConfig.KMININFORMATIVE: pass else: templatePage.__init__(self, fd) heading = "Add to Collection" detail = [ "The number of informative strains in your trait is less than %d, this trait can not be added to the selection" % webqtlConfig.KMININFORMATIVE ] self.error(heading=heading, detail=detail) return self.cursor.execute( "delete Temp, TempData from Temp, TempData where Temp.DataId = TempData.Id and UNIX_TIMESTAMP()-UNIX_TIMESTAMP(CreateTime)>%d;" % webqtlConfig.MAXLIFE ) ct0 = time.localtime(time.time()) ct = time.strftime("%B/%d %H:%M:%S", ct0) if not fd.identification: fd.identification = "Unnamed Trait" user_ip = fd.remote_ip newDescription = "%s entered at %s from IP %s" % (fd.identification, ct, user_ip) newProbeSetID = webqtlUtil.genRandStr("USER_Tmp_") self.cursor.execute("SelecT max(id) from TempData") try: DataId = self.cursor.fetchall()[0][0] + 1 except: DataId = 1 self.cursor.execute('SelecT Id from InbredSet where Name = "%s"' % fd.RISet) InbredSetId = self.cursor.fetchall()[0][0] self.cursor.execute( "insert into Temp(Name,description, createtime,DataId,InbredSetId,IP) values(%s,%s,Now(),%s,%s,%s)", (newProbeSetID, newDescription, DataId, InbredSetId, user_ip), ) k = 0 for Strain in self.strainlist: self.cursor.execute( 'SelecT Strain.Id from Strain,StrainXRef where Strain.Name = "%s" and Strain.Id = StrainXRef.StrainId and StrainXRef.InbredSetId=%d' % (Strain, InbredSetId) ) StrainId = self.cursor.fetchall()[0][0] self.cursor.execute( "insert into TempData(Id, StrainId, value, SE) values(%s,%s,%s,%s)", (DataId, StrainId, self.vals[k][0], self.vals[k][1]), ) k += 1 self.searchResult = ["Temp::%s" % newProbeSetID] if self.genSelection(fd=fd): self.writeHTML(fd)
def run_analysis(self, requestform): print("Starting WGCNA analysis on dataset") self.r_enableWGCNAThreads() # Enable multi threading self.trait_db_list = [ trait.strip() for trait in requestform['trait_list'].split(',') ] print("Retrieved phenotype data from database", requestform['trait_list']) helper_functions.get_trait_db_obs(self, self.trait_db_list) self.input = { } # self.input contains the phenotype values we need to send to R strains = [] # All the strains we have data for (contains duplicates) traits = [ ] # All the traits we have data for (should not contain duplicates) for trait in self.trait_list: traits.append(trait[0].name) self.input[trait[0].name] = {} for strain in trait[0].data: strains.append(strain) self.input[trait[0].name][strain] = trait[0].data[strain].value # Transfer the load data from python to R uStrainsR = r_unique(ro.Vector(strains)) # Unique strains in R vector uTraitsR = r_unique(ro.Vector(traits)) # Unique traits in R vector r_cat("The number of unique strains:", r_length(uStrainsR), "\n") r_cat("The number of unique traits:", r_length(uTraitsR), "\n") # rM is the datamatrix holding all the data in R /rows = strains columns = traits rM = ro.r.matrix(ri.NA_Real, nrow=r_length(uStrainsR), ncol=r_length(uTraitsR), dimnames=r_list(uStrainsR, uTraitsR)) for t in uTraitsR: trait = t[0] # R uses vectors every single element is a vector for s in uStrainsR: strain = s[ 0] # R uses vectors every single element is a vector #DEBUG: print(trait, strain, " in python: ", self.input[trait].get(strain), "in R:", rM.rx(strain,trait)[0]) rM.rx[strain, trait] = self.input[trait].get( strain) # Update the matrix location sys.stdout.flush() self.results = {} self.results['nphe'] = r_length(uTraitsR)[ 0] # Number of phenotypes/traits self.results['nstr'] = r_length(uStrainsR)[0] # Number of strains self.results['phenotypes'] = uTraitsR # Traits used self.results['strains'] = uStrainsR # Strains used in the analysis self.results[ 'requestform'] = requestform # Store the user specified parameters for the output page # Calculate soft threshold if the user specified the SoftThreshold variable if requestform.get('SoftThresholds') is not None: powers = [ int(threshold.strip()) for threshold in requestform['SoftThresholds'].rstrip().split(",") ] rpow = r_unlist(r_c(powers)) print "SoftThresholds: {} == {}".format(powers, rpow) self.sft = self.r_pickSoftThreshold(rM, powerVector=rpow, verbose=5) print "PowerEstimate: {}".format(self.sft[0]) self.results['PowerEstimate'] = self.sft[0] if self.sft[0][0] is ri.NA_Integer: print "No power is suitable for the analysis, just use 1" self.results['Power'] = 1 # No power could be estimated else: self.results['Power'] = self.sft[0][ 0] # Use the estimated power else: # The user clicked a button, so no soft threshold selection self.results['Power'] = requestform.get( 'Power') # Use the power value the user gives # Create the block wise modules using WGCNA network = self.r_blockwiseModules( rM, power=self.results['Power'], TOMType=requestform['TOMtype'], minModuleSize=requestform['MinModuleSize'], verbose=3) # Save the network for the GUI self.results['network'] = network # How many modules and how many gene per module ? print "WGCNA found {} modules".format(r_table(network[1])) self.results['nmod'] = r_length(r_table(network[1]))[0] # The iconic WCGNA plot of the modules in the hanging tree self.results['imgurl'] = webqtlUtil.genRandStr("WGCNAoutput_") + ".png" self.results['imgloc'] = webqtlConfig.TMPDIR + self.results['imgurl'] r_png(self.results['imgloc'], width=1000, height=600) mergedColors = self.r_labels2colors(network[1]) self.r_plotDendroAndColors(network[5][0], mergedColors, "Module colors", dendroLabels=False, hang=0.03, addGuide=True, guideHang=0.05) r_dev_off() sys.stdout.flush()
def __init__(self, fd): templatePage.__init__(self, fd) filename = webqtlUtil.genRandStr("Export_") workbook = xl.Writer('%s.xls' % (webqtlConfig.TMPDIR+filename)) style_formats = [] #Array with Excel style formats - Zach 9/2/2011 heading = workbook.add_format(align = 'center', bold = 1, border = 1, size=13, fg_color = 0x1E, color="white") #Style for the header cells right = workbook.add_format(align = 'right') #Style to align cell contents to the right style_formats.append(heading) style_formats.append(right) worksheet = workbook.add_worksheet() primaryStrainNames = fd.formdata.getvalue('strainNames', '').split(',') primaryVals = fd.formdata.getvalue('strainVals', '').split(',') primaryVars = fd.formdata.getvalue('strainVars', '').split(',') otherStrainNames = fd.formdata.getvalue('otherStrainNames', '').split(',') otherVals = fd.formdata.getvalue('otherStrainVals', '').split(',') otherVars = fd.formdata.getvalue('otherStrainVars', '').split(',') attributeData = fd.formdata.getvalue('extra_attributes', '') otherAttributeData = fd.formdata.getvalue('other_extra_attributes', '') #ZS: This section is to parse the attribute formdata string attributeTypes = attributeData.split('/') otherAttributeTypes = otherAttributeData.split('/') attributeNames = [] attributeVals = [] for i in range(len(attributeTypes)): if i < len(attributeTypes) - 1: attributeNames.append(attributeTypes[i].split(':')[0]) attributeVals.append(attributeTypes[i].split(':')[1].split(',')) else: break otherAttributeNames = [] otherAttributeVals = [] for i in range(len(otherAttributeTypes)): if i < len(otherAttributeTypes) - 1: otherAttributeNames.append(otherAttributeTypes[i].split(':')[0]) otherAttributeVals.append(otherAttributeTypes[i].split(':')[1].split(',')) else: break varsExist = 0 #ZS: Even if there are no variances "primaryVars" would still be populated with empty values, so we need to check if there really are any for i in range(len(primaryVars)): if primaryVars[i] != '': varsExist = 1 break otherStrainsExist = 0 #ZS: Same as above; checking to see if there's a set of "other" (non-primary) strains for i in range(len(otherStrainNames)): if otherStrainNames[i] != '': otherStrainsExist = 1 break if varsExist == 1: column_headers = ["Sample", "Value", " SE "] #ZS: Names of the header for each column in the excel worksheet else: column_headers = ["Sample", "Value"] for attr_name in attributeNames: column_headers.append(attr_name) start_line = 0 #Gets last line of "primary" strain values to define a start-point for "other" strain values for ncol, item in enumerate(column_headers): worksheet.write([start_line, ncol], item, style_formats[0]) worksheet.set_column([ncol, ncol], 2*len(item)) start_line += 1 last_line = start_line for i in range(len(primaryStrainNames)): ncol = 0 if varsExist == 1: for ncol, item in enumerate([primaryStrainNames[i], primaryVals[i], primaryVars[i]]): worksheet.write([start_line + i, ncol], item, style_formats[1]) ncol += 1 else: for ncol, item in enumerate([primaryStrainNames[i], primaryVals[i]]): worksheet.write([start_line + i, ncol], item, style_formats[1]) ncol += 1 for attribute_type in attributeVals: worksheet.write([start_line + i, ncol], attribute_type[i], style_formats[1]) ncol += 1 last_line += 1 if otherStrainsExist == 1: start_line = last_line + 2 for ncol, item in enumerate(column_headers): worksheet.write([start_line, ncol], item, style_formats[0]) worksheet.set_column([ncol, ncol], 2*len(item)) start_line += 1 for i in range(len(otherStrainNames)): ncol = 0 if varsExist == 1: for ncol, item in enumerate([otherStrainNames[i], otherVals[i], otherVars[i]]): worksheet.write([start_line + i, ncol], item, style_formats[1]) ncol += 1 else: for ncol, item in enumerate([otherStrainNames[i], otherVals[i]]): worksheet.write([start_line + i, ncol], item, style_formats[1]) for attribute_type in otherAttributeVals: worksheet.write([start_line + i, ncol], attribute_type[i], style_formats[1]) ncol += 1 workbook.close() full_filename = os.path.join(webqtlConfig.TMPDIR, '%s.xls' % filename) fp = open(full_filename, 'rb') text = fp.read() fp.close() self.content_type = 'application/xls' self.content_disposition = 'attachment; filename=%s' % ('%s.xls' % filename) self.attachment = text
def __init__(self, fd): templatePage.__init__(self, fd) self.initializeDisplayParameters(fd) if not fd.genotype: fd.readGenotype() if fd.allstrainlist: mdpchoice = fd.formdata.getvalue('MDPChoice') if mdpchoice == "1": strainlist = fd.f1list + fd.strainlist elif mdpchoice == "2": strainlist = [] strainlist2 = fd.f1list + fd.strainlist for strain in fd.allstrainlist: if strain not in strainlist2: strainlist.append(strain) #So called MDP Panel if strainlist: strainlist = fd.f1list+fd.parlist+strainlist else: strainlist = fd.allstrainlist fd.readData(fd.allstrainlist) else: mdpchoice = None strainlist = fd.strainlist fd.readData() #if fd.allstrainlist: # fd.readData(fd.allstrainlist) # strainlist = fd.allstrainlist #else: # fd.readData() # strainlist = fd.strainlist if not self.openMysql(): return isSampleCorr = 0 #XZ: initial value is false isTissueCorr = 0 #XZ: initial value is false #Javascript functions (showCorrelationPlot2, showTissueCorrPlot) have made sure the correlation type is either sample correlation or tissue correlation. if (self.database and (self.ProbeSetID != 'none')): isSampleCorr = 1 elif (self.X_geneSymbol and self.Y_geneSymbol): isTissueCorr = 1 else: heading = "Correlation Type Error" detail = ["For the input parameters, GN can not recognize the correlation type is sample correlation or tissue correlation."] self.error(heading=heading,detail=detail) return TD_LR = HT.TD(colspan=2,height=200,width="100%",bgColor='#eeeeee', align="left", wrap="off") dataX=[] dataY=[] dataZ=[] # shortname fullTissueName=[] if isTissueCorr: dataX, dataY, xlabel, ylabel, dataZ, fullTissueName = self.getTissueLabelsValues(X_geneSymbol=self.X_geneSymbol, Y_geneSymbol=self.Y_geneSymbol, TissueProbeSetFreezeId=self.TissueProbeSetFreezeId) plotHeading = HT.Paragraph('Tissue Correlation Scatterplot') plotHeading.__setattr__("class","title") if self.xAxisLabel == '': self.xAxisLabel = xlabel if self.yAxisLabel == '': self.yAxisLabel = ylabel if isSampleCorr: plotHeading = HT.Paragraph('Sample Correlation Scatterplot') plotHeading.__setattr__("class","title") #XZ: retrieve trait 1 info, Y axis trait1_data = [] #trait 1 data trait1Url = '' try: Trait1 = webqtlTrait(db=self.database, name=self.ProbeSetID, cellid=self.CellID, cursor=self.cursor) Trait1.retrieveInfo() Trait1.retrieveData() except: heading = "Retrieve Data" detail = ["The database you just requested has not been established yet."] self.error(heading=heading,detail=detail) return trait1_data = Trait1.exportData(strainlist) if Trait1.db.type == 'Publish' and Trait1.confidential: trait1Url = Trait1.genHTML(dispFromDatabase=1, privilege=self.privilege, userName=self.userName, authorized_users=Trait1.authorized_users) else: trait1Url = Trait1.genHTML(dispFromDatabase=1) if self.yAxisLabel == '': self.yAxisLabel= '%s : %s' % (Trait1.db.shortname, Trait1.name) if Trait1.cellid: self.yAxisLabel += ' : ' + Trait1.cellid #XZ, retrieve trait 2 info, X axis traitdata2 = [] #trait 2 data _vals = [] #trait 2 data trait2Url = '' if ( self.database2 and (self.ProbeSetID2 != 'none') ): try: Trait2 = webqtlTrait(db=self.database2, name=self.ProbeSetID2, cellid=self.CellID2, cursor=self.cursor) Trait2.retrieveInfo() Trait2.retrieveData() except: heading = "Retrieve Data" detail = ["The database you just requested has not been established yet."] self.error(heading=heading,detail=detail) return if Trait2.db.type == 'Publish' and Trait2.confidential: trait2Url = Trait2.genHTML(dispFromDatabase=1, privilege=self.privilege, userName=self.userName, authorized_users=Trait2.authorized_users) else: trait2Url = Trait2.genHTML(dispFromDatabase=1) traitdata2 = Trait2.exportData(strainlist) _vals = traitdata2[:] if self.xAxisLabel == '': self.xAxisLabel = '%s : %s' % (Trait2.db.shortname, Trait2.name) if Trait2.cellid: self.xAxisLabel += ' : ' + Trait2.cellid else: for item in strainlist: if fd.allTraitData.has_key(item): _vals.append(fd.allTraitData[item].val) else: _vals.append(None) if self.xAxisLabel == '': if fd.identification: self.xAxisLabel = fd.identification else: self.xAxisLabel = "User Input Data" try: Trait2 = webqtlTrait(fullname=fd.formdata.getvalue('fullname'), cursor=self.cursor) trait2Url = Trait2.genHTML(dispFromDatabase=1) except: trait2Url = self.xAxisLabel if (_vals and trait1_data): if len(_vals) != len(trait1_data): errors = HT.Blockquote(HT.Font('Error: ',color='red'),HT.Font('The number of traits are inconsistent, Program quit',color='black')) errors.__setattr__("class","subtitle") TD_LR.append(errors) self.dict['body'] = str(TD_LR) return for i in range(len(_vals)): if _vals[i]!= None and trait1_data[i]!= None: dataX.append(_vals[i]) dataY.append(trait1_data[i]) strainName = strainlist[i] if self.showstrains: dataZ.append(webqtlUtil.genShortStrainName(RISet=fd.RISet, input_strainName=strainName)) else: heading = "Correlation Plot" detail = ['Empty Dataset for sample correlation, please check your data.'] self.error(heading=heading,detail=detail) return #XZ: We have gotten all data for both traits. if len(dataX) >= self.corrMinInformative: if self.rankOrder == 0: rankPrimary = 0 rankSecondary = 1 else: rankPrimary = 1 rankSecondary = 0 lineColor = self.setLineColor(); symbolColor = self.setSymbolColor(); idColor = self.setIdColor(); c = pid.PILCanvas(size=(self.plotSize, self.plotSize*0.90)) data_coordinate = Plot.plotXY(canvas=c, dataX=dataX, dataY=dataY, rank=rankPrimary, dataLabel = dataZ, labelColor=pid.black, lineSize=self.lineSize, lineColor=lineColor, idColor=idColor, idFont=self.idFont, idSize=self.idSize, symbolColor=symbolColor, symbolType=self.symbol, filled=self.filled, symbolSize=self.symbolSize, XLabel=self.xAxisLabel, connectdot=0, YLabel=self.yAxisLabel, title='', fitcurve=self.showline, displayR =1, offset= (90, self.plotSize/20, self.plotSize/10, 90), showLabel = self.showIdentifiers) if rankPrimary == 1: dataXlabel, dataYlabel = webqtlUtil.calRank(xVals=dataX, yVals=dataY, N=len(dataX)) else: dataXlabel, dataYlabel = dataX, dataY gifmap1 = HT.Map(name='CorrelationPlotImageMap1') for i, item in enumerate(data_coordinate): one_rect_coordinate = "%d, %d, %d, %d" % (item[0] - 5, item[1] - 5, item[0] + 5, item[1] + 5) if isTissueCorr: one_rect_title = "%s (%s, %s)" % (fullTissueName[i], dataXlabel[i], dataYlabel[i]) else: one_rect_title = "%s (%s, %s)" % (dataZ[i], dataXlabel[i], dataYlabel[i]) gifmap1.areas.append(HT.Area(shape='rect',coords=one_rect_coordinate, title=one_rect_title) ) filename= webqtlUtil.genRandStr("XY_") c.save(webqtlConfig.IMGDIR+filename, format='gif') img1=HT.Image('/image/'+filename+'.gif',border=0, usemap='#CorrelationPlotImageMap1') mainForm_1 = HT.Form( cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name='showDatabase', submit=HT.Input(type='hidden')) hddn = {'FormID':'showDatabase','ProbeSetID':'_','database':'_','CellID':'_','RISet':fd.RISet, 'ProbeSetID2':'_', 'database2':'_', 'CellID2':'_', 'allstrainlist':string.join(fd.strainlist, " "), 'traitList': fd.formdata.getvalue("traitList")} if fd.incparentsf1: hddn['incparentsf1'] = 'ON' for key in hddn.keys(): mainForm_1.append(HT.Input(name=key, value=hddn[key], type='hidden')) if isSampleCorr: mainForm_1.append(HT.P(), HT.Blockquote(HT.Strong('X axis:'),HT.Blockquote(trait2Url),HT.Strong('Y axis:'),HT.Blockquote(trait1Url), style='width: %spx;' % self.plotSize, wrap="hard")) graphForm = HT.Form(cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), name='MDP_Form',submit=HT.Input(type='hidden')) graph_hddn = self.setHiddenParameters(fd, rankPrimary) webqtlUtil.exportData(graph_hddn, fd.allTraitData) #XZ: This is necessary to replot with different groups of strains for key in graph_hddn.keys(): graphForm.append(HT.Input(name=key, value=graph_hddn[key], type='hidden')) options = self.createOptionsMenu(fd, mdpchoice) if (self.showOptions == '0'): showOptionsButton = HT.Input(type='button' ,name='optionsButton',value='Hide Options', onClick="showHideOptions();", Class="button") else: showOptionsButton = HT.Input(type='button' ,name='optionsButton',value='Show Options', onClick="showHideOptions();", Class="button") # updated by NL: 12-07-2011 add variables for tissue abbreviation page if isTissueCorr: graphForm.append(HT.Input(name='shortTissueName', value='', type='hidden')) graphForm.append(HT.Input(name='fullTissueName', value='', type='hidden')) shortTissueNameStr=string.join(dataZ, ",") fullTissueNameStr=string.join(fullTissueName, ",") tissueAbbrButton=HT.Input(type='button' ,name='tissueAbbrButton',value='Show Abbreviations', onClick="showTissueAbbr('MDP_Form','%s','%s')" % (shortTissueNameStr,fullTissueNameStr), Class="button") graphForm.append(showOptionsButton,' ',tissueAbbrButton, HT.BR(), HT.BR()) else: graphForm.append(showOptionsButton, HT.BR(), HT.BR()) graphForm.append(options, HT.BR()) graphForm.append(HT.HR(), HT.BR(), HT.P()) TD_LR.append(plotHeading, HT.BR(),graphForm, HT.BR(), gifmap1, HT.P(), img1, HT.P(), mainForm_1) TD_LR.append(HT.BR(), HT.HR(color="grey", size=5, width="100%")) c = pid.PILCanvas(size=(self.plotSize, self.plotSize*0.90)) data_coordinate = Plot.plotXY(canvas=c, dataX=dataX, dataY=dataY, rank=rankSecondary, dataLabel = dataZ, labelColor=pid.black,lineColor=lineColor, lineSize=self.lineSize, idColor=idColor, idFont=self.idFont, idSize=self.idSize, symbolColor=symbolColor, symbolType=self.symbol, filled=self.filled, symbolSize=self.symbolSize, XLabel=self.xAxisLabel, connectdot=0, YLabel=self.yAxisLabel,title='', fitcurve=self.showline, displayR =1, offset= (90, self.plotSize/20, self.plotSize/10, 90), showLabel = self.showIdentifiers) if rankSecondary == 1: dataXlabel, dataYlabel = webqtlUtil.calRank(xVals=dataX, yVals=dataY, N=len(dataX)) else: dataXlabel, dataYlabel = dataX, dataY gifmap2 = HT.Map(name='CorrelationPlotImageMap2') for i, item in enumerate(data_coordinate): one_rect_coordinate = "%d, %d, %d, %d" % (item[0] - 6, item[1] - 6, item[0] + 6, item[1] + 6) if isTissueCorr: one_rect_title = "%s (%s, %s)" % (fullTissueName[i], dataXlabel[i], dataYlabel[i]) else: one_rect_title = "%s (%s, %s)" % (dataZ[i], dataXlabel[i], dataYlabel[i]) gifmap2.areas.append(HT.Area(shape='rect',coords=one_rect_coordinate, title=one_rect_title) ) filename= webqtlUtil.genRandStr("XY_") c.save(webqtlConfig.IMGDIR+filename, format='gif') img2=HT.Image('/image/'+filename+'.gif',border=0, usemap='#CorrelationPlotImageMap2') mainForm_2 = HT.Form( cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name='showDatabase2', submit=HT.Input(type='hidden')) hddn = {'FormID':'showDatabase2','ProbeSetID':'_','database':'_','CellID':'_','RISet':fd.RISet, 'ProbeSetID2':'_', 'database2':'_', 'CellID2':'_', 'allstrainlist':string.join(fd.strainlist, " "), 'traitList': fd.formdata.getvalue("traitList")} if fd.incparentsf1: hddn['incparentsf1'] = 'ON' for key in hddn.keys(): mainForm_2.append(HT.Input(name=key, value=hddn[key], type='hidden')) if isSampleCorr: mainForm_2.append(HT.P(), HT.Blockquote(HT.Strong('X axis:'),HT.Blockquote(trait2Url),HT.Strong('Y axis:'),HT.Blockquote(trait1Url), style='width:%spx;' % self.plotSize)) TD_LR.append(HT.BR(), HT.P()) TD_LR.append('\n', gifmap2, HT.P(), HT.P(), img2, HT.P(), mainForm_2) self.dict['body'] = str(TD_LR) else: heading = "Correlation Plot" detail = ['Fewer than %d strain data were entered for %s data set. No statitical analysis has been attempted.' % (self.corrMinInformative, fd.RISet)] self.error(heading=heading,detail=detail) return
def run_rqtl_geno(self): print("Calling R/qtl") self.geno_to_rqtl_function() ## Get pointers to some common R functions r_library = ro.r["library"] # Map the library function r_c = ro.r["c"] # Map the c function r_sum = ro.r["sum"] # Map the sum function plot = ro.r["plot"] # Map the plot function postscript = ro.r["postscript"] # Map the postscript function png = ro.r["png"] # Map the png function dev_off = ro.r["dev.off"] # Map the device off function print(r_library("qtl")) # Load R/qtl ## Get pointers to some R/qtl functions scanone = ro.r["scanone"] # Map the scanone function scantwo = ro.r["scantwo"] # Map the scantwo function calc_genoprob = ro.r["calc.genoprob"] # Map the calc.genoprob function read_cross = ro.r["read.cross"] # Map the read.cross function write_cross = ro.r["write.cross"] # Map the write.cross function GENOtoCSVR = ro.r["GENOtoCSVR"] # Map the GENOtoCSVR function genofilelocation = webqtlConfig.HTMLPATH + "genotypes/" + self.dataset.group.name + ".geno" crossfilelocation = webqtlConfig.HTMLPATH + "genotypes/" + self.dataset.group.name + ".cross" print("Conversion of geno to cross at location:", genofilelocation, " to ", crossfilelocation) cross_object = GENOtoCSVR(genofilelocation, crossfilelocation) # TODO: Add the SEX if that is available if self.manhattan_plot: cross_object = calc_genoprob(cross_object) else: cross_object = calc_genoprob(cross_object, step=1, stepwidth="max") cross_object = self.add_phenotype(cross_object, self.sanitize_rqtl_phenotype()) # Add the phenotype # for debug: write_cross(cross_object, "csvr", "test.csvr") # Scan for QTLs covar = self.create_covariates(cross_object) # Create the additive covariate matrix if self.pair_scan: if self.do_control == "true": # If sum(covar) > 0 we have a covariate matrix print("Using covariate") result_data_frame = scantwo( cross_object, pheno="the_pheno", addcovar=covar, model=self.model, method=self.method, n_cluster=16 ) else: print("No covariates") result_data_frame = scantwo( cross_object, pheno="the_pheno", model=self.model, method=self.method, n_cluster=16 ) print("Pair scan results:", result_data_frame) self.pair_scan_filename = webqtlUtil.genRandStr("scantwo_") + ".png" png(file=webqtlConfig.TMPDIR + self.pair_scan_filename) plot(result_data_frame) dev_off() return self.process_pair_scan_results(result_data_frame) else: if self.do_control == "true": print("Using covariate") result_data_frame = scanone( cross_object, pheno="the_pheno", addcovar=covar, model=self.model, method=self.method ) else: print("No covariates") result_data_frame = scanone(cross_object, pheno="the_pheno", model=self.model, method=self.method) if int(self.num_perm) > 0: # Do permutation (if requested by user) if self.do_control == "true": perm_data_frame = scanone( cross_object, pheno_col="the_pheno", addcovar=covar, n_perm=int(self.num_perm), model=self.model, method=self.method, ) else: perm_data_frame = scanone( cross_object, pheno_col="the_pheno", n_perm=int(self.num_perm), model=self.model, method=self.method, ) self.process_rqtl_perm_results( perm_data_frame ) # Functions that sets the thresholds for the webinterface return self.process_rqtl_results(result_data_frame)
def run_analysis(self, requestform): print("Starting CTL analysis on dataset") self.trait_db_list = [trait.strip() for trait in requestform['trait_list'].split(',')] self.trait_db_list = [x for x in self.trait_db_list if x] print("strategy:", requestform.get("strategy")) strategy = requestform.get("strategy") print("nperm:", requestform.get("nperm")) nperm = int(requestform.get("nperm")) print("parametric:", requestform.get("parametric")) parametric = bool(requestform.get("parametric")) print("significance:", requestform.get("significance")) significance = float(requestform.get("significance")) # Get the name of the .geno file belonging to the first phenotype datasetname = self.trait_db_list[0].split(":")[1] dataset = data_set.create_dataset(datasetname) genofilelocation = locate(dataset.group.name + ".geno", "genotype") parser = genofile_parser.ConvertGenoFile(genofilelocation) parser.process_csv() print(dataset.group) # Create a genotype matrix individuals = parser.individuals markers = [] markernames = [] for marker in parser.markers: markernames.append(marker["name"]) markers.append(marker["genotypes"]) genotypes = list(itertools.chain(*markers)) print(len(genotypes) / len(individuals), "==", len(parser.markers)) rGeno = r_t(ro.r.matrix(r_unlist(genotypes), nrow=len(markernames), ncol=len(individuals), dimnames = r_list(markernames, individuals), byrow=True)) # Create a phenotype matrix traits = [] for trait in self.trait_db_list: print("retrieving data for", trait) if trait != "": ts = trait.split(':') gt = TRAIT.GeneralTrait(name = ts[0], dataset_name = ts[1]) gt = TRAIT.retrieve_sample_data(gt, dataset, individuals) for ind in individuals: if ind in gt.data.keys(): traits.append(gt.data[ind].value) else: traits.append("-999") rPheno = r_t(ro.r.matrix(r_as_numeric(r_unlist(traits)), nrow=len(self.trait_db_list), ncol=len(individuals), dimnames = r_list(self.trait_db_list, individuals), byrow=True)) print(rPheno) # Use a data frame to store the objects rPheno = r_data_frame(rPheno, check_names = False) rGeno = r_data_frame(rGeno, check_names = False) # Debug: Print the genotype and phenotype files to disk #r_write_table(rGeno, "~/outputGN/geno.csv") #r_write_table(rPheno, "~/outputGN/pheno.csv") # Perform the CTL scan res = self.r_CTLscan(rGeno, rPheno, strategy = strategy, nperm = nperm, parametric = parametric, ncores = 6) # Get significant interactions significant = self.r_CTLsignificant(res, significance = significance) # Create an image for output self.results = {} self.results['imgurl1'] = webqtlUtil.genRandStr("CTLline_") + ".png" self.results['imgloc1'] = GENERATED_IMAGE_DIR + self.results['imgurl1'] self.results['ctlresult'] = significant self.results['requestform'] = requestform # Store the user specified parameters for the output page # Create the lineplot r_png(self.results['imgloc1'], width=1000, height=600, type='cairo-png') self.r_lineplot(res, significance = significance) r_dev_off() n = 2 # We start from 2, since R starts from 1 :) for trait in self.trait_db_list: # Create the QTL like CTL plots self.results['imgurl' + str(n)] = webqtlUtil.genRandStr("CTL_") + ".png" self.results['imgloc' + str(n)] = GENERATED_IMAGE_DIR + self.results['imgurl' + str(n)] r_png(self.results['imgloc' + str(n)], width=1000, height=600, type='cairo-png') self.r_plotCTLobject(res, (n-1), significance = significance, main='Phenotype ' + trait) r_dev_off() n = n + 1 # Flush any output from R sys.stdout.flush() # Create the interactive graph for cytoscape visualization (Nodes and Edges) print(type(significant)) if not type(significant) == ri.RNULLType: for x in range(len(significant[0])): print(significant[0][x], significant[1][x], significant[2][x]) # Debug to console tsS = significant[0][x].split(':') # Source tsT = significant[2][x].split(':') # Target gtS = TRAIT.GeneralTrait(name = tsS[0], dataset_name = tsS[1]) # Retrieve Source info from the DB gtT = TRAIT.GeneralTrait(name = tsT[0], dataset_name = tsT[1]) # Retrieve Target info from the DB self.addNode(gtS) self.addNode(gtT) self.addEdge(gtS, gtT, significant, x) significant[0][x] = gtS.symbol + " (" + gtS.name + ")" # Update the trait name for the displayed table significant[2][x] = gtT.symbol + " (" + gtT.name + ")" # Update the trait name for the displayed table self.elements = json.dumps(self.nodes_list + self.edges_list)
def __init__(self,fd): templatePage.__init__(self,fd) if not self.openMysql(): return self.database = fd.formdata.getvalue('database', '') db = webqtlDataset(self.database, self.cursor) try: self.openURL = webqtlConfig.CGIDIR + webqtlConfig.SCRIPTFILE + \ '?FormID=showDatabase&database=%s&incparentsf1=1&ProbeSetID=' % self.database if db.type != "ProbeSet" or not db.id: raise DbNameError self.cursor.execute(""" Select InbredSet.Name From ProbeSetFreeze, ProbeFreeze, InbredSet whERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id AND ProbeFreeze.InbredSetId = InbredSet.Id AND ProbeSetFreeze.Id = %d """ % db.id) thisRISet = self.cursor.fetchone()[0] if thisRISet =='BXD300': thisRISet = 'BXD' ################################################## # exon data is too huge for GenoGraph, skip it # ################################################## self.cursor.execute('select count(*) from ProbeSetXRef where ProbeSetFreezeId=%d' % db.id) amount = self.cursor.fetchall() if amount: amount = amount[0][0] if amount>100000: heading = "Whole Transcriptome Mapping" detail = ["Whole Transcriptome Mapping is not available for this data set."] self.error(heading=heading,detail=detail) return self.cursor.execute(""" Select ProbeSet.Id, ProbeSet.Name, ProbeSet.Chr, ProbeSet.Mb, ProbeSetXRef.Locus, ProbeSetXRef.pValue From ProbeSet, ProbeSetXRef whERE ProbeSetXRef.ProbeSetFreezeId = %d AND ProbeSetXRef.ProbeSetId = ProbeSet.Id AND ProbeSetXRef.Locus is not NULL """ % db.id) results = self.cursor.fetchall() if results: self.mouseChrLengthDict, sum = self.readMouseGenome(thisRISet) import reaper markerGMb = {} genotype_1 = reaper.Dataset() genotype_1.read(os.path.join(webqtlConfig.GENODIR, thisRISet + '.geno')) for chr in genotype_1: chrlen = self.mouseChrLengthDict[chr.name] for locus in chr: markerGMb[locus.name] = locus.Mb + chrlen try: FDR = float(fd.formdata.getvalue("fdr", "")) except: FDR = 0.2 self.grid = fd.formdata.getvalue("grid", "") NNN = len(results) results = list(results) results.sort(self.cmppValue) MbCoord = [] MbCoord2 = [] for j in range(NNN, 0, -1): if results[j-1][-1] <= (FDR*j)/NNN: break if j > 0: for i in range(j-1, -1, -1): _Id, _probeset, _chr, _Mb, _marker, _pvalue = results[i] try: MbCoord.append([markerGMb[_marker], _Mb+self.mouseChrLengthDict[string.strip(_chr)], _probeset, _chr, _Mb, _marker, _pvalue]) except: pass filename=webqtlUtil.genRandStr("gScan_") canvas = pid.PILCanvas(size=(1280,880)) self.drawGraph(canvas, MbCoord, cLength=sum) canvas.save(os.path.join(webqtlConfig.IMGDIR, filename), format='png') canvasSVG = self.drawSVG(MbCoord, cLength=sum, size=(1280,880)) canvasSVG.toXml(os.path.join(webqtlConfig.IMGDIR, filename+'.svg')) #and write it to file img = HT.Embed(src='/image/'+filename+'.png', width=1280, height=880, border=0, alt='Genome Scan') img2 = HT.Embed(src='/image/'+filename+'.svg', width=1280, height=880, border=0) TD_LR = HT.TD(colspan=2,height=200,width="100%",bgColor='#eeeeee') heading = HT.Paragraph('Whole Transcriptome Mapping') heading.__setattr__("class","title") intro = HT.Blockquote() intro.append('The plot below is the Whole Transcriptome Mapping of Database ') intro.append(HT.Href(text=db.fullname, url = webqtlConfig.INFOPAGEHREF % db.name ,target='_blank',Class="normalsize")) intro.append(". %d from a total of %d ProbeSets were utilized to generate this graph." % (len(MbCoord), len(results))) mainfm = HT.Form(cgi = os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', \ name=webqtlUtil.genRandStr("fm_"), submit=HT.Input(type='hidden')) mainfm.append(HT.Input(name='database', value=self.database, type='hidden')) mainfm.append(HT.Input(name='FormID', value='transciptMapping', type='hidden')) mainfm.append("<BLOCKQUOTE>") mainfm.append("0 < FDR <= 1.0 ") mainfm.append(HT.Input(name='fdr', value=FDR, type='text')) mainfm.append(HT.Input(name='submit', value='Redraw Graph', type='submit', Class='button')) mainfm.append("</BLOCKQUOTE>") mainfm.append(""" <center> <div id="gallery"> <div class="on" title="img1"><span>Static</span></div> <div class="off" title="img2"><span>Interactive</span></div> </div> <div id="img1" class="show"> """) mainfm.append(img) mainfm.append(""" </div> <div id="img2" class="hide"> """) mainfm.append(img2) mainfm.append(""" </div> </center> """) TD_LR.append(heading, intro, HT.Paragraph(mainfm)) self.dict['title'] = 'Whole Transcriptome Mapping' self.dict['body'] = TD_LR else: heading = "Whole Transcriptome Mapping" detail = ["Database calculation is not finished."] self.error(heading=heading,detail=detail) return except: heading = "Whole Transcriptome Mapping" detail = ["Whole Transcriptome Mapping only apply to Microarray database."] self.error(heading=heading,detail=detail) return
def insertUpdateCheck(self, fd, warning= ""): self.dict['title'] = "%s GeneWiki Entry for %s" % (self.action.title(), self.symbol) #mailsrch = re.compile('([\w\-][\w\-\.]*@[\w\-][\w\-\.]+[a-zA-Z]{1,4})([\s,;])*') mailsrch = re.compile('([\w\-][\w\-\.]*)@([\w\-\.]+)\.([a-zA-Z]{1,4})([\s,;])*') httpsrch = re.compile('((?:http|ftp|gopher|file)://(?:[^ \n\r<\)]+))([\s,;])*') if not self.comment or not self.email: heading = self.dict['title'] detail = ["Please don't leave text field or email field empty."] self.error(heading=heading,detail=detail,error="Error") return if self.action == 'update' and not self.reason: heading = self.dict['title'] detail = ["Please submit your reason for this modification."] self.error(heading=heading,detail=detail,error="Error") return if len(self.comment) >500: heading = self.dict['title'] detail = ["Your entry is more than 500 characters."] self.error(heading=heading,detail=detail,error="Error") return if self.email and re.sub(mailsrch, "", self.email) != "": heading = self.dict['title'] detail = ["The format of your email address is incorrect."] self.error(heading=heading,detail=detail,error="Error") return if self.weburl == "http://": self.weburl = "" if self.weburl and re.sub(httpsrch, "", self.weburl) != "": heading = self.dict['title'] detail = ["The format of web resource link is incorrect."] self.error(heading=heading,detail=detail,error="Error") return if self.pubmedid: try: test = map(int, string.split(self.pubmedid)) except: heading = self.dict['title'] detail = ["PubMed IDs can only be integers."] self.error(heading=heading,detail=detail,error="Error") return form = HT.Form(cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), name='addgenerif',submit=HT.Input(type='hidden')) recordInfoTable = HT.TableLite(border=0, cellspacing=1, cellpadding=5,align="center") addButton = HT.Input(type='submit',name='submit', value='%s GeneWiki Entry' % self.action.title(),Class="button") hddn = {'curStatus':'insertResult', 'FormID':'geneWiki', 'symbol':self.symbol, 'comment':self.comment, 'email':self.email, 'species':self.species, 'action':self.action, 'reason':self.reason} if self.Id: hddn['Id']=self.Id formBody = HT.TableLite() formBody.append(HT.TR( HT.TD(HT.Strong("Species: ")), HT.TD(width=10), HT.TD(string.split(self.species, ":")[0]) )) if self.pubmedid: try: formBody.append(HT.TR( HT.TD(HT.Strong("PubMed IDs: ")), HT.TD(width=10), HT.TD(self.pubmedid) )) hddn['pubmedid'] = self.pubmedid except: pass if self.weburl: try: formBody.append(HT.TR( HT.TD(HT.Strong("Web URL: ")), HT.TD(width=10), HT.TD(HT.Href(text=self.weburl, url=self.weburl, Class='fwn')) )) hddn['weburl'] = self.weburl except: pass formBody.append(HT.TR( HT.TD(HT.Strong("Gene Notes: ")), HT.TD(width=10), HT.TD(self.comment) )) formBody.append(HT.TR( HT.TD(HT.Strong("Email: ")), HT.TD(width=10), HT.TD(self.email) )) if self.initial: formBody.append(HT.TR( HT.TD(HT.Strong("Initial: ")), HT.TD(width=10), HT.TD(self.initial) )) hddn['initial'] = self.initial if self.genecategory: cTD = HT.TD() if type(self.genecategory) == type(""): self.genecategory = string.split(self.genecategory) self.cursor.execute("Select Id, Name from GeneCategory where Id in (%s) order by Name " % string.join(self.genecategory, ', ')) results = self.cursor.fetchall() for item in results: cTD.append(item[1], HT.BR()) formBody.append(HT.TR( HT.TD(HT.Strong("Category: ")), HT.TD(width=10), cTD )) hddn['genecategory'] = string.join(self.genecategory, " ") formBody.append(HT.TR( HT.TD( HT.BR(), HT.BR(), HT.Div("For security reasons, enter the code (case insensitive) in the image below to finalize your submission"), HT.BR(), addButton, HT.Input(type="password", size = 25, name="password"), colspan=3) )) code = webqtlUtil.genRandStr(length=5, chars="abcdefghkmnpqrstuvwxyzABCDEFGHJKMNPQRSTUVWXYZ23456789") filename= webqtlUtil.genRandStr("Sec_") hddn['filename'] = filename securityCanvas = pid.PILCanvas(size=(300,100)) Plot.plotSecurity(securityCanvas, text=code) os.system("touch %s_.%s" % (os.path.join(webqtlConfig.IMGDIR,filename), code)) securityCanvas.save(os.path.join(webqtlConfig.IMGDIR,filename), format='png') formBody.append(HT.TR( HT.TD(HT.Image("/image/"+filename+".png"), colspan=3) )) hddn['filename'] = filename TD_LR = HT.TD(valign="top", bgcolor="#eeeeee") title = HT.Paragraph("%s GeneWiki Entry for %s" % (self.action.title(), self.symbol), Class="title") form.append(HT.P(), HT.Blockquote(formBody)) for key in hddn.keys(): form.append(HT.Input(name=key, value=hddn[key], type='hidden')) TD_LR.append(title, HT.Blockquote(warning, Id="red"), form) self.dict['body'] = TD_LR
class IntervalAnalystPage(templatePage): filename = webqtlUtil.genRandStr("Itan_") _scriptfile = "main.py?FormID=intervalAnalyst" #A dictionary that lets us map the html form names "txStart_mm6" -> "Mb Start (mm8)" #the first item is the short name (column headers) and the second item is the long name (dropdown list) # [short name, long name, category] columnNames = {"GeneSymbol" : ["Gene", "Gene Name", 'gene'], "GeneDescription" : ["Description", "Gene Description", 'species'], 'GeneNeighborsCount' : ["Neighbors", "Gene Neighbors", 'gene'], 'GeneNeighborsRange' : ["Neighborhood", "Gene Neighborhood (Mb)", 'gene'], 'GeneNeighborsDensity' : ["Gene Density", "Gene Density (Neighbors/Mb)", 'gene'], "ProteinID" : ["Prot ID", "Protein ID", 'protein'], "Chromosome" : ["Chr", "Chromosome", 'species'], "TxStart" : ["Start", "Mb Start", 'species'], "TxEnd" : ["End", "Mb End", 'species'], "GeneLength" : ["Length", "Kb Length", 'species'], "cdsStart" : ["CDS Start", "Mb CDS Start", 'species'], "cdsEnd" : ["CDS End", "Mb CDS End", 'species'], "exonCount" : ["Num Exons", "Exon Count", 'species'], "exonStarts" : ["Exon Starts", "Exon Starts", 'species'], "exonEnds" : ["Exon Ends", "Exon Ends", 'species'], "Strand" : ["Strand", "Strand", 'species'], "GeneID" : ["Gene ID", "Gene ID", 'species'], "GenBankID" : ["GenBank", "GenBank ID", 'species'], "UnigenID" : ["Unigen", "Unigen ID", 'species'], "NM_ID" : ["NM ID", "NM ID", 'species'], "kgID" : ["kg ID", "kg ID", 'species'], "snpCount" : ["SNPs", "SNP Count", 'species'], "snpDensity" : ["SNP Density", "SNP Density", 'species'], "lrs" : ["LRS", "Likelihood Ratio Statistic", 'misc'], "lod" : ["LOD", "Likelihood Odds Ratio", 'misc'], "pearson" : ["Pearson", "Pearson Product Moment", 'misc'], "literature" : ["Lit Corr", "Literature Correlation", 'misc'], } ###Species Freeze speciesFreeze = {'mouse':'mm9', 'rat':'rn3', 'human':'hg19'} for key in speciesFreeze.keys(): speciesFreeze[speciesFreeze[key]] = key def __init__(self, fd): templatePage.__init__(self, fd) fd.formdata['remote_ip'] = fd.remote_ip if not self.openMysql(): return self.species = fd.formdata.getvalue("species", "mouse") try: self.startMb = float(fd.formdata.getvalue("startMb")) except: self.startMb = 10 try: self.endMb = float(fd.formdata.getvalue("endMb")) except: self.endMb = self.startMb + 10 self.Chr = fd.formdata.getvalue("chromosome", "1") self.xls = fd.formdata.getvalue("xls", "1") try: s1 = int(fd.formdata.getvalue("s1")) s2 = int(fd.formdata.getvalue("s2")) self.diffColDefault = self.diffCol = [s1, s2] except: self.diffColDefault = self.diffCol = [] if self.species != 'mouse': self.diffColDefault = [2, 3]#default is B6 and D2 for other species controlFrm, dispFields = self.genControlForm(fd) geneTable, filename = self.genGeneTable(fd, dispFields) infoTD = HT.TD(width=400, valign= "top") infoTD.append(HT.Paragraph("Interval Analyst : Chr %s" % self.Chr, Class="title"), HT.Strong("Species : "), self.species.title(), HT.BR(), HT.Strong("Database : "), "UCSC %s" % self.speciesFreeze[self.species], HT.BR(), HT.Strong("Range : "), "%2.6f Mb - %2.6f Mb" % (self.startMb, self.endMb), HT.BR(), ) if filename: infoTD.append(HT.BR(), HT.BR(), HT.Href(text="Download", url = "/tmp/" + filename, Class="normalsize") , " output in MS excel format.") mainTable = HT.TableLite(HT.TR(infoTD, HT.TD(controlFrm, Class="doubleBorder", width=400), HT.TD(" ", width="")), cellpadding=10) mainTable.append(HT.TR(HT.TD(geneTable, colspan=3))) self.dict['body'] = HT.TD(mainTable) self.dict['title'] = "Interval Analyst" def genGeneTable(self, fd, dispFields): filename = "" if self.xls: #import pyXLWriter as xl filename = "IntAn_Chr%s_%2.6f-%2.6f" % (self.Chr, self.startMb, self.endMb) filename += ".xls" # Create a new Excel workbook workbook = xl.Writer(os.path.join(webqtlConfig.TMPDIR, filename)) worksheet = workbook.add_worksheet() titleStyle = workbook.add_format(align = 'left', bold = 0, size=18, border = 1, border_color="gray") headingStyle = workbook.add_format(align = 'center', bold = 1, size=13, fg_color = 0x1E, color="white", border = 1, border_color="gray") ##Write title Info worksheet.write([0, 0], "GeneNetwork Interval Analyst Table", titleStyle) worksheet.write([1, 0], "%s%s" % (webqtlConfig.PORTADDR, os.path.join(webqtlConfig.CGIDIR, self._scriptfile))) # worksheet.write([2, 0], "Date : %s" % time.strftime("%B %d, %Y", time.gmtime())) worksheet.write([3, 0], "Time : %s GMT" % time.strftime("%H:%M ", time.gmtime())) worksheet.write([4, 0], "Search by : %s" % fd.formdata['remote_ip']) worksheet.write([5, 0], "view region : Chr %s %2.6f - %2.6f Mb" % (self.Chr, self.startMb, self.endMb)) nTitleRow = 7 geneTable = HT.TableLite(Class="collap", cellpadding=5) headerRow = HT.TR(HT.TD(" ", Class="fs13 fwb ffl b1 cw cbrb", width="1")) if self.xls: worksheet.write([nTitleRow, 0], "Index", headingStyle) for ncol, column in enumerate(dispFields): if len(column) == 1: headerRow.append(HT.TD(self.columnNames[column[0]][0], Class="fs13 fwb ffl b1 cw cbrb", NOWRAP=1,align="Center")) if self.xls: colTitle = self.columnNames[column[0]][0] worksheet.write([nTitleRow, ncol+1], colTitle, headingStyle) worksheet.set_column([ncol+1, ncol+1], 2*len(colTitle)) else: headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(), " (%s)" % self.speciesFreeze[column[1]], Class="fs13 fwb ffl b1 cw cbrb", NOWRAP=1, align="Center")) if self.xls: colTitle = self.columnNames[column[0]][0] + " (%s)" % self.speciesFreeze[column[1]] worksheet.write([nTitleRow, ncol+1], colTitle, headingStyle) worksheet.set_column([ncol+1, ncol+1], 2*len(colTitle)) #headerRow.append(HT.TD(self.columnNames[column[0]][0], HT.BR(), # "(%s %s)" % (column[1].title(), self.speciesFreeze[column[1]]), # Class="colorBlue", NOWRAP=1, align="Center")) geneTable.append(headerRow) geneCol = GeneUtil.loadGenes(self.cursor, self.Chr, self.diffColDefault, self.startMb, self.endMb, species=self.species) for gIndex, theGO in enumerate(geneCol): geneRow = HT.TR(HT.TD(gIndex+1, Class="fs12 fwn b1", align="right")) if self.xls: nTitleRow += 1 worksheet.write([nTitleRow, 0], gIndex + 1) for ncol, column in enumerate(dispFields): if len(column) == 1 or column[1]== self.species: keyValue = "" fieldName = column[0] curSpecies = self.species curGO = theGO if theGO.has_key(fieldName): keyValue = theGO[fieldName] else: fieldName , othSpec = column curSpecies = othSpec subGO = '%sGene' % othSpec keyValue = "" curGO = theGO[subGO] if theGO[subGO].has_key(fieldName): keyValue = theGO[subGO][fieldName] if self.xls: worksheet.write([nTitleRow, ncol+1], keyValue) geneRow.append(self.formatTD(keyValue, fieldName, curSpecies, curGO)) geneTable.append(geneRow) if self.xls: workbook.close() return geneTable, filename def formatTD(self, keyValue, fieldName, Species, theGO): if keyValue is None: keyValue = "" if keyValue != "": if fieldName in ("exonStarts", "exonEnds"): keyValue = string.replace(keyValue, ',', ' ') return HT.TD(HT.Span(keyValue, Class="code", Id="green"), width=350, Class="fs12 fwn b1") elif fieldName in ("GeneDescription"): if keyValue == "---": keyValue = "" return HT.TD(keyValue, Class="fs12 fwn b1", width=300) elif fieldName in ("GeneSymbol"): webqtlLink = HT.Href("./%s?cmd=sch&gene=%s&alias=1&species=%s" % (webqtlConfig.SCRIPTFILE, keyValue, Species), HT.Image("/images/webqtl_search.gif", border=0, valign="top"), target="_blank") if theGO['GeneID']: geneSymbolLink = HT.Href(webqtlConfig.NCBI_LOCUSID % theGO['GeneID'], keyValue, Class="normalsize", target="_blank") else: geneSymbolLink = keyValue return HT.TD(webqtlLink, geneSymbolLink, Class="fs12 fwn b1",NOWRAP=1) elif fieldName == 'UnigenID': try: gurl = HT.Href(webqtlConfig.UNIGEN_ID % tuple(string.split(keyValue,'.')[:2]), keyValue, Class="normalsize", target="_blank") except: gurl = keyValue return HT.TD(gurl, Class="fs12 fwn b1",NOWRAP=1) elif fieldName in ("exonCount", "Chromosome"): return HT.TD(keyValue, Class="fs12 fwn b1",align="right") elif fieldName in ("snpCount"): if keyValue: snpString = HT.Href(url="%s&chr=%s&start=%s&end=%s&geneName=%s&s1=%d&s2=%d" % (os.path.join(webqtlConfig.CGIDIR, 'main.py?FormID=snpBrowser'), theGO["Chromosome"], theGO["TxStart"], theGO["TxEnd"], theGO["GeneSymbol"], self.diffColDefault[0], self.diffColDefault[1]), text=theGO["snpCount"], target="_blank", Class="normalsize") else: snpString = keyValue return HT.TD(snpString, Class="fs12 fwn b1",align="right") elif fieldName in ("snpDensity", "GeneLength"): if keyValue: keyValue = "%2.3f" % keyValue else: keyValue = "" return HT.TD(keyValue, Class="fs12 fwn b1",align="right") elif fieldName in ("TxStart", "TxEnd"): return HT.TD("%2.6f" % keyValue, Class="fs12 fwn b1",align="right") else: return HT.TD(keyValue, Class="fs12 fwn b1",NOWRAP=1) else: return HT.TD(keyValue, Class="fs12 fwn b1",NOWRAP=1,align="right") def genControlForm(self, fd): ##desc GeneList self.cursor.execute("Desc GeneList") GeneListFields = self.cursor.fetchall() GeneListFields = map(lambda X: X[0], GeneListFields) #group columns by category--used for creating the dropdown list of possible columns categories = {} for item in self.columnNames.keys(): category = self.columnNames[item] if category[-1] not in categories.keys(): categories[category[-1]] = [item ] else: categories[category[-1]] = categories[category[-1]]+[item] ##List All Species in the Gene Table speciesDict = {} self.cursor.execute("select Species.Name, GeneList.SpeciesId from Species, GeneList where \ GeneList.SpeciesId = Species.Id group by GeneList.SpeciesId order by Species.Id") results = self.cursor.fetchall() speciesField = categories.pop('species', []) categoriesOrder = ['gene', 'protein'] for item in results: specName, specId = item categoriesOrder.append(specName) speciesDict[specName] = specId AppliedField = [] for item2 in speciesField: if item2 in GeneListFields: self.cursor.execute("select %s from GeneList where SpeciesId = %d and %s is not NULL limit 1 " % (item2, specId, item2)) columnApply = self.cursor.fetchone() if not columnApply: continue elif specName != 'mouse' and item2 in ('snpCount', 'snpDensity'): continue else: pass AppliedField.append(item2) categories[specName] = AppliedField categoriesOrder += ['misc'] ############################################################ ## Create the list of possible columns for the dropdown list ############################################################ allColumnsList = HT.Select(name="allColumns", Class="snpBrowserDropBox") for category in categoriesOrder: allFields = categories[category] if allFields: geneOpt = HT.Optgroup(label=category.title()) for item in allFields: if category in self.speciesFreeze.keys(): geneOpt.append(("%s (%s %s)" % (self.columnNames[item][1], category.title(), self.speciesFreeze[category]), "%s__%s" % (item, self.speciesFreeze[category]))) else: geneOpt.append((self.columnNames[item][1], item)) geneOpt.sort() allColumnsList.append(geneOpt) ###################################### ## Create the list of selected columns ###################################### #cols contains the value of all the selected columns submitCols = cols = fd.formdata.getvalue("columns", "default") if cols == "default": if self.species=="mouse": #these are the same columns that are shown on intervalPage.py cols = ['GeneSymbol', 'GeneDescription', 'Chromosome', 'TxStart', 'Strand', 'GeneLength', 'GeneID', 'NM_ID', 'snpCount', 'snpDensity'] elif self.species=="rat": cols = ['GeneSymbol', 'GeneDescription', 'Chromosome', 'TxStart', 'GeneLength', 'Strand', 'GeneID', 'UnigenID'] else: #should not happen cols = [] else: if type(cols)==type(""): cols = [cols] colsLst = [] dispFields = [] for column in cols: if submitCols == "default" and column not in ('GeneSymbol') and (column in GeneListFields or column in speciesField): colsLst.append(("%s (%s %s)" % (self.columnNames[column][1], self.species.title(), self.speciesFreeze[self.species]), "%s__%s" % (column, self.speciesFreeze[self.species]))) dispFields.append([column, self.species]) else: column2 = column.split("__") if len(column2) == 1: colsLst.append((self.columnNames[column2[0]][1], column)) dispFields.append([column]) else: thisSpecies = self.speciesFreeze[column2[1]] colsLst.append(("%s (%s %s)" % (self.columnNames[column2[0]][1], thisSpecies.title(), column2[1]), column)) dispFields.append((column2[0], thisSpecies)) selectedColumnsList = HT.Select(name="columns", Class="snpBrowserSelectBox", multiple="true", data=colsLst, size=6) ########################## ## Create the columns form ########################## columnsForm = HT.Form(name="columnsForm", submit=HT.Input(type='hidden'), cgi=os.path.join(webqtlConfig.CGIDIR, self._scriptfile), enctype="multipart/form-data") columnsForm.append(HT.Input(type="hidden", name="fromdatabase", value= fd.formdata.getvalue("fromdatabase", "unknown"))) columnsForm.append(HT.Input(type="hidden", name="species", value=self.species)) if self.diffCol: columnsForm.append(HT.Input(type="hidden", name="s1", value=self.diffCol[0])) columnsForm.append(HT.Input(type="hidden", name="s2", value=self.diffCol[1])) startBox = HT.Input(type="text", name="startMb", value=self.startMb, size=10) endBox = HT.Input(type="text", name="endMb", value=self.endMb, size=10) addButton = HT.Input(type="button", name="add", value="Add", Class="button", onClick="addToList(this.form.allColumns.options[this.form.allColumns.selectedIndex].text, this.form.allColumns.options[this.form.allColumns.selectedIndex].value, this.form.columns)") removeButton = HT.Input(type="button", name="remove", value="Remove", Class="button", onClick="removeFromList(this.form.columns.selectedIndex, this.form.columns)") upButton = HT.Input(type="button", name="up", value="Up", Class="button", onClick="swapOptions(this.form.columns.selectedIndex, this.form.columns.selectedIndex-1, this.form.columns)") downButton = HT.Input(type="button", name="down", value="Down", Class="button", onClick="swapOptions(this.form.columns.selectedIndex, this.form.columns.selectedIndex+1, this.form.columns)") clearButton = HT.Input(type="button", name="clear", value="Clear", Class="button", onClick="deleteAllElements(this.form.columns)") submitButton = HT.Input(type="submit", value="Refresh", Class="button", onClick="selectAllElements(this.form.columns)") selectChrBox = HT.Select(name="chromosome") self.cursor.execute(""" Select Chr_Length.Name, Length from Chr_Length, Species where Chr_Length.SpeciesId = Species.Id AND Species.Name = '%s' Order by Chr_Length.OrderId """ % self.species) results = self.cursor.fetchall() for chrInfo in results: selectChrBox.append((chrInfo[0], chrInfo[0])) selectChrBox.selected.append(self.Chr) innerColumnsTable = HT.TableLite(border=0, Class="collap", cellpadding = 2) innerColumnsTable.append(HT.TR(HT.TD(selectedColumnsList)), HT.TR(HT.TD(clearButton, removeButton, upButton, downButton))) columnsTable = HT.TableLite(border=0, cellpadding=2, cellspacing=0) columnsTable.append(HT.TR(HT.TD(HT.Font("Chr: ", size=-1)), HT.TD(selectChrBox, submitButton)), HT.TR(HT.TD(HT.Font("View: ", size=-1)), HT.TD(startBox, HT.Font("Mb to ", size=-1), endBox, HT.Font("Mb", size=-1))), HT.TR(HT.TD(HT.Font("Show: ", size=-1)), HT.TD(allColumnsList, addButton)), HT.TR(HT.TD(""), HT.TD(innerColumnsTable))) columnsForm.append(columnsTable) #columnsForm.append(HT.Input(type="hidden", name="sort", value=diffCol), # HT.Input(type="hidden", name="identification", value=identification), # HT.Input(type="hidden", name="traitInfo", value=traitInfo)) return columnsForm, dispFields