def export_dataStage01PhysiologyRatesAverages_js(self,analysis_id_I,data_dir_I="tmp"): """Export data_stage01_physiology_ratesAverages to js file""" # get the analysis information experiment_ids,sample_name_abbreviations = [],[]; experiment_ids,sample_name_abbreviations = self.get_experimentIDAndSampleName_analysisID_dataStage01PhysiologyAnalysis(analysis_id_I); data_O = []; for sna_cnt,sna in enumerate(sample_name_abbreviations): data_tmp = []; data_tmp = self.get_rows_experimentIDAndSampleNameAbbreviation_dataStage01PhysiologyRatesAverages(experiment_ids[sna_cnt],sna); if data_tmp: data_O.extend(data_tmp); # visualization parameters data1_keys = ['experiment_id','sample_name_abbreviation', 'met_id','rate_units']; data1_nestkeys = ['met_id']; data1_keymap = {'xdata':'met_id','ydata':'rate_average', 'serieslabel':'sample_name_abbreviation','featureslabel':'met_id', 'ydatalb':'rate_lb','ydataub':'rate_ub'}; nsvgtable = ddt_container_filterMenuAndChart2dAndTable(); nsvgtable.make_filterMenuAndChart2dAndTable( data_filtermenu=data_O, data_filtermenu_keys=data1_keys, data_filtermenu_nestkeys=data1_nestkeys, data_filtermenu_keymap=data1_keymap, data_svg_keys=None, data_svg_nestkeys=None, data_svg_keymap=None, data_table_keys=None, data_table_nestkeys=None, data_table_keymap=None, data_svg=None, data_table=None, svgtype='verticalbarschart2d_01', tabletype='responsivetable_01', svgx1axislabel='met_id', svgy1axislabel='Rate', tablekeymap = [data1_keymap], svgkeymap = [data1_keymap], #calculated on the fly formtile2datamap=[0], tabletile2datamap=[0], svgtile2datamap=[0], #calculated on the fly svgfilters=None, svgtileheader='Physiological data', tablefilters=None, tableheaders=None, svgparameters_I= { "svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 }, "svgwidth":500,"svgheight":350, 'colclass':"col-sm-8" } ); if data_dir_I=='tmp': filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js' elif data_dir_I=='data_json': data_json_O = nsvgtable.get_allObjects_js(); return data_json_O; with open(filename_str,'w') as file: file.write(nsvgtable.get_allObjects());
def export_dataStage01IsotopomerAveragesNormSum_js(self,experiment_id_I, time_points_I = None, sample_name_abbreviations_I = None, met_ids_I = None, scan_types_I = None,data_dir_I="tmp", single_plot_I = True): '''Export data_stage01_isotopomer_averagesNormSum to js file''' mids = mass_isotopomer_distributions(); print('plotting averagesNormSum...') data_O = []; data_dict_O = {}; # get time points if time_points_I: time_points = time_points_I; else: time_points = []; time_points = self.get_timePoint_experimentID_dataStage01AveragesNormSum(experiment_id_I); for tp in time_points: print('Plotting product and precursor for time-point ' + str(tp)); # get sample names and sample name abbreviations sample_abbreviations = []; sample_types = ['Unknown']; sample_types_lst = []; for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01AveragesNormSum(experiment_id_I,st,tp); sample_abbreviations.extend(sample_abbreviations_tmp); sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]); if sample_name_abbreviations_I: sample_abbreviations = [sna for sna in sample_abbreviations if sna in sample_name_abbreviations_I]; sample_types_lst = ['Unknown' for x in sample_abbreviations]; for sna_cnt,sna in enumerate(sample_abbreviations): print('Plotting product and precursor for sample name abbreviation ' + sna); data_dict_O[sna] = []; # get the scan_types if scan_types_I: scan_types = []; scan_types_tmp = []; scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); scan_types = [st for st in scan_types_tmp if st in scan_types_I]; else: scan_types = []; scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); for scan_type in scan_types: print('Plotting product and precursor for scan type ' + scan_type) # met_ids if not met_ids_I: met_ids = []; met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01AveragesNormSum( \ experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type); else: met_ids = met_ids_I; if not(met_ids): continue #no component information was found for met in met_ids: print('Plotting product and precursor for metabolite ' + met); # fragments fragment_formulas = []; fragment_formulas = self.get_fragmentFormula_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanTypeAndMetID_dataStage01AveragesNormSum( \ experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met); for frag in fragment_formulas: print('Plotting product and precursor for fragment ' + frag); # data data_mat = []; data_mat_cv = []; data_masses = []; data_mat,data_mat_cv,data_masses = self.get_spectrum_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanTypeAndMetIDAndFragmentFormula_dataStage01AveragesNormSum( \ experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met,frag); for i,d in enumerate(data_mat): stdev = 0.0; stderr = 0.0; lb = None; ub = None; if data_mat_cv[i]: stdev = data_mat[i]*data_mat_cv[i]/100; lb = data_mat[i]-stdev; ub = data_mat[i]+stdev; fragment_id = mids.make_fragmentID(met,frag,data_masses[i]); tmp = { 'experiment_id':experiment_id_I, 'sample_name_abbreviation':sna, 'sample_type':sample_types_lst[sna_cnt], 'time_point':tp, 'met_id':met, 'fragment_formula':frag, 'fragment_mass':data_masses[i], 'fragment_id':fragment_id, 'intensity_normalized_average':d, 'intensity_normalized_cv':data_mat_cv[i], 'intensity_normalized_lb':lb, 'intensity_normalized_ub':ub, 'intensity_normalized_units':"normSum", 'scan_type':scan_type, 'used_':True, 'comment_':None} data_O.append(tmp); data_dict_O[sna].append(tmp); print('tabulating averagesNormSum...') data_table_O = []; # get time points if time_points_I: time_points = time_points_I; else: time_points = []; time_points = self.get_timePoint_experimentID_dataStage01AveragesNormSum(experiment_id_I); for tp in time_points: print('Tabulating product and precursor for time-point ' + str(tp)); # get sample names and sample name abbreviations sample_abbreviations = []; sample_types = ['Unknown']; sample_types_lst = []; for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01AveragesNormSum(experiment_id_I,st,tp); sample_abbreviations.extend(sample_abbreviations_tmp); sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]); if sample_name_abbreviations_I: sample_abbreviations = [sna for sna in sample_abbreviations if sna in sample_name_abbreviations_I]; sample_types_lst = ['Unknown' for x in sample_abbreviations]; for sna_cnt,sna in enumerate(sample_abbreviations): print('Tabulating product and precursor for sample name abbreviation ' + sna); # get the scan_types if scan_types_I: scan_types = []; scan_types_tmp = []; scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); scan_types = [st for st in scan_types_tmp if st in scan_types_I]; else: scan_types = []; scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); for scan_type in scan_types: print('Tabulating product and precursor for scan type ' + scan_type) # met_ids if not met_ids_I: met_ids = []; met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01AveragesNormSum( \ experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type); else: met_ids = met_ids_I; if not(met_ids): continue #no component information was found for met in met_ids: print('Tabulating product and precursor for metabolite ' + met); # get rows rows = []; rows = self.get_rows_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanTypeAndMetID_dataStage01AveragesNormSum( \ experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met); if not rows: continue; for d in rows: d['fragment_id'] = mids.make_fragmentID(d['met_id'],d['fragment_formula'],d['fragment_mass']); data_table_O.extend(rows); print('exporting averagesNormSum...'); # visualization parameters data1_keys = ['sample_name_abbreviation', 'sample_type', 'met_id', 'time_point', 'fragment_formula', 'fragment_mass', 'scan_type', 'fragment_id' ]; data1_nestkeys = [ #'fragment_id', 'fragment_mass' ]; data1_keymap = { #'xdata':'fragment_id', 'xdata':'fragment_mass', 'ydata':'intensity_normalized_average', 'tooltiplabel':'sample_name_abbreviation', 'serieslabel':'scan_type', #'featureslabel':'fragment_id', 'featureslabel':'fragment_mass', 'ydatalb':'intensity_normalized_lb', 'ydataub':'intensity_normalized_ub'}; # initialize the ddt objects dataobject_O = []; parametersobject_O = []; tile2datamap_O = {}; # make the tile parameter objects formtileparameters_O = {'tileheader':'Filter menu','tiletype':'html','tileid':"filtermenu1",'rowid':"row1",'colid':"col1", 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"}; formparameters_O = {'htmlid':'filtermenuform1',"htmltype":'form_01',"formsubmitbuttonidtext":{'id':'submit1','text':'submit'},"formresetbuttonidtext":{'id':'reset1','text':'reset'},"formupdatebuttonidtext":{'id':'update1','text':'update'}}; formtileparameters_O.update(formparameters_O); dataobject_O.append({"data":data_O,"datakeys":data1_keys,"datanestkeys":data1_nestkeys}); parametersobject_O.append(formtileparameters_O); tile2datamap_O.update({"filtermenu1":[0]}); if not single_plot_I: rowcnt = 1; colcnt = 1; for cnt,sn in enumerate(sample_abbreviations): svgtileid = "tilesvg"+str(cnt); svgid = 'svg'+str(cnt); iter=cnt+1; #start at 1 if (cnt % 2 == 0): rowcnt = rowcnt+1;#even colcnt = 1; else: colcnt = colcnt+1; # make the svg object svgparameters1_O = {"svgtype":'verticalbarschart2d_01',"svgkeymap":[data1_keymap], #'svgid':'svg1', 'svgid':'svg'+str(cnt), "svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 }, "svgwidth":350,"svgheight":250,"svgy1axislabel":"intensity (norm)", "svgfilters":{'sample_name_abbreviation':[sn]} }; svgtileparameters1_O = {'tileheader':'Isotopomer distribution','tiletype':'svg', 'tileid':svgtileid, 'rowid':"row"+str(rowcnt), 'colid':"col"+str(colcnt), 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-6"}; svgtileparameters1_O.update(svgparameters1_O); parametersobject_O.append(svgtileparameters1_O); tile2datamap_O.update({svgtileid:[0]}); else: cnt = 0; svgtileid = "tilesvg"+str(cnt); svgid = 'svg'+str(cnt); rowcnt = 2; colcnt = 1; # make the svg object svgparameters1_O = {"svgtype":'verticalbarschart2d_01',"svgkeymap":[data1_keymap], 'svgid':'svg'+str(cnt), "svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 }, "svgwidth":500,"svgheight":350,"svgy1axislabel":"intensity (norm)", }; svgtileparameters1_O = {'tileheader':'Isotopomer distribution','tiletype':'svg', 'tileid':svgtileid, 'rowid':"row"+str(rowcnt), 'colid':"col"+str(colcnt), 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"}; svgtileparameters1_O.update(svgparameters1_O); parametersobject_O.append(svgparameters1_O); tile2datamap_O.update({svgtileid:[0]}); # make the table object tableparameters1_O = {"tabletype":'responsivetable_01', 'tableid':'table1', "tablefilters":None, #"tableheaders":[], "tableclass":"table table-condensed table-hover", 'tableformtileid':'tile1','tableresetbuttonid':'reset1','tablesubmitbuttonid':'submit1'}; tabletileparameters1_O = {'tileheader':'Isotopomer distribution','tiletype':'table','tileid':"tabletile1", 'rowid':"row100", 'colid':"col1", 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"}; tabletileparameters1_O.update(tableparameters1_O); dataobject_O.append({"data":data_table_O,"datakeys":data1_keys,"datanestkeys":data1_nestkeys}); parametersobject_O.append(tabletileparameters1_O); tile2datamap_O.update({"tabletile1":[1]}) nsvgtable = ddt_container_filterMenuAndChart2dAndTable(); nsvgtable.make_filterMenuAndChart2dAndTable( data_filtermenu=data_O, data_filtermenu_keys=data1_keys, data_filtermenu_nestkeys=data1_nestkeys, data_filtermenu_keymap=data1_keymap, data_svg_keys=data1_keys, data_svg_nestkeys=data1_nestkeys, data_svg_keymap=data1_keymap, data_table_keys=None, data_table_nestkeys=None, data_table_keymap=None, data_svg=data_dict_O, data_table=None, svgtype='verticalbarschart2d_01', tabletype='responsivetable_01', svgx1axislabel='', svgy1axislabel='', tablekeymap = [data1_keymap], svgkeymap = [], #calculated on the fly formtile2datamap=[0], tabletile2datamap=[0], svgtile2datamap=[], #calculated on the fly svgfilters=None, svgtileheader='Isotopomer distribution', tablefilters=None, tableheaders=None ); if data_dir_I=='tmp': filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js' elif data_dir_I=='data_json': data_json_O = nsvgtable.get_allObjects_js(); return data_json_O; with open(filename_str,'w') as file: file.write(nsvgtable.get_allObjects());
def export_dataStage01MQResultsTable_js( self,analysis_id_I, features_I=[ 'calculated_concentration', 'height_ratio', 'retention_time'], include_IS_I=False, data_dir_I="tmp"): ''' export a metric plot INPUT: experiment_id_I = experiment_id sample_names_I = sample_names component_names_I = component names features_I = features to plot, supports calculated_concentration, height_ratio, area_ratio, height, area, retention_time ''' #quantification_analysis_query = stage01_quantification_analysis_query(self.session,self.engine,self.settings) # get the data: data_O = []; data_dict_O = {}; data_tmp = []; data_tmp = self.getRowsJoin_analysisID_dataStage01QuantificationMQResultsTable(analysis_id_I); # make a unique index for each component component_names = list(set([d['component_name'] for d in data_tmp])); component_names_dict = {d:0 for d in component_names}; for d in data_tmp: tmp = {}; tmp['acquisition_date_and_time'] = self.convert_datetime2string(d['acquisition_date_and_time']) tmp['sample_name'] = d['sample_name']; tmp['sample_type'] = d['sample_type']; #tmp['experiment_id'] = d['experiment_id']; #tmp['analysis_id'] = d['analysis_id']; tmp['analysis_id']=analysis_id_I; tmp['component_name'] = d['component_name']; tmp['component_group_name'] = d['component_group_name']; tmp['component_index'] = component_names_dict[d['component_name']]; for m in features_I: if not m in data_dict_O: data_dict_O[m]=[]; if m in d.keys() and not d[m] is None: tmp1 = copy.copy(tmp); tmp1['feature_name'] = m; tmp1['feature_value'] = d[m]; data_dict_O[m].append(tmp1); data_O.append(tmp1); component_names_dict[d['component_name']]+=1; # dump chart parameters to a js files data1_keys = [ 'analysis_id', 'sample_name', 'component_name', 'feature_name', 'acquisition_date_and_time', 'sample_type', ]; data1_nestkeys = ['component_name']; data1_keymap = {'xdata':'component_index', 'ydata':'feature_value', 'serieslabel':'component_name', 'featureslabel':'sample_name'}; svgkeymap1 = [[data1_keymap,data1_keymap] for k in data_dict_O.keys()] parameters = {"chart1margin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 }, "chart1width":500,"chart1height":350, "chart1title":"Metric Plot", "chart1x1axislabel":"sample_name","chart1y1axislabel":"measurement"} ## make the data object #dataobject_O = [{"data":data_O,"datakeys":data1_keys,"datanestkeys":data1_nestkeys},{"data":data_O,"datakeys":data1_keys,"datanestkeys":data1_nestkeys}]; ## make the tile parameter objects #formtileparameters_O = {'tileheader':'Filter menu','tiletype':'html','tileid':"filtermenu1",'rowid':"row1",'colid':"col1", # 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-4"}; #formparameters_O = {'htmlid':'filtermenuform1','htmltype':'form_01',"formsubmitbuttonidtext":{'id':'submit1','text':'submit'},"formresetbuttonidtext":{'id':'reset1','text':'reset'},"formupdatebuttonidtext":{'id':'update1','text':'update'}}; #formtileparameters_O.update(formparameters_O); #svgparameters_O = {"svgtype":'scatterlineplot2d_01',"svgkeymap":[data1_keymap,data1_keymap], # 'svgid':'svg1', # "svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 }, # "svgwidth":500,"svgheight":350, # "svgx1axislabel":"sample_name","svgy1axislabel":"measurement_value", # 'svgformtileid':'filtermenu1','svgresetbuttonid':'reset1','svgsubmitbuttonid':'submit1'}; #svgtileparameters_O = {'tileheader':'Metric Plot','tiletype':'svg','tileid':"tile2",'rowid':"row1",'colid':"col2", # 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-8"}; #svgtileparameters_O.update(svgparameters_O); #tableparameters_O = {"tabletype":'responsivetable_01', # 'tableid':'table1', # "tablefilters":None, # "tableclass":"table table-condensed table-hover", # 'tableformtileid':'filtermenu1','tableresetbuttonid':'reset1','tablesubmitbuttonid':'submit1'}; #tabletileparameters_O = {'tileheader':'Metric plot','tiletype':'table','tileid':"tile3",'rowid':"row2",'colid':"col1", # 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"}; #tabletileparameters_O.update(tableparameters_O); #parametersobject_O = [formtileparameters_O,svgtileparameters_O,tabletileparameters_O]; #tile2datamap_O = {"filtermenu1":[0],"tile2":[0,1],"tile3":[0]}; ## dump the data to a json file #ddtutilities = ddt_container(parameters_I = parametersobject_O,data_I = dataobject_O,tile2datamap_I = tile2datamap_O,filtermenu_I = None); #if data_dir_I=='tmp': # filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js' #elif data_dir_I=='data_json': # data_json_O = ddtutilities.get_allObjects_js(); # return data_json_O; #with open(filename_str,'w') as file: # file.write(ddtutilities.get_allObjects()); nsvgtable = ddt_container_filterMenuAndChart2dAndTable(); nsvgtable.make_filterMenuAndChart2dAndTable( data_filtermenu=data_O, data_filtermenu_keys=data1_keys, data_filtermenu_nestkeys=data1_nestkeys, data_filtermenu_keymap=data1_keymap, data_svg_keys=None, data_svg_nestkeys=None, data_svg_keymap=None, data_table_keys=None, data_table_nestkeys=None, data_table_keymap=None, data_svg=data_dict_O, data_table=None, #svgtype='scatterlineplot2d_01', svgtype='scatterplot2d_01', tabletype='responsivetable_01', svgx1axislabel='', svgy1axislabel='', tablekeymap = [data1_keymap], svgkeymap = [], #calculated on the fly formtile2datamap=[0], tabletile2datamap=[0], svgtile2datamap=[], #calculated on the fly svgfilters=None, svgtileheader='Metric Plot', tablefilters=None, tableheaders=None, svgparameters_I = parameters ); if data_dir_I=='tmp': filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js' elif data_dir_I=='data_json': data_json_O = nsvgtable.get_allObjects_js(); return data_json_O; with open(filename_str,'w') as file: file.write(nsvgtable.get_allObjects());
def export_dataStage01PhysiologyData_js(self, analysis_id_I, data_units_I=['mM','OD600'], data_dir_I="tmp" ): """ Export data_stage01_physiology_data to js file """ physiology_analysis_query = stage01_physiology_analysis_query(self.session,self.engine,self.settings); # get the analysis information sample_name_shorts = []; sample_name_shorts = physiology_analysis_query.get_rows_analysisID_dataStage01PhysiologyAnalysis(analysis_id_I); data_O = []; #TODO optimize to a single query for sns_cnt,sns in enumerate(sample_name_shorts): data_tmp = []; data_tmp = self.get_sampleDateAndDataCorrected_experimentIDAndSampleNameShort(sns['experiment_id'],sns['sample_name_short'],data_units_I=data_units_I); for d in data_tmp: d['sample_date'] = d['sample_date'].year*8765.81277 + \ d['sample_date'].month*730.484 + d['sample_date'].day*365.242 + \ d['sample_date'].hour + d['sample_date'].minute / 60. + \ d['sample_date'].second / 3600.; #convert using datetime object data_O.append(d); # visualization parameters data1_keys = ['experiment_id', 'sample_id', 'sample_name_short', 'sample_name_abbreviation', 'sample_date', 'met_id', 'data_units' ]; #TODO xaxis = time #TODO seperate plot for each metabolite data1_nestkeys = ['met_id']; data1_keymap = { 'xdata':'sample_date', 'ydata':'data_corrected', 'serieslabel':'sample_name_short', 'featureslabel':'sample_id', }; nsvgtable = ddt_container_filterMenuAndChart2dAndTable(); nsvgtable.make_filterMenuAndChart2dAndTable( data_filtermenu=data_O, data_filtermenu_keys=data1_keys, data_filtermenu_nestkeys=data1_nestkeys, data_filtermenu_keymap=data1_keymap, data_svg_keys=None, data_svg_nestkeys=None, data_svg_keymap=None, data_table_keys=None, data_table_nestkeys=None, data_table_keymap=None, data_svg=None, data_table=None, svgtype='scatterplot2d_01', tabletype='responsivetable_01', svgx1axislabel='time (hrs)', svgy1axislabel='data_corrected', tablekeymap = [data1_keymap], svgkeymap = [data1_keymap], #calculated on the fly formtile2datamap=[0], tabletile2datamap=[0], svgtile2datamap=[0], #calculated on the fly svgfilters=None, svgtileheader='Physiological data', tablefilters=None, tableheaders=None, svgparameters_I= { "svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 }, "svgwidth":500,"svgheight":350, 'colclass':"col-sm-8" } ); if data_dir_I=='tmp': filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js' elif data_dir_I=='data_json': data_json_O = nsvgtable.get_allObjects_js(); return data_json_O; with open(filename_str,'w') as file: file.write(nsvgtable.get_allObjects());
def export_dataStage01RNASequencingGenesFpkmTracking_pairWisePlot_js(self,analysis_id_I,log2normalization_I=True,data_dir_I='tmp'): '''Export data for a pairwise scatter plot INPUT: analysis_id = String, analysis_id log2normalization_I = Boolean, apply a log2 normalization the FPKM values (default: True) data_dir_I = string, data directory OUTPUT: ''' # get the analysis information experiment_ids,sample_names = [],[]; experiment_ids,sample_names = self.get_experimentIDAndSampleName_analysisID_dataStage01RNASequencingAnalysis(analysis_id_I); data_O = []; for sample_name_cnt,sample_name in enumerate(sample_names): # query fpkm data: fpkms = []; fpkms = self.get_rows_experimentIDAndSampleName_dataStage01RNASequencingGenesFpkmTracking(experiment_ids[sample_name_cnt],sample_name); if log2normalization_I: for f in fpkms: if f['FPKM'] == 0.0: f['FPKM'] = 0.0; else: f['FPKM'] = log2(f['FPKM']); data_O.extend(fpkms); # reorganize the data listdict = listDict(data_O); data_O,columnValueHeader_O = listdict.convert_listDict2ColumnGroupListDict( #value_labels_I = ['FPKM','FPKM_conf_lo','FPKM_conf_hi'], value_labels_I = ['FPKM',], column_labels_I = ['experiment_id','sample_name'], feature_labels_I = ['gene_id','gene_short_name'], na_str_I=0.0, columnValueConnector_str_I='_', ); # make the tile object #data1 = filtermenu/table data1_keymap_table = { 'xdata':'svd_method', 'ydata':'singular_value_index', 'zdata':'d_vector', 'rowslabel':'svd_method', 'columnslabel':'singular_value_index', }; #data2 = svg #if single plot, data2 = filter menu, data2, and table data1_keys = ['gene_id','gene_short_name' ]; data1_nestkeys = ['gene_short_name']; data1_keymap_svg = []; svgtype = []; svgtile2datamap = []; data_svg_keymap = []; for cnt1,column1 in enumerate(columnValueHeader_O): for cnt2,column2 in enumerate(columnValueHeader_O[cnt1+1:]): keymap = { 'xdata':column1, 'ydata':column2, 'serieslabel':'', 'featureslabel':'gene_short_name', 'tooltipdata':'gene_short_name', }; data1_keymap_svg.append([keymap]); data_svg_keymap.append(keymap); svgtype.append('pcaplot2d_scores_01'); svgtile2datamap.append([0]); nsvgtable = ddt_container_filterMenuAndChart2dAndTable(); nsvgtable.make_filterMenuAndChart2dAndTable( data_filtermenu=data_O, data_filtermenu_keys=data1_keys, data_filtermenu_nestkeys=data1_nestkeys, data_filtermenu_keymap=data1_keymap_table, data_svg_keys=data1_keys, data_svg_nestkeys=data1_nestkeys, data_svg_keymap=data_svg_keymap, data_table_keys=data1_keys, data_table_nestkeys=data1_nestkeys, data_table_keymap=data1_keymap_table, data_svg=None, data_table=None, svgtype=svgtype, tabletype='responsivetable_01', svgx1axislabel='', svgy1axislabel='', tablekeymap = [data1_keymap_table], svgkeymap = data1_keymap_svg, formtile2datamap=[0], tabletile2datamap=[0], svgtile2datamap=svgtile2datamap, svgfilters=None, svgtileheader='Pair-wise scatter plot', tablefilters=None, tableheaders=None ); if data_dir_I=='tmp': filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js' elif data_dir_I=='data_json': data_json_O = nsvgtable.get_allObjects_js(); return data_json_O; with open(filename_str,'w') as file: file.write(nsvgtable.get_allObjects());
def export_dataStage02PhysiologySampledPoints_js(self, analysis_id_I, simulation_ids_I = [], rxn_ids_I = [], query_I={}, data_dir_I='tmp' ): '''Visualize the sampling distribution DESCRIPTION: tile1=filtermenu tile2=sampling distribution tile3=table ''' calculatehistogram = calculate_histogram(); physiology_analysis_query = stage02_physiology_analysis_query(self.session,self.engine,self.settings); data_sampledPoints_O = []; data_O = []; #get the analysis info simulation_ids = physiology_analysis_query.get_simulationID_analysisID_dataStage02PhysiologyAnalysis(analysis_id_I); if simulation_ids_I: simulation_ids = [s for s in simulation_ids if s in simulation_ids_I]; for simulation_id in simulation_ids: #get the data_dirs for the simulations and read in the points sampledPoints = self.get_rows_simulationID_dataStage02PhysiologySampledPoints(simulation_id); sampledPoints = sampledPoints[0]; data_sampledPoints_O.append(sampledPoints); # get simulation information simulation_info_all = []; simulation_info_all = self.get_rows_simulationIDAndSimulationType_dataStage02PhysiologySimulation(simulation_id,'sampling'); if not simulation_info_all: print('simulation not found!') return; simulation_info = simulation_info_all[0]; # unique constraint guarantees only 1 row will be returned # get simulation parameters simulation_parameters_all = []; simulation_parameters_all = self.get_rows_simulationID_dataStage02PhysiologySamplingParameters(simulation_id); if not simulation_parameters_all: print('simulation not found!') return; simulation_parameters = simulation_parameters_all[0]; # unique constraint guarantees only 1 row will be returned #fill the sampledData with the actual points sampling = cobra_sampling(model_I=None,data_dir_I = sampledPoints['data_dir'],loops_I=sampledPoints['infeasible_loops']); if simulation_parameters['sampler_id']=='gpSampler': # load the results of sampling sampling.get_points_matlab(matlab_data=None,sampler_model_name='sampler_out'); sampling.remove_loopsFromPoints(); elif simulation_parameters['sampler_id']=='optGpSampler': sampling.get_points_json(); sampling.remove_loopsFromPoints(); else: print('sampler_id not recognized'); #store the sampledPoints for k,v in sampling.points.items(): if rxn_ids_I: if not k in rxn_ids_I: continue; n_bins = 100; calc_bins_I = False; x_O,dx_O,y_O = calculatehistogram.histogram(data_I=v,n_bins_I=n_bins,calc_bins_I=calc_bins_I); for i,b in enumerate(x_O): tmp = { 'simulation_id':simulation_id, 'rxn_id':k, #'feature_units':feature_units, 'bin':b, 'bin_width':dx_O[i], 'frequency':int(y_O[i]), 'used_':True, 'comment_':None}; data_O.append(tmp); #data_listDict = listDict(); #data_listDict.set_dictList(sampling.points); #data_listDict.convert_dictList2DataFrame(); #points, rxn_ids = data_listDict.get_flattenedDataAndColumnLabels(); #data_listDict.clear_allData(); #data_listDict.add_column2DataFrame('rxn_id',rxn_ids); #data_listDict.add_column2DataFrame('points',points); #data_listDict.add_column2DataFrame('simulation_id',simulation_id); #data_listDict.convert_dataFrame2ListDict(); #data_O.extend(data_listDict.get_listDict()); #make the DDT histograms # visualization parameters data1_keys = ['simulation_id', 'rxn_id', 'bin', ]; data1_nestkeys = [ 'bin' ]; data1_keymap = { 'xdata':'bin', 'ydata':'frequency', 'serieslabel':'simulation_id', 'featureslabel':'bin', 'tooltiplabel':'rxn_id', 'ydatalb':None, 'ydataub':None}; nsvgtable = ddt_container_filterMenuAndChart2dAndTable(); nsvgtable.make_filterMenuAndChart2dAndTable( data_filtermenu=data_O, data_filtermenu_keys=data1_keys, data_filtermenu_nestkeys=data1_nestkeys, data_filtermenu_keymap=data1_keymap, data_svg_keys=None, data_svg_nestkeys=None, data_svg_keymap=None, data_table_keys=None, data_table_nestkeys=None, data_table_keymap=None, data_svg=None, data_table=None, svgtype='verticalbarschart2d_01', tabletype='responsivetable_01', svgx1axislabel='', svgy1axislabel='', tablekeymap = [data1_keymap], svgkeymap = [data1_keymap], formtile2datamap=[0], tabletile2datamap=[0], svgtile2datamap=[0], #calculated on the fly svgfilters=None, svgtileheader='Sampled points', tablefilters=None, tableheaders=None ); if data_dir_I=='tmp': filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js' elif data_dir_I=='data_json': data_json_O = nsvgtable.get_allObjects_js(); return data_json_O; with open(filename_str,'w') as file: file.write(nsvgtable.get_allObjects());
def export_dataStage02PhysiologyGraphDataShortestPaths_js(self,analysis_id_I,data_dir_I='tmp'): '''export graph of shortest paths''' data_O = []; data_reactions_O = []; data_tmp = self.get_rows_analysisID_dataStage02PhysiologyGraphDataShortestPaths(analysis_id_I); for d in data_tmp: d['path_id'] = d['path_start'] + '-->' + d['path_stop'] data_O.append(d); for i,p in enumerate(d['paths']): if i==0: continue; tmp = {}; tmp['left']=d['paths'][i-1]; tmp['right']=p; link_id = p; node_id = 'rxn'; if i%2==0: link_id=d['paths'][i-1]; node_id = 'met'; tmp['link_id']=link_id; tmp['weight']=1; tmp['node_id']=node_id; tmp['path_id'] = d['path_id']; tmp['path_start'] = d['path_start'] tmp['path_stop'] = d['path_stop'] tmp['algorithm'] = d['algorithm'] tmp['analysis_id'] = d['analysis_id'] tmp['simulation_id'] = d['simulation_id'] data_reactions_O.append(tmp); # make the data parameters data1_keys = [ 'simulation_id', 'path_start', 'path_stop', 'algorithm', ]; data1_nestkeys = [ 'left', 'right', ]; data1_keymap = {'xdata':'node_id', 'ydata':'weight', 'serieslabel':'algorithm', 'featureslabel':''}; data2_keymap = { 'xdata':'node_id', 'ydata':'weight', 'xdatalabel':'', 'ydatalabel':'', 'serieslabel':'algorithm', 'featureslabel':'', #'tooltiplabel':'component_name', }; svgparameters = { "svgmargin":{ 'top': 100, 'right': 100, 'bottom': 100, 'left': 100 }, "svgwidth":500, "svgheight":500, "svgduration":750, 'colclass':"col-sm-12", 'svgcolorcategory':'blue2red64', 'svgcolordomain':'min,max', 'svgcolordatalabel':'parent_classes', 'svgcolorscale':'quantile', }; nsvgtable = ddt_container_filterMenuAndChart2dAndTable(); nsvgtable.make_filterMenuAndChart2dAndTable( data_filtermenu=data_reactions_O, data_filtermenu_keys=data1_keys, data_filtermenu_nestkeys=data1_nestkeys, data_filtermenu_keymap=data1_keymap, data_svg_keys=None, data_svg_nestkeys=None, data_svg_keymap=data2_keymap, data_table_keys=None, data_table_nestkeys=None, data_table_keymap=None, data_svg=None, data_table=data_O, #svgtype='sankeydiagram2d_01', svgtype='forcedirectedgraph2d_01', tabletype='responsivetable_01', svgx1axislabel='', svgy1axislabel='', tablekeymap = [data1_keymap], svgkeymap = [data2_keymap], formtile2datamap=[0], tabletile2datamap=[1], svgtile2datamap=[0], svgfilters=None, svgtileheader='Shortest Path', tablefilters=None, tableheaders=None, svgparameters_I= svgparameters, ); if data_dir_I=='tmp': filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js' elif data_dir_I=='data_json': data_json_O = nsvgtable.get_allObjects_js(); return data_json_O; with open(filename_str,'w') as file: file.write(nsvgtable.get_allObjects());
def export_dataStage01ResequencingEndpointLineages_js(self,analysis_id_I, query_I={}, data_dir_I='tmp'): ''' Pie chart or bar chart of mutation counts INPUT: analysis_id_I = string, query_I = additional query deliminators ''' data_O = self.getGroupAndCount_analysisIDAndMutationTypeAndMutationPositionAndMutationGenes_analysisID_dataStage01ResequencingEndpointLineages( analysis_id_I, column_name_I = 'analysis_id', aggregate_function_I='count', aggregate_label_I='count_1', query_I=query_I, output_O='listDict', dictColumn_I=None); #add in the mutationID genomediff = genome_diff(); for d in data_O: d['mutation_id'] = genomediff._make_mutationID( d['mutation_genes'],d['mutation_type'],d['mutation_position'] ); if d['mutation_genes']: d['mutation_genes']=";".join([x for x in d['mutation_genes'] if x is not None]); else: d['mutation_genes']=d['mutation_genes']; # make the tile objects #data1 = filter menu and table data1_keys = ['analysis_id', 'mutation_type', 'mutation_position', 'mutation_genes', 'count_1', 'mutation_id' ]; data1_nestkeys = ['mutation_id']; data1_keymap = { 'xdata':'mutation_id', 'ydata':'count_1', 'serieslabel':'analysis_id', 'featureslabel':'mutation_id', 'tooltiplabel':'mutation_id', }; nsvgtable = ddt_container_filterMenuAndChart2dAndTable(); nsvgtable.make_filterMenuAndChart2dAndTable( data_filtermenu=data_O, data_filtermenu_keys=data1_keys, data_filtermenu_nestkeys=data1_nestkeys, data_filtermenu_keymap=data1_keymap, data_svg_keys=None, data_svg_nestkeys=None, data_svg_keymap=None, data_table_keys=None, data_table_nestkeys=None, data_table_keymap=None, data_svg=None, data_table=None, svgtype='verticalbarschart2d_01', tabletype='responsivetable_01', svgx1axislabel='', svgy1axislabel='', tablekeymap = [data1_keymap], svgkeymap = [data1_keymap], formtile2datamap=[0], tabletile2datamap=[0], svgtile2datamap=[0], #calculated on the fly svgfilters=None, svgtileheader='Mutation counts', tablefilters=None, tableheaders=None ); if data_dir_I=='tmp': filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js' elif data_dir_I=='data_json': data_json_O = nsvgtable.get_allObjects_js(); return data_json_O; with open(filename_str,'w') as file: file.write(nsvgtable.get_allObjects());
def export_dataStage01RNASequencingGeneExpDiffFpkmTracking_js( self,analysis_ids_I,gene_short_names_I=[],query_I={},data_dir_I='tmp'): '''Export data as a bullet chart INPUT: OUTPUT: ''' data_O = []; # get fpkm tracking data data_O = self.get_rows_analysisIDAndGeneShortNames_dataStage01RNASequencingGeneExpDiffFpkmTracking( analysis_ids_I, gene_short_names_I, query_I=query_I, output_O='listDict', dictColumn_I=None); # make the data parameters data1_keys = [ 'analysis_id', 'experiment_id', 'sample_name_abbreviation', #'gene_id', 'gene_short_name', ]; data1_nestkeys = ['gene_short_name']; data1_keymap = { 'ydata':'FPKM', 'ydatamean':'FPKM', 'ydatalb':'FPKM_conf_lo', 'ydataub':'FPKM_conf_hi', 'xdata':'gene_short_name', 'serieslabel':'sample_name_abbreviation', 'featureslabel':'sample_name_abbreviation', 'tooltiplabel':'gene_short_name'}; nsvgtable = ddt_container_filterMenuAndChart2dAndTable(); nsvgtable.make_filterMenuAndChart2dAndTable( data_filtermenu=data_O, data_filtermenu_keys=data1_keys, data_filtermenu_nestkeys=data1_nestkeys, data_filtermenu_keymap=data1_keymap, data_svg_keys=None, data_svg_nestkeys=None, data_svg_keymap=None, data_table_keys=None, data_table_nestkeys=None, data_table_keymap=None, data_svg=None, data_table=None, svgtype='boxandwhiskersplot2d_01', tabletype='responsivetable_01', svgx1axislabel='', svgy1axislabel='FPKM', tablekeymap = [data1_keymap], svgkeymap = [data1_keymap], formtile2datamap=[0], tabletile2datamap=[0], svgtile2datamap=[0], svgfilters=None, svgtileheader='Cuffdiff FPKM', tablefilters=None, tableheaders=None, svgparameters_I={"svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 }, "svgwidth":500,"svgheight":350,} ); if data_dir_I=='tmp': filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js' elif data_dir_I=='data_json': data_json_O = nsvgtable.get_allObjects_js(); return data_json_O; with open(filename_str,'w') as file: file.write(nsvgtable.get_allObjects());
def export_dataStage01RNASequencingGeneExpDiff_count_js(self, query_I={'where':[ {"table_name":'data_stage01_rnasequencing_geneExpDiff', 'column_name':'experiment_id_1', 'value':'ALEsKOs01', 'operator':'LIKE', 'connector':'AND' }, ], }, data_dir_I='tmp'): '''Export data for a count of differentially expressed genes ''' # get the analysis information query = {} tables = ['data_stage01_rnasequencing_geneExpDiff']; query['select'] = [ {"table_name":tables[0], "column_name":'experiment_id_1', },{"table_name":tables[0], "column_name":'experiment_id_2', },{"table_name":tables[0], "column_name":'sample_name_abbreviation_1', },{"table_name":tables[0], "column_name":'sample_name_abbreviation_2', }, ]; query['group_by'] = [ {"table_name":tables[0], "column_name":'experiment_id_1', },{"table_name":tables[0], "column_name":'experiment_id_2', },{"table_name":tables[0], "column_name":'sample_name_abbreviation_1', },{"table_name":tables[0], "column_name":'sample_name_abbreviation_2', }, ]; query['order_by'] = [ {"table_name":tables[0], "column_name":'experiment_id_1', "order":'ASC', },{"table_name":tables[0], "column_name":'experiment_id_2', "order":'ASC', },{"table_name":tables[0], "column_name":'sample_name_abbreviation_1', "order":'ASC', },{"table_name":tables[0], "column_name":'sample_name_abbreviation_2', "order":'ASC', }, ]; query['where'] = [ {"table_name":tables[0], 'column_name':'significant', 'value':'yes', 'operator':'LIKE', 'connector':'AND' }, ]; #additional query blocks for k,v in query_I.items(): if not k in query.keys(): query[k] = []; for r in v: query[k].append(r); data_O = []; data_O = self.getAggregateFunction_rows_dataStage01RNASequencingGeneExpDiff( column_name_I = 'gene', aggregate_function_I='count', aggregate_label_I='count_1', query_I=query, output_O='listDict', dictColumn_I=None); # add in new column for d in data_O: d['condition'] = ('%s_%s'%(d['sample_name_abbreviation_1'],d['sample_name_abbreviation_2'])) # make the data parameters data1_keys = list(data_O[0].keys()); data1_nestkeys = ['sample_name_abbreviation_1']; data1_keymap = {'ydata':'count_1', 'xdata':'condition', 'serieslabel':'sample_name_abbreviation_2', 'featureslabel':'condition', 'tooltiplabel':'condition'}; # make the data object nsvgtable = ddt_container_filterMenuAndChart2dAndTable(); nsvgtable.make_filterMenuAndChart2dAndTable( data_filtermenu=data_O, data_filtermenu_keys=data1_keys, data_filtermenu_nestkeys=data1_nestkeys, data_filtermenu_keymap=data1_keymap, data_svg_keys=None, data_svg_nestkeys=None, data_svg_keymap=None, data_table_keys=None, data_table_nestkeys=None, data_table_keymap=None, data_svg=None, data_table=None, svgtype='verticalbarschart2d_01', tabletype='responsivetable_01', svgx1axislabel='Condition', svgy1axislabel='Count', tablekeymap = [data1_keymap], svgkeymap = [data1_keymap], formtile2datamap=[0], tabletile2datamap=[0], svgtile2datamap=[0], svgfilters=None, svgtileheader='Significant DE count', tablefilters=None, tableheaders=None ); if data_dir_I=='tmp': filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js' elif data_dir_I=='data_json': data_json_O = nsvgtable.get_allObjects_js(); return data_json_O; with open(filename_str,'w') as file: file.write(nsvgtable.get_allObjects());