def plot_averageSpectrumNormSum(self,experiment_id_I, time_points_I = None, sample_name_abbreviations_I = None, met_ids_I = None, scan_types_I = None): '''calculate the average normalized intensity for all samples and scan types''' '''Assumptions: only a single fragment:spectrum is used_ per sample name abbreviation, time-point, replicate, scan_type (i.e. there are no multiple dilutions of the same precursor:spectrum that are used_) ''' mids = mass_isotopomer_distributions(); print('plot_averagesNormSum...') plot = matplot(); # get time points if time_points_I: time_points = time_points_I; else: time_points = []; time_points = self.get_timePoint_experimentID_dataStage01AveragesNormSum(experiment_id_I); for tp in time_points: print('Plotting product and precursor for time-point ' + str(tp)); # get sample names and sample name abbreviations if sample_name_abbreviations_I: sample_abbreviations = sample_name_abbreviations_I; sample_types_lst = ['Unknown' for x in sample_abbreviations]; else: sample_abbreviations = []; sample_types = ['Unknown']; sample_types_lst = []; for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01AveragesNormSum(experiment_id_I,st,tp); sample_abbreviations.extend(sample_abbreviations_tmp); sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]); for sna_cnt,sna in enumerate(sample_abbreviations): print('Plotting product and precursor for sample name abbreviation ' + sna); # get the scan_types if scan_types_I: scan_types = []; scan_types_tmp = []; scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); scan_types = [st for st in scan_types_tmp if st in scan_types_I]; else: scan_types = []; scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); for scan_type in scan_types: print('Plotting product and precursor for scan type ' + scan_type) # met_ids if not met_ids_I: met_ids = []; met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01AveragesNormSum( \ experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type); else: met_ids = met_ids_I; if not(met_ids): continue #no component information was found for met in met_ids: print('Plotting product and precursor for metabolite ' + met); # fragments fragment_formulas = []; fragment_formulas = self.get_fragmentFormula_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanTypeAndMetID_dataStage01AveragesNormSum( \ experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met); for frag in fragment_formulas: print('Plotting product and precursor for fragment ' + frag); # data data_mat = []; data_mat_cv = []; data_masses = []; data_mat,data_mat_cv,data_masses = self.get_spectrum_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanTypeAndMetIDAndFragmentFormula_dataStage01AveragesNormSum( \ experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met,frag); data_stdev = []; for i,d in enumerate(data_mat): stdev = 0.0; stderr = 0.0; if data_mat_cv[i]: stdev = data_mat[i]*data_mat_cv[i]/100; data_stdev.append(stdev); title = sna+'_'+met+'_'+frag; plot.barPlot(title,data_masses,'intensity','m/z',data_mat,var_I=None,se_I=data_stdev,add_labels_I=True)
def execute_normalizeSpectrumFromReference_v1(self,experiment_id_I,sample_name_abbreviations_I = None,use_mrm_ref = True): # 1. import used peak spectrum to normalized table after multiplying by measured # scaling factor calculated from used MRM spectrum # 2. be sure that the MRMs in the normalized table have been finalized '''NOTES: Broken for the following reason: cannot follow the forloop pattern used in buildSpectrumFromPeakData (i.e. starting with sample name) must use the forloop pattern used in updateNormalizedSpectrum (i.e. time-point to dilutions to sample name abbreviations to scan types to mets) buildSpectrumFromPeakData and updatePeakSpectrum methods process one product:spectrum from a single precursor at a time; each precursor:product:spectrum is associated with only one sample name However, because the entire range of precursor:product:spectrum for a given met can encompass multiple dilutions and therefore different sample names, a more generic approach must be used Please use current version''' mids = mass_isotopomer_distributions(); # extract out the peakSpectrum # get sample name for the experiment print('execute_normalizeSpectrumFromReference...') if sample_name_abbreviations_I: sample_names = []; sample_types = ['Unknown','QC']; sample_types_lst = []; for sna in sample_name_abbreviations_I: for st in sample_types: sample_names_tmp = []; sample_names_tmp = self.get_sampleNames_experimentIDAndSampleTypeAndSampleNameAbbreviation_peakSpectrum(experiment_id_I,st,sna); sample_names.extend(sample_names_tmp); sample_types_lst.extend([st for i in range(len(sample_names_tmp))]); else: sample_names = []; sample_types = ['Unknown','QC']; sample_types_lst = []; for st in sample_types: sample_names_tmp = []; sample_names_tmp = self.get_sampleNames_experimentIDAndSampleType_peakSpectrum(experiment_id_I,st); sample_names.extend(sample_names_tmp); sample_types_lst.extend([st for i in range(len(sample_names_tmp))]); for sn_cnt,sn in enumerate(sample_names): print('normalizing peak spectrum for sample ' + sn); # get other information about the sample for later use sample_name_abbreviation,time_point,dilution,replicate_numbers = None,None,None,None; sample_name_abbreviation,time_point,dilution,replicate_numbers = self.get_sampleNameAbbreviationsAndOther_experimentIDAndSampleName_peakSpectrum(experiment_id_I,sn); # get met_id and precursor_formula for each sample scan_type = []; scan_type = self.get_scanType_experimentIDAndSampleName_peakSpectrum(experiment_id_I,sn); for scantype in scan_type: print('normalizing spectrum for scan type ' + scantype); # get met_id med_id = []; met_id = self.get_metID_experimentIDAndSampleNameAndScanType_peakSpectrum(experiment_id_I,sn,scantype); for met in met_id: print('normalizing peak spectrum for met_id ' + met); # get precursor formula and mass precursor_formula, precursor_mass = [], []; precursor_formula, precursor_mass = self.get_precursorFormulaAndMass_experimentIDAndSampleNameAndMetIDAndScanType_peakSpectrum(experiment_id_I,sn,met,scantype); peak_data_all = {}; scaling_factors_all = {}; for precursor_cnt, precursor in enumerate(precursor_formula): peak_data_all[precursor] = None; scaling_factors_all[precursor] = None; print('normalizing peak spectrum for precursor ' + precursor); precursor_str = re.sub('[+-]', '', precursor); # get all product fragments for the met_id/precursor product_formulas = []; product_formulas = self.get_productFormulas_experimentIDAndSampleNameAndMetIDAndPrecursorFormulaAndScanType_peakSpectrum(experiment_id_I,sn,met,precursor,scantype); # get the m+0 precursor_formula precursor_formula_monoisotopic = self.get_precursorFormula_metID(met,'-','tuning'); precursor_monoisotopic_str = re.sub('[+-]', '', precursor_formula_monoisotopic); precursor_monoisotpoic_mass = int(numpy.round(Formula(precursor_monoisotopic_str).isotope.mass)); # get peakSpectrum data peak_data = {}; #Change to sna+rep+timepoint:peak_data = self.get_normalizedIntensity_experimentIDAndSampleNameAndMetIDAndPrecursorFormulaAndScanType_peakSpectrum(experiment_id_I,sn,met,precursor,scantype); peak_data_all[precursor] = peak_data; if scantype == 'ER': scaling_factors_all[precursor] = 1.0; # there is no need to scale ER or other precursor ion scans else: if use_mrm_ref: # get reference MRM spectrum scaling factor for the sample #scaling_factor,scaling_factor_cv = None,None; # will need to incorporate propogation of error #scaling_factor,scaling_factor_cv = self.get_normalizedIntensity_experimentIDAndSampleAbbreviationAndTimePointAndMetIDAndFragmentFormulaAndMassAndScanType_dataStage01Averages(experiment_id_I,sample_name_abbreviation,time_point,met,precursor_formula_monoisotopic,precursor_mass[precursor_cnt],'MRM'); scaling_factor = None; # does not require the propogation of error scaling_factor = self.get_normalizedIntensity_experimentIDAndSampleAbbreviationAndTimePointAndReplicateNumberAndMetIDAndFragmentFormulaAndMassAndScanType_dataStage01Normalized(experiment_id_I,sample_name_abbreviation,time_point,replicate_numbers,met,precursor_formula_monoisotopic,precursor_mass[precursor_cnt],'MRM'); if scaling_factor: scaling_factors_all[precursor] = scaling_factor; else: scaling_factors_all[precursor] = 0.0; ## substitute with reference spectrum #refspec = mids.report_fragmentSpectrum_normMax([precursor_formula_monoisotopic],True); #scaling_factor = refspec[precursor_formula_monoisotopic][precursor_mass[precursor_cnt]]; #scaling_factors_all[precursor] = scaling_factor; else: # get reference ER spectrum scaling factor for the sample scaling_factor = None; scaling_factor = self.get_normalizedIntensity_experimentIDAndSampleAbbreviationAndTimePointAndReplicateNumberAndMetIDAndPrecursorFormulaAndMassAndScanType_peakSpectrum(experiment_id_I,sample_name_abbreviation,time_point,replicate_numbers,met,precursor_formula_monoisotopic,precursor_mass[precursor_cnt],'ER'); if scaling_factor: scaling_factors_all[precursor] = scaling_factor; else: scaling_factors_all[precursor] = 0.0; ## substitute with reference spectrum #refspec = mids.report_fragmentSpectrum_normMax([precursor_formula_monoisotopic],True); #scaling_factor = refspec[precursor_formula_monoisotopic][precursor_mass[precursor_cnt]]; #scaling_factors_all[precursor] = scaling_factor; # normalize spectrum to reference MRM for each precursor (m+0,m+1,...) peakSpectrum_normalized = mids.normalize_peakSpectrum_normMax(peak_data_all,scaling_factors_all); peakSpectrum_stats,peakSpectrum_theoretical = mids.compare_peakSpectrum_normMax([peakSpectrum_normalized],True); # update data_stage01_isotopomer_peakSpectrum for frag,spec in peakSpectrum_theoretical.items(): if spec: product_str = re.sub('[+-]', '', frag); product_mass = Formula(product_str).isotope.mass; for k,v in peakSpectrum_theoretical[frag].items(): if k in peakSpectrum_normalized[frag]: row = None; row = data_stage01_isotopomer_normalized(experiment_id_I,sn,sample_name_abbreviation,sample_types_lst[sn_cnt],time_point,dilution,replicate_numbers, met,frag,int(numpy.round(k)), None,'cps',None,'cps', peakSpectrum_normalized[frag][k],'normMax', v,peakSpectrum_stats[frag][k]['absDev'],scantype,True); self.session.add(row); self.session.commit();
def export_dataStage01IsotopomerAveragesNormSum_js(self,experiment_id_I, time_points_I = None, sample_name_abbreviations_I = None, met_ids_I = None, scan_types_I = None,data_dir_I="tmp", single_plot_I = True): '''Export data_stage01_isotopomer_averagesNormSum to js file''' mids = mass_isotopomer_distributions(); print('plotting averagesNormSum...') data_O = []; data_dict_O = {}; # get time points if time_points_I: time_points = time_points_I; else: time_points = []; time_points = self.get_timePoint_experimentID_dataStage01AveragesNormSum(experiment_id_I); for tp in time_points: print('Plotting product and precursor for time-point ' + str(tp)); # get sample names and sample name abbreviations sample_abbreviations = []; sample_types = ['Unknown']; sample_types_lst = []; for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01AveragesNormSum(experiment_id_I,st,tp); sample_abbreviations.extend(sample_abbreviations_tmp); sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]); if sample_name_abbreviations_I: sample_abbreviations = [sna for sna in sample_abbreviations if sna in sample_name_abbreviations_I]; sample_types_lst = ['Unknown' for x in sample_abbreviations]; for sna_cnt,sna in enumerate(sample_abbreviations): print('Plotting product and precursor for sample name abbreviation ' + sna); data_dict_O[sna] = []; # get the scan_types if scan_types_I: scan_types = []; scan_types_tmp = []; scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); scan_types = [st for st in scan_types_tmp if st in scan_types_I]; else: scan_types = []; scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); for scan_type in scan_types: print('Plotting product and precursor for scan type ' + scan_type) # met_ids if not met_ids_I: met_ids = []; met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01AveragesNormSum( \ experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type); else: met_ids = met_ids_I; if not(met_ids): continue #no component information was found for met in met_ids: print('Plotting product and precursor for metabolite ' + met); # fragments fragment_formulas = []; fragment_formulas = self.get_fragmentFormula_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanTypeAndMetID_dataStage01AveragesNormSum( \ experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met); for frag in fragment_formulas: print('Plotting product and precursor for fragment ' + frag); # data data_mat = []; data_mat_cv = []; data_masses = []; data_mat,data_mat_cv,data_masses = self.get_spectrum_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanTypeAndMetIDAndFragmentFormula_dataStage01AveragesNormSum( \ experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met,frag); for i,d in enumerate(data_mat): stdev = 0.0; stderr = 0.0; lb = None; ub = None; if data_mat_cv[i]: stdev = data_mat[i]*data_mat_cv[i]/100; lb = data_mat[i]-stdev; ub = data_mat[i]+stdev; fragment_id = mids.make_fragmentID(met,frag,data_masses[i]); tmp = { 'experiment_id':experiment_id_I, 'sample_name_abbreviation':sna, 'sample_type':sample_types_lst[sna_cnt], 'time_point':tp, 'met_id':met, 'fragment_formula':frag, 'fragment_mass':data_masses[i], 'fragment_id':fragment_id, 'intensity_normalized_average':d, 'intensity_normalized_cv':data_mat_cv[i], 'intensity_normalized_lb':lb, 'intensity_normalized_ub':ub, 'intensity_normalized_units':"normSum", 'scan_type':scan_type, 'used_':True, 'comment_':None} data_O.append(tmp); data_dict_O[sna].append(tmp); print('tabulating averagesNormSum...') data_table_O = []; # get time points if time_points_I: time_points = time_points_I; else: time_points = []; time_points = self.get_timePoint_experimentID_dataStage01AveragesNormSum(experiment_id_I); for tp in time_points: print('Tabulating product and precursor for time-point ' + str(tp)); # get sample names and sample name abbreviations sample_abbreviations = []; sample_types = ['Unknown']; sample_types_lst = []; for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01AveragesNormSum(experiment_id_I,st,tp); sample_abbreviations.extend(sample_abbreviations_tmp); sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]); if sample_name_abbreviations_I: sample_abbreviations = [sna for sna in sample_abbreviations if sna in sample_name_abbreviations_I]; sample_types_lst = ['Unknown' for x in sample_abbreviations]; for sna_cnt,sna in enumerate(sample_abbreviations): print('Tabulating product and precursor for sample name abbreviation ' + sna); # get the scan_types if scan_types_I: scan_types = []; scan_types_tmp = []; scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); scan_types = [st for st in scan_types_tmp if st in scan_types_I]; else: scan_types = []; scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); for scan_type in scan_types: print('Tabulating product and precursor for scan type ' + scan_type) # met_ids if not met_ids_I: met_ids = []; met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01AveragesNormSum( \ experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type); else: met_ids = met_ids_I; if not(met_ids): continue #no component information was found for met in met_ids: print('Tabulating product and precursor for metabolite ' + met); # get rows rows = []; rows = self.get_rows_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanTypeAndMetID_dataStage01AveragesNormSum( \ experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met); if not rows: continue; for d in rows: d['fragment_id'] = mids.make_fragmentID(d['met_id'],d['fragment_formula'],d['fragment_mass']); data_table_O.extend(rows); print('exporting averagesNormSum...'); # visualization parameters data1_keys = ['sample_name_abbreviation', 'sample_type', 'met_id', 'time_point', 'fragment_formula', 'fragment_mass', 'scan_type', 'fragment_id' ]; data1_nestkeys = [ #'fragment_id', 'fragment_mass' ]; data1_keymap = { #'xdata':'fragment_id', 'xdata':'fragment_mass', 'ydata':'intensity_normalized_average', 'tooltiplabel':'sample_name_abbreviation', 'serieslabel':'scan_type', #'featureslabel':'fragment_id', 'featureslabel':'fragment_mass', 'ydatalb':'intensity_normalized_lb', 'ydataub':'intensity_normalized_ub'}; # initialize the ddt objects dataobject_O = []; parametersobject_O = []; tile2datamap_O = {}; # make the tile parameter objects formtileparameters_O = {'tileheader':'Filter menu','tiletype':'html','tileid':"filtermenu1",'rowid':"row1",'colid':"col1", 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"}; formparameters_O = {'htmlid':'filtermenuform1',"htmltype":'form_01',"formsubmitbuttonidtext":{'id':'submit1','text':'submit'},"formresetbuttonidtext":{'id':'reset1','text':'reset'},"formupdatebuttonidtext":{'id':'update1','text':'update'}}; formtileparameters_O.update(formparameters_O); dataobject_O.append({"data":data_O,"datakeys":data1_keys,"datanestkeys":data1_nestkeys}); parametersobject_O.append(formtileparameters_O); tile2datamap_O.update({"filtermenu1":[0]}); if not single_plot_I: rowcnt = 1; colcnt = 1; for cnt,sn in enumerate(sample_abbreviations): svgtileid = "tilesvg"+str(cnt); svgid = 'svg'+str(cnt); iter=cnt+1; #start at 1 if (cnt % 2 == 0): rowcnt = rowcnt+1;#even colcnt = 1; else: colcnt = colcnt+1; # make the svg object svgparameters1_O = {"svgtype":'verticalbarschart2d_01',"svgkeymap":[data1_keymap], #'svgid':'svg1', 'svgid':'svg'+str(cnt), "svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 }, "svgwidth":350,"svgheight":250,"svgy1axislabel":"intensity (norm)", "svgfilters":{'sample_name_abbreviation':[sn]} }; svgtileparameters1_O = {'tileheader':'Isotopomer distribution','tiletype':'svg', 'tileid':svgtileid, 'rowid':"row"+str(rowcnt), 'colid':"col"+str(colcnt), 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-6"}; svgtileparameters1_O.update(svgparameters1_O); parametersobject_O.append(svgtileparameters1_O); tile2datamap_O.update({svgtileid:[0]}); else: cnt = 0; svgtileid = "tilesvg"+str(cnt); svgid = 'svg'+str(cnt); rowcnt = 2; colcnt = 1; # make the svg object svgparameters1_O = {"svgtype":'verticalbarschart2d_01',"svgkeymap":[data1_keymap], 'svgid':'svg'+str(cnt), "svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 }, "svgwidth":500,"svgheight":350,"svgy1axislabel":"intensity (norm)", }; svgtileparameters1_O = {'tileheader':'Isotopomer distribution','tiletype':'svg', 'tileid':svgtileid, 'rowid':"row"+str(rowcnt), 'colid':"col"+str(colcnt), 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"}; svgtileparameters1_O.update(svgparameters1_O); parametersobject_O.append(svgparameters1_O); tile2datamap_O.update({svgtileid:[0]}); # make the table object tableparameters1_O = {"tabletype":'responsivetable_01', 'tableid':'table1', "tablefilters":None, #"tableheaders":[], "tableclass":"table table-condensed table-hover", 'tableformtileid':'tile1','tableresetbuttonid':'reset1','tablesubmitbuttonid':'submit1'}; tabletileparameters1_O = {'tileheader':'Isotopomer distribution','tiletype':'table','tileid':"tabletile1", 'rowid':"row100", 'colid':"col1", 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"}; tabletileparameters1_O.update(tableparameters1_O); dataobject_O.append({"data":data_table_O,"datakeys":data1_keys,"datanestkeys":data1_nestkeys}); parametersobject_O.append(tabletileparameters1_O); tile2datamap_O.update({"tabletile1":[1]}) nsvgtable = ddt_container_filterMenuAndChart2dAndTable(); nsvgtable.make_filterMenuAndChart2dAndTable( data_filtermenu=data_O, data_filtermenu_keys=data1_keys, data_filtermenu_nestkeys=data1_nestkeys, data_filtermenu_keymap=data1_keymap, data_svg_keys=data1_keys, data_svg_nestkeys=data1_nestkeys, data_svg_keymap=data1_keymap, data_table_keys=None, data_table_nestkeys=None, data_table_keymap=None, data_svg=data_dict_O, data_table=None, svgtype='verticalbarschart2d_01', tabletype='responsivetable_01', svgx1axislabel='', svgy1axislabel='', tablekeymap = [data1_keymap], svgkeymap = [], #calculated on the fly formtile2datamap=[0], tabletile2datamap=[0], svgtile2datamap=[], #calculated on the fly svgfilters=None, svgtileheader='Isotopomer distribution', tablefilters=None, tableheaders=None ); if data_dir_I=='tmp': filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js' elif data_dir_I=='data_json': data_json_O = nsvgtable.get_allObjects_js(); return data_json_O; with open(filename_str,'w') as file: file.write(nsvgtable.get_allObjects());
def execute_updatePeakSpectrum(self,experiment_id_I,sample_name_abbreviations_I = None): '''re-calculate intensity_normalized from intensity_corrected and used''' mids = mass_isotopomer_distributions(); # extract out the peakSpectrum dataListUpdated = []; # get sample names for the experiment print('execute_updatePeakSpectrum...') if sample_name_abbreviations_I: sample_names = []; sample_types = ['Unknown','QC']; sample_types_lst = []; for sna in sample_name_abbreviations_I: for st in sample_types: sample_names_tmp = []; sample_names_tmp = self.get_sampleNames_experimentIDAndSampleTypeAndSampleNameAbbreviation_peakSpectrum(experiment_id_I,st,sna); sample_names.extend(sample_names_tmp); sample_types_lst.extend([st for i in range(len(sample_names_tmp))]); else: sample_names = []; sample_types = ['Unknown','QC']; sample_types_lst = []; for st in sample_types: sample_names_tmp = []; sample_names_tmp = self.get_sampleNames_experimentIDAndSampleType_peakSpectrum(experiment_id_I,st); sample_names.extend(sample_names_tmp); sample_types_lst.extend([st for i in range(len(sample_names_tmp))]); # create database table for sn_cnt,sn in enumerate(sample_names): print('updating peak spectrum for sample ' + sn); # get other information about the sample for later use sample_name_abbreviation,time_point,replicate_numbers = None,None,None; sample_name_abbreviation,time_point,replicate_numbers = self.get_sampleNameAbbreviationsAndTimePointAndReplicateNumber_experimentIDAndSampleName_peakSpectrum(experiment_id_I,sn); # get met_id and precursor_formula for each sample scan_type = []; scan_type = self.get_scanType_experimentIDAndSampleName_peakSpectrum(experiment_id_I,sn); for scantype in scan_type: print('building spectrum for scan type ' + scantype); # get met_id and precursor formula for each sample met_id, precursor_formula = [], []; met_id, precursor_formula = self.get_metIDAndPrecursorFormula_experimentIDAndSampleNameAndScanType_peakSpectrum(experiment_id_I,sn,scantype); for precursor_cnt, precursor in enumerate(precursor_formula): print('updating peak spectrum for met_id/precursor ' + met_id[precursor_cnt] + '/' + precursor); precursor_str = re.sub('[+-]', '', precursor); precursor_mass = Formula(precursor_str).isotope.mass # get all product fragments for the met_id/precursor precursor_formulas_monoisotopic, product_formulas = [], []; precursor_formulas_monoisotopic, product_formulas = self.get_precursorAndProductFormulas_metID(met_id[precursor_cnt],'-','tuning'); product_formulas.append(precursor_formulas_monoisotopic[0]); # add precursor to list of fragments # get peak data for the sample/met_id/precursor_formula peak_data = []; peak_data = self.get_data_experimentIDAndSampleNameAndMetIDAndPrecursorFormulaAndScanType_peakSpectrum(experiment_id_I,sn,met_id[precursor_cnt],precursor,scantype); peakSpectrum_corrected, peakSpectrum_normalized = mids.extract_peakList_normMax(\ peak_data, product_formulas,True); peakSpectrum_stats,peakSpectrum_theoretical = mids.compare_peakSpectrum_normMax([peakSpectrum_normalized],True); # update data_stage01_isotopomer_peakSpectrum for frag,spec in peakSpectrum_theoretical.items(): if spec: product_str = re.sub('[+-]', '', frag); product_mass = Formula(product_str).isotope.mass; for k,v in peakSpectrum_theoretical[frag].items(): dataListUpdated.append({'experiment_id':experiment_id_I, 'sample_name':sn, 'sample_name_abbreviation':sample_name_abbreviation, 'sample_type':sample_types_lst[sn_cnt], 'time_point':time_point, 'replicate_number':replicate_numbers, 'met_id':met_id[precursor_cnt], 'precursor_formula':precursor, 'precursor_mass':int(numpy.round(precursor_mass)), 'product_formula':frag, 'product_mass':int(numpy.round(k)), 'intensity_corrected':peakSpectrum_corrected[frag][k], 'intensity_corrected_units':'cps', 'intensity_normalized':peakSpectrum_normalized[frag][k], 'intensity_normalized_units':'normMax', 'intensity_theoretical':v, 'abs_devFromTheoretical':peakSpectrum_stats[frag][k]['absDev'], 'scan_type':scantype}); self.update_data_stage01_isotopomer_peakSpectrum(dataListUpdated);
def execute_buildSpectrumFromPeakData(self,experiment_id_I,ms_methodtype_I='isotopomer_13C',sample_name_abbreviations_I = None,met_ids_I = None): '''Build spectrum from raw peak data''' '''Assumptions: Only 1 precursur:spectrum per sample name and only 1 precursor:spectrum per dilution (i.e. the best/most representative precursor:spectrum was chose from the available EPI scans and dilutions of that particular precursor) ''' mids = mass_isotopomer_distributions(); # extract out the peakSpectrum # get sample names for the experiment print('execute_buildSpectrumFromPeakData...') if sample_name_abbreviations_I: sample_names = []; sample_types = ['Unknown','QC']; sample_types_lst = []; for sna in sample_name_abbreviations_I: for st in sample_types: sample_names_tmp = []; sample_names_tmp = self.get_sampleNames_experimentIDAndSampleTypeAndSampleNameAbbreviation_peakData(experiment_id_I,st,sna); sample_names.extend(sample_names_tmp); sample_types_lst.extend([st for i in range(len(sample_names_tmp))]); else: sample_names = []; sample_types = ['Unknown','QC']; sample_types_lst = []; for st in sample_types: sample_names_tmp = []; sample_names_tmp = self.get_sampleNames_experimentIDAndSampleType_peakData(experiment_id_I,st); sample_names.extend(sample_names_tmp); sample_types_lst.extend([st for i in range(len(sample_names_tmp))]); # create database table for sn_cnt,sn in enumerate(sample_names): print('building spectrum for sample ' + sn); # get other information about the sample for later use sample_name_abbreviation,time_point,replicate_numbers = None,None,None; sample_name_abbreviation,time_point,replicate_numbers = self.get_sampleNameAbbreviationsAndOther_experimentIDAndSampleName_peakData(experiment_id_I,sn); # get met_id and precursor_formula for each sample scan_type = []; scan_type = self.get_scanType_experimentIDAndSampleName_peakData(experiment_id_I,sn); for scantype in scan_type: print('building spectrum for scan type ' + scantype); # get met_id and precursor formula for each sample if met_ids_I: met_id, precursor_formula = [], []; for met in met_ids_I: met_id_tmp, precursor_formula_tmp = [], [] met_id_tmp, precursor_formula_tmp = self.get_metIDAndPrecursorFormula_experimentIDAndSampleNameAndScanTypeAndMetID_peakData(experiment_id_I,sn,scantype,met); met_id.extend(met_id_tmp); precursor_formula.extend(precursor_formula_tmp); else: met_id, precursor_formula = [], []; met_id, precursor_formula = self.get_metIDAndPrecursorFormula_experimentIDAndSampleNameAndScanType_peakData(experiment_id_I,sn,scantype); for precursor_cnt, precursor in enumerate(precursor_formula): print('building spectrum for met_id/precursor ' + met_id[precursor_cnt] + '/' + precursor); precursor_str = re.sub('[+-]', '', precursor); precursor_mass = Formula(precursor_str).isotope.mass # get all product fragments for the met_id/precursor precursor_formulas_monoisotopic, product_formulas = [], []; precursor_formulas_monoisotopic, product_formulas = self.get_precursorAndProductFormulas_metID(met_id[precursor_cnt],'-','tuning'); product_formulas.append(precursor_formulas_monoisotopic[0]); # add precursor to list of fragments # get peak data for the sample/met_id/precursor_formula peak_data = []; peak_data = self.get_data_experimentIDAndSampleNameAndMetIDAndPrecursorFormulaAndScanType_peakData(experiment_id_I,sn,met_id[precursor_cnt],precursor,scantype); peakSpectrum_measured,\ peakSpectrum_corrected, peakSpectrum_normalized = mids.extract_peakData_normMax(\ peak_data, product_formulas, 0.3, True); peakSpectrum_stats,peakSpectrum_theoretical = mids.compare_peakSpectrum_normMax([peakSpectrum_normalized],True); # update data_stage01_isotopomer_normalized for frag,spec in peakSpectrum_theoretical.items(): if spec: product_str = re.sub('[+-]', '', frag); product_mass = Formula(product_str).isotope.mass; for k,v in peakSpectrum_theoretical[frag].items(): row1 = None; row1 = data_stage01_isotopomer_peakSpectrum(experiment_id_I,sn,sample_name_abbreviation, sample_types_lst[sn_cnt],time_point,replicate_numbers, met_id[precursor_cnt],precursor,int(numpy.round(precursor_mass)), frag,int(numpy.round(k)), peakSpectrum_measured[frag][k],'cps', peakSpectrum_corrected[frag][k],'cps', peakSpectrum_normalized[frag][k],'normMax', v,peakSpectrum_stats[frag][k]['absDev'],scantype,True,None); self.session.add(row1); self.session.commit();
def export_dataStage01IsotopomerNormalized_js(self,experiment_id_I,sample_names_I=[],sample_name_abbreviations_I=[],time_points_I=[],scan_types_I=[],met_ids_I=[],data_dir_I="tmp", single_plot_I = True): """Export data_stage01_isotopomer_normalized to js file""" mids = mass_isotopomer_distributions(); # get the data data_O = []; data_dict_O = {}; sample_names_O = []; # get time points if time_points_I: time_points = time_points_I; else: time_points = self.get_timePoint_experimentID_dataStage01Normalized(experiment_id_I); for tp in time_points: print('Plotting precursor and product spectrum from isotopomer normalized for time-point ' + str(tp)); if sample_names_I: sample_abbreviations = []; sample_types = ['Unknown','QC']; sample_types_lst = []; for sn in sample_names_I: for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndSampleName_dataStage01Normalized(experiment_id_I,st,tp,sn); sample_abbreviations.extend(sample_abbreviations_tmp); sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]); elif sample_name_abbreviations_I: sample_abbreviations = []; sample_types = ['Unknown','QC']; sample_types_lst = []; for sn in sample_name_abbreviations_I: for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndSampleNameAbbreviation_dataStage01Normalized(experiment_id_I,st,tp,sn); sample_abbreviations.extend(sample_abbreviations_tmp); sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]); # query sample types from sample name abbreviations and time-point from data_stage01_isotopomer_normalized else: # get sample names and sample name abbreviations sample_abbreviations = []; sample_types = ['Unknown','QC']; sample_types_lst = []; for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01Normalized(experiment_id_I,st,tp); sample_abbreviations.extend(sample_abbreviations_tmp); sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]); for sna_cnt,sna in enumerate(sample_abbreviations): print('Plotting precursor and product spectrum from isotopomer normalized for sample name abbreviation ' + sna); # get the scan_types if scan_types_I: scan_types = []; scan_types_tmp = []; scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01Normalized(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); scan_types = [st for st in scan_types_tmp if st in scan_types_I]; else: scan_types = []; scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01Normalized(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); for scan_type in scan_types: print('Plotting precursor and product spectrum for scan type ' + scan_type) # met_ids if not met_ids_I: met_ids = []; met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01Normalized( \ experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type); else: met_ids = met_ids_I; if not(met_ids): continue #no component information was found for met in met_ids: print('Plotting precursor and product spectrum for metabolite ' + met); if sample_names_I: sample_names = sample_names_I; else: sample_names,replicate_numbers,sample_types = [],[],[]; sample_names,replicate_numbers,sample_types = self.get_sampleNamesAndReplicateNumbersAndSampleTypes_experimentIDAndSampleNameAbbreviationAndMetIDAndTimePointAndScanType_dataStage01Normalized( \ experiment_id_I,sna,met,tp,scan_type); if not(replicate_numbers): continue; #no replicates found for rep_cnt,rep in enumerate(replicate_numbers): #if not(sample_names): continue; #no replicates found #for sn_cnt,sn in enumerate(sample_names): print('Plotting precursor and product spectrum for replicate_number ' + str(rep)); #print('Plotting precursor and product spectrum for sample_name ' + sn); #sample_names_O.append(sn); #get data peakData_I = {}; peakData_I = self.get_dataNormalized_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetIDAndReplicateNumber_dataStage01Normalized( \ experiment_id_I,sna,tp,scan_type,met,rep); #peakData_I = self.get_dataNormalized_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetIDAndSampleName_dataStage01Normalized( \ # experiment_id_I,sna,tp,scan_type,met,sn); if peakData_I: data_sample_names_tmp=[]; fragment_formulas = list(peakData_I.keys()); peakSpectrum_corrected, peakSpectrum_normalized = mids.extract_peakList_normMax(\ peakData_I, fragment_formulas, True); for fragment_formula in fragment_formulas: for fragment_mass,intensity_normalized in peakSpectrum_normalized[fragment_formula].items(): sample_name = sna + "_" + str(rep); sample_names_O.append(sample_name); fragment_id = mids.make_fragmentID(met,fragment_formula,fragment_mass); intensity = 0.0; if intensity_normalized: intensity = intensity_normalized; data_tmp = { 'experiment_id':experiment_id_I, 'sample_name':sample_name, #'sample_name':sn, 'sample_name_abbreviation':sna, 'sample_type':sample_types_lst[sna_cnt], 'time_point':tp, #'dilution':dil, 'replicate_number':rep, 'met_id':met, 'fragment_formula':fragment_formula, 'fragment_mass':fragment_mass, 'intensity_normalized':intensity, 'intensity_normalized_units':"normMax", 'scan_type':scan_type, 'fragment_id':fragment_id}; data_O.append(data_tmp); if not sample_name in data_dict_O.keys(): data_dict_O[sample_name] = []; data_dict_O[sample_name].append(data_tmp); else: data_dict_O[sample_name].append(data_tmp); # record the unique sample names: sample_names_unique = list(set(sample_names_O)); sample_names_unique.sort(); # get the table data data_table_O = []; if time_points_I: time_points = time_points_I; else: time_points = self.get_timePoint_experimentID_dataStage01Normalized(experiment_id_I); for tp in time_points: print('Tabulating precursor and product spectrum from isotopomer normalized for time-point ' + str(tp)); dataListUpdated = []; # get dilutions dilutions = []; dilutions = self.get_sampleDilution_experimentIDAndTimePoint_dataStage01Normalized(experiment_id_I,tp); for dil in dilutions: print('Tabulating precursor and product spectrum from isotopomer normalized for dilution ' + str(dil)); if sample_names_I: sample_abbreviations = []; sample_types = ['Unknown','QC']; for sn in sample_names_I: for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndDilutionAndSampleName_dataStage01Normalized(experiment_id_I,st,tp,dil,sn); sample_abbreviations.extend(sample_abbreviations_tmp); elif sample_name_abbreviations_I: sample_abbreviations = sample_name_abbreviations_I; else: # get sample names and sample name abbreviations sample_abbreviations = []; sample_types = ['Unknown','QC']; for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndDilution_dataStage01Normalized(experiment_id_I,st,tp,dil); sample_abbreviations.extend(sample_abbreviations_tmp); for sna_cnt,sna in enumerate(sample_abbreviations): print('Tabulating precursor and product spectrum from isotopomer normalized for sample name abbreviation ' + sna); # get the scan_types if scan_types_I: scan_types = scan_types_I; else: scan_types = []; scan_types = self.get_scanTypes_experimentIDAndTimePointAndDilutionAndSampleAbbreviations_dataStage01Normalized(experiment_id_I,tp,dil,sna); for scan_type in scan_types: print('Tabulting precursor and product spectrum for scan type ' + scan_type) # met_ids if not met_ids_I: met_ids = []; met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndDilutionAndScanType_dataStage01Normalized( \ experiment_id_I,sna,tp,dil,scan_type); else: met_ids = met_ids_I; if not(met_ids): continue #no component information was found for met in met_ids: # get the data data = []; data = self.get_rows_experimentIDAndSampleAbbreviationAndTimePointAndDilutionAndScanTypeAndMetID_dataStage01Normalized( \ experiment_id_I,sna,tp,dil,scan_type,met); for d in data: d['sample_name'] = d['sample_name_abbreviation']+"_"+str(d['replicate_number']); d['fragment_id'] = mids.make_fragmentID(d['met_id'],d['fragment_formula'],d['fragment_mass']); data_table_O.extend(data); # visualization parameters data1_keys = ['sample_name','sample_name_abbreviation', 'sample_type', 'met_id','time_point','fragment_formula','fragment_mass','scan_type','fragment_id']; data1_nestkeys = [ #'fragment_id', 'fragment_mass' ]; data1_keymap = { #'xdata':'fragment_id', 'xdata':'fragment_mass', 'ydata':'intensity_normalized', 'serieslabel':'scan_type', #'serieslabel':'sample_name', #single plot #'featureslabel':'fragment_id', 'featureslabel':'fragment_mass', 'tooltipdata':'sample_name_abbreviation', 'ydatalb':None, 'ydataub':None}; # initialize the ddt objects dataobject_O = []; parametersobject_O = []; tile2datamap_O = {}; # make the tile parameter objects formtileparameters_O = {'tileheader':'Filter menu','tiletype':'html','tileid':"filtermenu1",'rowid':"row1",'colid':"col1", 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"}; formparameters_O = {'htmlid':'filtermenuform1',"htmltype":'form_01',"formsubmitbuttonidtext":{'id':'submit1','text':'submit'},"formresetbuttonidtext":{'id':'reset1','text':'reset'},"formupdatebuttonidtext":{'id':'update1','text':'update'}}; formtileparameters_O.update(formparameters_O); dataobject_O.append({"data":data_O,"datakeys":data1_keys,"datanestkeys":data1_nestkeys}); parametersobject_O.append(formtileparameters_O); tile2datamap_O.update({"filtermenu1":[0]}); if not single_plot_I: rowcnt = 1; colcnt = 1; cnt = 0; for sn in sample_names_unique: svgtileid = "tilesvg"+str(cnt); svgid = 'svg'+str(cnt); iter=cnt+1; #start at 1 if (cnt % 2 == 0): rowcnt = rowcnt+1;#even colcnt = 1; else: colcnt = colcnt+1; # make the svg object svgparameters1_O = {"svgtype":'verticalbarschart2d_01',"svgkeymap":[data1_keymap], #'svgid':'svg1', 'svgid':'svg'+str(cnt), "svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 }, "svgwidth":350,"svgheight":250,"svgy1axislabel":"intensity (norm)", }; svgtileparameters1_O = {'tileheader':sn,'tiletype':'svg', #'tileid':"tile2", 'tileid':svgtileid, #'rowid':"row1", 'rowid':"row"+str(rowcnt), #'colid':"col1", 'colid':"col"+str(colcnt), 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-6"}; svgtileparameters1_O.update(svgparameters1_O); parametersobject_O.append(svgtileparameters1_O); tile2datamap_O.update({svgtileid:[iter]}); dataobject_O.append({"data":data_dict_O[sn],"datakeys":data1_keys,"datanestkeys":data1_nestkeys}); cnt+=1; else: cnt = 0; svgtileid = "tilesvg"+str(cnt); svgid = 'svg'+str(cnt); rowcnt = 2; colcnt = 1; # make the svg object svgparameters1_O = {"svgtype":'verticalbarschart2d_01',"svgkeymap":[data1_keymap], #'svgid':'svg1', 'svgid':'svg'+str(cnt), "svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 }, "svgwidth":500,"svgheight":350,"svgy1axislabel":"intensity (norm)", }; svgtileparameters1_O = {'tileheader':'Isotopomer distribution','tiletype':'svg', #'tileid':"tile2", 'tileid':svgtileid, #'rowid':"row1", #'colid':"col1", 'rowid':"row"+str(rowcnt), 'colid':"col"+str(colcnt), 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"}; svgtileparameters1_O.update(svgparameters1_O); parametersobject_O.append(svgparameters1_O); tile2datamap_O.update({svgtileid:[0]}); iter+=1; # make the table object tableparameters1_O = {"tabletype":'responsivetable_01', 'tableid':'table1', "tablefilters":None, #"tableheaders":[], "tableclass":"table table-condensed table-hover", 'tableformtileid':'tile1','tableresetbuttonid':'reset1','tablesubmitbuttonid':'submit1'}; tabletileparameters1_O = {'tileheader':'Isotopomer distribution','tiletype':'table','tileid':"tabletile1", 'rowid':"row"+str(rowcnt+1), 'colid':"col1", 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"}; tabletileparameters1_O.update(tableparameters1_O); #dataobject_O.append({"data":data_table_O,"datakeys":data1_keys,"datanestkeys":data1_nestkeys}); parametersobject_O.append(tabletileparameters1_O); tile2datamap_O.update({"tabletile1":[0]}) #filtermenunsvgtable = ddt_container_filterMenuNSvgTable(); #filtermenunsvgtable.make_filterMenuNSvgTable( # data_filtermenu=data_O, # data_filtermenu_keys=[data1_keymap], # data_filtermenu_nestkeys=data1_nestkeys, # data_filtermenu_keymap=[data1_keymap], # data_svg_keys=[data1_keymap], # data_svg_nestkeys=data1_nestkeys, # data_svg_keymap=[data1_keymap], # data_table_keys=data1_keys, # data_table_nestkeys=data1_nestkeys, # data_table_keymap=[data1_keymap], # data_svg=data_dict_O, # data_table=data_table_O, # svgtype='verticalbarschart2d_01', # tabletype='responsivetable_01', # svgx1axislabel=None, # svgy1axislabel="intensity (norm)", # single_plot_I=True, # formtile2datamap=[0], # tabletile2datamap=[1], # svgtile2datamap=None, # svgfilters=None, # svgtileheader='Isotopomer distribution', # tablefilters=None, # tableheaders=None # ); # dump the data to a json file ddtutilities = ddt_container(parameters_I = parametersobject_O,data_I = dataobject_O,tile2datamap_I = tile2datamap_O,filtermenu_I = None); if data_dir_I=='tmp': filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js' elif data_dir_I=='data_json': data_json_O = ddtutilities.get_allObjects_js(); return data_json_O; with open(filename_str,'w') as file: file.write(ddtutilities.get_allObjects());
def plot_normalizedSpectrumNormSum(self,experiment_id_I, sample_names_I = None, sample_name_abbreviations_I = None, met_ids_I = None, scan_types_I = None): '''calculate the average normalized intensity for all samples and scan types''' '''Assumptions: only a single fragment:spectrum is used_ per sample name abbreviation, time-point, replicate, scan_type (i.e. there are no multiple dilutions of the same precursor:spectrum that are used_) ''' mids = mass_isotopomer_distributions(); print('plot_normalizedSpectrumNormSum...') plot = matplot(); # get time points time_points = self.get_timePoint_experimentID_dataStage01Normalized(experiment_id_I); for tp in time_points: print('Plotting precursor and product spectrum from isotopomer normalized for time-point ' + str(tp)); if sample_names_I: sample_abbreviations = []; sample_types = ['Unknown','QC']; sample_types_lst = []; for sn in sample_names_I: for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndSampleName_dataStage01Normalized(experiment_id_I,st,tp,sn); sample_abbreviations.extend(sample_abbreviations_tmp); sample_types_lst.extend([st for i in range(len(sample_names_tmp))]); elif sample_name_abbreviations_I: sample_abbreviations = sample_name_abbreviations_I; sample_types_lst = ['Unknown' for x in sample_abbreviations]; # query sample types from sample name abbreviations and time-point from data_stage01_isotopomer_normalized else: # get sample names and sample name abbreviations sample_abbreviations = []; sample_types = ['Unknown','QC']; sample_types_lst = []; for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01Normalized(experiment_id_I,st,tp); sample_abbreviations.extend(sample_abbreviations_tmp); sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]); for sna_cnt,sna in enumerate(sample_abbreviations): print('Plotting precursor and product spectrum from isotopomer normalized for sample name abbreviation ' + sna); # get the scan_types if scan_types_I: scan_types = []; scan_types_tmp = []; scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01Normalized(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); scan_types = [st for st in scan_types_tmp if st in scan_types_I]; else: scan_types = []; scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01Normalized(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); for scan_type in scan_types: print('Plotting precursor and product spectrum for scan type ' + scan_type) # met_ids if not met_ids_I: met_ids = []; met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01Normalized( \ experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type); else: met_ids = met_ids_I; if not(met_ids): continue #no component information was found for met in met_ids: print('Plotting precursor and product spectrum for metabolite ' + met); # get replicates replicate_numbers = []; replicate_numbers = self.get_replicateNumbers_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetID_dataStage01Normalized( \ experiment_id_I,sna,tp,scan_type,met); peakSpectrum_normalized_lst = []; for rep in replicate_numbers: print('Plotting precursor and product spectrum for replicate_number ' + str(rep)); #get data peakData_I = {}; peakData_I = self.get_dataNormalized_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetIDAndReplicateNumber_dataStage01Normalized( \ experiment_id_I,sna,tp,scan_type,met,rep); fragment_formulas = list(peakData_I.keys()); peakSpectrum_corrected, peakSpectrum_normalized = mids.extract_peakList_normSum(\ peakData_I, fragment_formulas, True); peakSpectrum_normalized_lst.append(peakSpectrum_normalized); # plot spectrum data for all replicates and fragments fragment_formulas_unique = list(set(fragment_formulas_lst)); for fragment in fragment_formulas_unique: panelLabels = []; xticklabels = []; mean = []; xlabel = 'm/z' ylabel = 'intensity' for rep,spectrum in enumerate(peakSpectrum_normalized_lst): panelLabels_tmp = sna+'_'+met+'_'+fragment+'_'+str(rep+1) xticklabels_tmp = []; mean_tmp = []; for mass,intensity in spectrum[fragment].items(): intensity_tmp = intensity; if not intensity_tmp: intensity_tmp=0.0 mean_tmp.append(intensity_tmp); xticklabels_tmp.append(mass); panelLabels.append(panelLabels_tmp); xticklabels.append(xticklabels_tmp); mean.append(mean_tmp); plot.multiPanelBarPlot('',xticklabels,xlabel,ylabel,panelLabels,mean);
def execute_analyzeAveragesNormSum(self,experiment_id_I, sample_names_I = None, sample_name_abbreviations_I = None, met_ids_I = None, scan_types_I = None): '''calculate the average normalized intensity for all samples and scan types''' '''Assumptions: only a single fragment:spectrum is used_ per sample name abbreviation, time-point, replicate, scan_type (i.e. there are no multiple dilutions of the same precursor:spectrum that are used_) ''' mids = mass_isotopomer_distributions(); print('execute_analyzeAveragesNormSum...') # get time points time_points = self.get_timePoint_experimentID_dataStage01Normalized(experiment_id_I); for tp in time_points: print('Calculating average precursor and product spectrum from isotopomer normalized for time-point ' + str(tp)); if sample_names_I: sample_abbreviations = []; sample_types = ['Unknown','QC']; sample_types_lst = []; for sn in sample_names_I: for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndSampleName_dataStage01Normalized(experiment_id_I,st,tp,sn); sample_abbreviations.extend(sample_abbreviations_tmp); sample_types_lst.extend([st for i in range(len(sample_names_tmp))]); elif sample_name_abbreviations_I: sample_abbreviations = sample_name_abbreviations_I; sample_types_lst = ['Unknown' for x in sample_abbreviations]; # query sample types from sample name abbreviations and time-point from data_stage01_isotopomer_normalized else: # get sample names and sample name abbreviations sample_abbreviations = []; sample_types = ['Unknown','QC']; sample_types_lst = []; for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01Normalized(experiment_id_I,st,tp); sample_abbreviations.extend(sample_abbreviations_tmp); sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]); for sna_cnt,sna in enumerate(sample_abbreviations): print('Calculating average precursor and product spectrum from isotopomer normalized for sample name abbreviation ' + sna); # get the scan_types if scan_types_I: scan_types = []; scan_types_tmp = []; scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01Normalized(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); scan_types = [st for st in scan_types_tmp if st in scan_types_I]; else: scan_types = []; scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01Normalized(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); for scan_type in scan_types: print('Calculating average precursor and product spectrum for scan type ' + scan_type) # met_ids if not met_ids_I: met_ids = []; met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01Normalized( \ experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type); else: met_ids = met_ids_I; if not(met_ids): continue #no component information was found for met in met_ids: print('Calculating average precursor and product spectrum for metabolite ' + met); # get replicates replicate_numbers = []; replicate_numbers = self.get_replicateNumbers_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetID_dataStage01Normalized( \ experiment_id_I,sna,tp,scan_type,met); peakSpectrum_normalized_lst = []; for rep in replicate_numbers: print('Calculating average precursor and product spectrum for replicate_number ' + str(rep)); #get data peakData_I = {}; peakData_I = self.get_dataNormalized_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetIDAndReplicateNumber_dataStage01Normalized( \ experiment_id_I,sna,tp,scan_type,met,rep); fragment_formulas = list(peakData_I.keys()); peakSpectrum_corrected, peakSpectrum_normalized = mids.extract_peakList_normSum(\ peakData_I, fragment_formulas, True); peakSpectrum_normalized_lst.append(peakSpectrum_normalized); peakSpectrum_stats,peakSpectrum_theoretical = mids.compare_peakSpectrum_normSum(peakSpectrum_normalized_lst,True); # update data_stage01_isotopomer_normalized for frag,spec in peakSpectrum_theoretical.items(): if spec: fragment_str = re.sub('[+-]', '', frag); fragment_mass = Formula(fragment_str).isotope.mass; for k,v in peakSpectrum_theoretical[frag].items(): if v and k in peakSpectrum_stats[frag]: if peakSpectrum_stats[frag][k]['mean']> 0.0: intensities_cv = peakSpectrum_stats[frag][k]['stdDev']/peakSpectrum_stats[frag][k]['mean']*100; else: intensities_cv = 0.0; row = []; row = data_stage01_isotopomer_averagesNormSum(experiment_id_I, sna, sample_types_lst[sna_cnt], tp, met,frag, k, peakSpectrum_stats[frag][k]['n'], peakSpectrum_stats[frag][k]['mean'], intensities_cv, 'normSum', v, peakSpectrum_stats[frag][k]['absDev'], scan_type, True); elif v and k not in peakSpectrum_stats[frag]: intensities_cv = None; row = []; row = data_stage01_isotopomer_averagesNormSum(experiment_id_I, sna, sample_types_lst[sna_cnt], tp, met,frag, k, None, None, intensities_cv, 'normSum', v, None, scan_type, True); elif not v and k in peakSpectrum_stats[frag]: if peakSpectrum_stats[frag][k]['mean']> 0.0: intensities_cv = peakSpectrum_stats[frag][k]['stdDev']/peakSpectrum_stats[frag][k]['mean']*100; else: intensities_cv = 0.0; row = []; row = data_stage01_isotopomer_averagesNormSum(experiment_id_I, sna, sample_types_lst[sna_cnt], tp, met,frag, k, peakSpectrum_stats[frag][k]['n'], peakSpectrum_stats[frag][k]['mean'], intensities_cv, 'normSum', None, peakSpectrum_stats[frag][k]['absDev'], scan_type, True); self.session.add(row); self.session.commit();
def execute_analyzeAverages(self,experiment_id_I, sample_names_I = None, sample_name_abbreviations_I = None, met_ids_I = None, scan_types_I = None): '''calculate the average normalized intensity for MRM samples''' '''Assumptions: only a single fragment:spectrum is used_ per sample name abbreviation, time-point, replicate, scan_type (i.e. there are no multiple dilutions of the same precursor:spectrum that are used_) ''' mids = mass_isotopomer_distributions(); print('execute_analyzeAverages...') # get time points time_points = self.get_timePoint_experimentID_dataStage01Normalized(experiment_id_I); for tp in time_points: print('Calculating average precursor and product spectrum from isotopomer normalized for time-point ' + str(tp)); if sample_names_I: sample_abbreviations = []; sample_types = ['Unknown','QC']; sample_types_lst = []; for sn in sample_names_I: for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndSampleName_dataStage01Normalized(experiment_id_I,st,tp,sn); sample_abbreviations.extend(sample_abbreviations_tmp); sample_types_lst.extend([st for i in range(len(sample_names_tmp))]); elif sample_name_abbreviations_I: sample_abbreviations = sample_name_abbreviations_I; sample_types_lst = ['Unknown' for x in sample_abbreviations]; # query sample types from sample name abbreviations and time-point from data_stage01_isotopomer_normalized else: # get sample names and sample name abbreviations sample_abbreviations = []; sample_types = ['Unknown','QC']; sample_types_lst = []; for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01Normalized(experiment_id_I,st,tp); sample_abbreviations.extend(sample_abbreviations_tmp); sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]); for sna_cnt,sna in enumerate(sample_abbreviations): print('Calculating average precursor and product spectrum from isotopomer normalized for sample name abbreviation ' + sna); # get the scan_types if scan_types_I: scan_types = []; scan_types_tmp = []; scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01Normalized(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); scan_types = [st for st in scan_types_tmp if st in scan_types_I]; else: scan_types = []; scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01Normalized(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]); for scan_type in scan_types: print('Calculating average precursor and product spectrum for scan type ' + scan_type) # met_ids if not met_ids_I: met_ids = []; met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01Normalized( \ experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type); else: met_ids = met_ids_I; if not(met_ids): continue #no component information was found for met in met_ids: print('Calculating average precursor and product spectrum for metabolite ' + met); ## get fragment formulas and masses #fragment_formulas, fragment_masses = [],[]; #fragment_formulas,fragment_masses = self.get_fragmentFormulasAndMass_experimentIDAndSampleAbbreviationAndTimePointAndAndSampleTypeAndScanTypeAndMetID_dataStage01Normalized(experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met); #for mass_cnt,mass in enumerate(fragment_masses): # print 'Calculating average precursor and product spectrum for fragment/mass ' + fragment_formulas[mass_cnt] + '/' + str(mass); # # get data # intensities = []; # intensities = self.get_normalizedIntensity_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndMetIDAndFragmentFormulaAndMassAndScanType_dataStage01Normalized(experiment_id_I,sna,tp,sample_types_lst[sna_cnt],met,fragment_formulas[mass_cnt],mass,scan_type); # # calculate the average and cv # n_replicates = len(intensities); # intensities_average = 0.0; # intensities_var = 0.0; # intensities_cv = 0.0; # # calculate average and CV of intensities # if (not(intensities)): # #continue # intensities_average = 0.0; # intensities_var = 0.0; # intensities_cv = 0.0; # elif n_replicates<2: # require at least 2 replicates # #continue # intensities_average = 0.0; # intensities_var = 0.0; # intensities_cv = 0.0; # else: # intensities_average = numpy.mean(numpy.array(intensities)); # intensities_var = numpy.var(numpy.array(intensities)); # if (intensities_average <= 0.0): intensities_cv = 0.0; # else: intensities_cv = sqrt(intensities_var)/intensities_average*100; # # calculate the theoretical spectrum for the pecursor/mass # peakSpectrum_theoretical = mids.report_fragmentSpectrum_normMax([fragment_formulas[mass_cnt]],True); # # calculate the absolute deviation from the theoretical # intensity_theoretical = peakSpectrum_theoretical[fragment_formulas[mass_cnt]][mass]; # if intensity_theoretical > 0.0:abs_devFromTheoretical = abs(intensity_theoretical-intensities_average)/intensity_theoretical*100; # else: abs_devFromTheoretical = None; # # add to data_stage01_isotopomer_averages # row = []; # row = data_stage01_isotopomer_averages(experiment_id_I, sna, sample_types_lst[sna_cnt], tp, met,fragment_formulas[mass_cnt], mass, # n_replicates, intensities_average, intensities_cv, # 'normMax', intensity_theoretical, abs_devFromTheoretical, scan_type, True) # self.session.add(row); # get replicates replicate_numbers = []; replicate_numbers = self.get_replicateNumbers_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetID_dataStage01Normalized( \ experiment_id_I,sna,tp,scan_type,met); peakSpectrum_normalized_lst = []; for rep in replicate_numbers: print('Calculating average precursor and product spectrum for replicate_number ' + str(rep)); #get data peakData_I = {}; peakData_I = self.get_dataNormalized_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetIDAndReplicateNumber_dataStage01Normalized( \ experiment_id_I,sna,tp,scan_type,met,rep); fragment_formulas = list(peakData_I.keys()); peakSpectrum_corrected, peakSpectrum_normalized = mids.extract_peakList_normMax(\ peakData_I, fragment_formulas, True); peakSpectrum_normalized_lst.append(peakSpectrum_normalized); peakSpectrum_stats,peakSpectrum_theoretical = mids.compare_peakSpectrum_normMax(peakSpectrum_normalized_lst,True); # update data_stage01_isotopomer_averages for frag,spec in peakSpectrum_theoretical.items(): if spec: fragment_str = re.sub('[+-]', '', frag); fragment_mass = Formula(fragment_str).isotope.mass; for k,v in peakSpectrum_theoretical[frag].items(): if v and k in peakSpectrum_stats[frag]: if peakSpectrum_stats[frag][k]['mean']> 0.0: intensities_cv = peakSpectrum_stats[frag][k]['stdDev']/peakSpectrum_stats[frag][k]['mean']*100; else: intensities_cv = 0.0; row = []; row = data_stage01_isotopomer_averages(experiment_id_I, sna, sample_types_lst[sna_cnt], tp, met,frag, k, peakSpectrum_stats[frag][k]['n'], peakSpectrum_stats[frag][k]['mean'], intensities_cv, 'normMax', v, peakSpectrum_stats[frag][k]['absDev'], scan_type, True); elif v and k not in peakSpectrum_stats[frag]: intensities_cv = None; row = []; row = data_stage01_isotopomer_averages(experiment_id_I, sna, sample_types_lst[sna_cnt], tp, met,frag, k, None, None, intensities_cv, 'normMax', v, None, scan_type, True); elif not v and k in peakSpectrum_stats[frag]: if peakSpectrum_stats[frag][k]['mean']> 0.0: intensities_cv = peakSpectrum_stats[frag][k]['stdDev']/peakSpectrum_stats[frag][k]['mean']*100; else: intensities_cv = 0.0; row = []; row = data_stage01_isotopomer_averages(experiment_id_I, sna, sample_types_lst[sna_cnt], tp, met,frag, k, peakSpectrum_stats[frag][k]['n'], peakSpectrum_stats[frag][k]['mean'], intensities_cv, 'normMax', None, peakSpectrum_stats[frag][k]['absDev'], scan_type, True); self.session.add(row); self.session.commit();
def execute_recombineNormalizedSpectrum(self,experiment_id_I, sample_names_I = None, sample_name_abbreviations_I = None, met_ids_I = None): '''recombine intensity_normalized from a lower and higher dilution''' '''Assumptions: only a single fragment:spectrum is used_ per sample name abbreviation, time-point, replicate, scan_type (i.e. there are no multiple dilutions of the same precursor:spectrum that are used_) ''' mids = mass_isotopomer_distributions(); print('execute_recombineNormalizedSpectrum...') # get time points time_points = self.get_timePoint_experimentIDAndComment_dataStage01Normalized(experiment_id_I,'Recombine'); for tp in time_points: print('recombining spectrum for time-point ' + str(tp)); dataListUpdated = []; if sample_names_I: sample_abbreviations = []; sample_types = ['Unknown','QC']; for sn in sample_names_I: for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndSampleNameAndComment_dataStage01Normalized(experiment_id_I,st,tp,sn,'Recombine'); sample_abbreviations.extend(sample_abbreviations_tmp); elif sample_name_abbreviations_I: sample_abbreviations = sample_name_abbreviations_I; else: # get sample names and sample name abbreviations sample_abbreviations = []; sample_types = ['Unknown','QC']; for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndComment_dataStage01Normalized(experiment_id_I,st,tp,'Recombine'); sample_abbreviations.extend(sample_abbreviations_tmp); for sna_cnt,sna in enumerate(sample_abbreviations): print('recombining spectrum for sample name abbreviation ' + sna); # get the scan_types scan_types = []; scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndComment_dataStage01Normalized(experiment_id_I,tp,sna,'Recombine'); for scan_type in scan_types: print('recombining spectrum for scan type ' + scan_type) # met_ids if not met_ids_I: met_ids = []; met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndComment_dataStage01Normalized( \ experiment_id_I,sna,tp,scan_type,'Recombine'); else: met_ids = met_ids_I; if not(met_ids): continue #no component information was found for met in met_ids: print('recombining spectrum for metabolite ' + met); # get replicates replicate_numbers = []; replicate_numbers = self.get_replicateNumbers_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetID_dataStage01Normalized( \ experiment_id_I,sna,tp,scan_type,met); for rep in replicate_numbers: print('recombining spectrum for replicate_number ' + str(rep)); #get data peakData_I = {}; peakData_I = self.get_data_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetIDAndReplicateNumber_dataStage01Normalized( \ experiment_id_I,sna,tp,scan_type,met,rep); peakData_O,peakData_O_false,peakData_intensities_O = mids.recombine_dilutionsMRMs(peakData_I); peakSpectrum_stats = mids.compare_peakSpectrum_normMax([peakData_intensities_O]); # update data_stage01_isotopomer_normalized for frag,spec in peakSpectrum_stats.items(): if spec: fragment_str = re.sub('[+-]', '', frag); fragment_mass = Formula(fragment_str).isotope.mass; for k,v in peakSpectrum_stats[frag].items(): if int(numpy.round(k)) in peakData_O[frag]: dataListUpdated.append({'experiment_id':experiment_id_I, 'sample_name_abbreviation':sna, 'time_point':tp, 'dilution':peakData_O[frag][int(numpy.round(k))]['dilution'], 'replicate_number':rep, 'met_id':met, 'fragment_formula':frag, 'fragment_mass':int(numpy.round(k)), 'intensity_normalized':peakData_O[frag][int(numpy.round(k))]['intensity'], 'intensity_normalized_units':'normMax', 'abs_devFromTheoretical':v['absDev'], 'scan_type':scan_type, 'used_':peakData_O[frag][int(numpy.round(k))]['used_'], 'comment_':peakData_O[frag][int(numpy.round(k))]['comment_']}); # update data_stage01_isotopomer_normalized (rows changed to false) for frag,spec in peakData_O_false.items(): if spec: fragment_str = re.sub('[+-]', '', frag); fragment_mass = Formula(fragment_str).isotope.mass; for k,v in peakData_O_false[frag].items(): if v: dataListUpdated.append({'experiment_id':experiment_id_I, 'sample_name_abbreviation':sna, 'time_point':tp, 'dilution':v['dilution'], 'replicate_number':rep, 'met_id':met, 'fragment_formula':frag, 'fragment_mass':int(numpy.round(k)), 'intensity_normalized':v['intensity'], 'intensity_normalized_units':'normMax', 'abs_devFromTheoretical':None, 'scan_type':scan_type, 'used_':v['used_'], 'comment_':v['comment_']}); self.update_data_stage01_isotopomer_normalized(dataListUpdated);
def execute_updateNormalizedSpectrum(self,experiment_id_I, sample_names_I = None, sample_name_abbreviations_I = None, met_ids_I = None, scan_types_I = None): '''re-calculate intensity_normalized from intensity_corrected and used''' mids = mass_isotopomer_distributions(); print('execute_updateNormalizedSpectrum...') # get time points time_points = self.get_timePoint_experimentID_dataStage01Normalized(experiment_id_I); for tp in time_points: print('Building precursor and product spectrum from isotopomer normalized for time-point ' + str(tp)); dataListUpdated = []; # get dilutions dilutions = []; dilutions = self.get_sampleDilution_experimentIDAndTimePoint_dataStage01Normalized(experiment_id_I,tp); for dil in dilutions: print('Building precursor and product spectrum from isotopomer normalized for dilution ' + str(dil)); if sample_names_I: sample_abbreviations = []; sample_types = ['Unknown','QC']; for sn in sample_names_I: for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndDilutionAndSampleName_dataStage01Normalized(experiment_id_I,st,tp,dil,sn); sample_abbreviations.extend(sample_abbreviations_tmp); elif sample_name_abbreviations_I: sample_abbreviations = sample_name_abbreviations_I; else: # get sample names and sample name abbreviations sample_abbreviations = []; sample_types = ['Unknown','QC']; for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndDilution_dataStage01Normalized(experiment_id_I,st,tp,dil); sample_abbreviations.extend(sample_abbreviations_tmp); for sna_cnt,sna in enumerate(sample_abbreviations): print('Building precursor and product spectrum from isotopomer normalized for sample name abbreviation ' + sna); # get the scan_types if scan_types_I: scan_types = scan_types_I; else: scan_types = []; scan_types = self.get_scanTypes_experimentIDAndTimePointAndDilutionAndSampleAbbreviations_dataStage01Normalized(experiment_id_I,tp,dil,sna); for scan_type in scan_types: print('Building precursor and product spectrum for scan type ' + scan_type) # met_ids if not met_ids_I: met_ids = []; met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndDilutionAndScanType_dataStage01Normalized( \ experiment_id_I,sna,tp,dil,scan_type); else: met_ids = met_ids_I; if not(met_ids): continue #no component information was found for met in met_ids: print('Building precursor and product spectrum from isotopomer normalized for metabolite ' + met); # get sample names sample_names = []; sample_names,replicate_numbers,sample_types = self.get_sampleNamesAndReplicateNumbersAndSampleTypes_experimentIDAndSampleNameAbbreviationAndMetIDAndTimePointAndDilutionAndScanType_dataStage01Normalized(experiment_id_I,sna,met,tp,dil,scan_type); # iterate through sample names then mets/fragments in order to calculate the spectrum for each sample and component for sn_cnt,sn in enumerate(sample_names): print('Building precursor and product spectrum from isotopomer normalized for sample ' + sn); # get peak data for the sample/met_id/scan_type peak_data = []; peak_data = self.get_data_experimentIDAndSampleNameAndMetIDAndAndScanType_normalized(experiment_id_I,sn,met,scan_type); fragment_formulas = list(peak_data.keys()); peakSpectrum_corrected, peakSpectrum_normalized = mids.extract_peakList_normMax(\ peak_data, fragment_formulas, True); peakSpectrum_stats,peakSpectrum_theoretical = mids.compare_peakSpectrum_normMax([peakSpectrum_normalized],True); # update data_stage01_isotopomer_normalized for frag,spec in peakSpectrum_theoretical.items(): if spec: fragment_str = re.sub('[+-]', '', frag); fragment_mass = Formula(fragment_str).isotope.mass; for k,v in peakSpectrum_theoretical[frag].items(): dataListUpdated.append({'experiment_id':experiment_id_I, 'sample_name':sn, 'sample_name_abbreviation':sna, 'sample_type':sample_types[sn_cnt], 'time_point':tp, 'dilution':dil, 'replicate_number':replicate_numbers[sn_cnt], 'met_id':met, 'fragment_formula':frag, 'fragment_mass':int(numpy.round(k)), 'intensity_corrected':peakSpectrum_corrected[frag][k], 'intensity_corrected_units':'cps', 'intensity_normalized':peakSpectrum_normalized[frag][k], 'intensity_normalized_units':'normMax', 'intensity_theoretical':v, 'abs_devFromTheoretical':peakSpectrum_stats[frag][k]['absDev'], 'scan_type':scan_type}); self.update_data_stage01_isotopomer_normalized(dataListUpdated);
def execute_buildSpectrumFromMRMs(self,experiment_id_I,ms_methodtype_I='isotopomer_13C',sample_name_abbreviations_I=[],sample_names_I=[],met_ids_I=[]): '''Extract peak spectrum for each fragment from MRMs''' # Input: # experiment_id # sample_names = (optional) list of specific samples # Output: # sample_name # sample_id # component_group_name # component_name # calculated_concentration # calculated_concentration_units # used_ # assumptions: # 1. there is only spectrum of MRMs for each components mids = mass_isotopomer_distributions(); print('build_precursorSpectrumFromMRMs...') # get time points time_points = self.get_timePoint_experimentID(experiment_id_I); for tp in time_points: print('Building precursor and product spectrum from MRMs for time-point ' + str(tp)); # get dilutions dilutions = self.get_sampleDilution_experimentIDAndTimePoint(experiment_id_I,tp); for dil in dilutions: print('Building precursor and product spectrum from MRMs for dilution ' + str(dil)); if sample_names_I: sample_abbreviations = []; for sn in sample_names_I: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleName(experiment_id_I,sn); sample_abbreviations.extend(sample_abbreviations_tmp); elif sample_name_abbreviations_I: sample_abbreviations = sample_name_abbreviations_I; else: # get sample names and sample name short sample_abbreviations = []; sample_types = ['Unknown','QC']; for st in sample_types: sample_abbreviations_tmp = []; sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndDilution(experiment_id_I,st,tp,dil); sample_abbreviations.extend(sample_abbreviations_tmp); for sna_cnt,sna in enumerate(sample_abbreviations): print('Building precursor and product spectrum from MRMs for sample name abbreviation ' + sna); ##BUG alert: ##there is a potential bug whereby if the entire spectra per compound is not returned ##e.g. one of the samples is not returned because "used_" is set to false in isotopomer_MQResultsTable ##the spectra could potentially be shifted ##get_componentsNamesAndOther_experimentIDAndSampleNameAndMSMethodTypeAndTimePointAndDilution has been adjusted to ##return all component_names even if the sample from which it came is not "used_" as a temporary fix ##this works for C12 validation experiments, but ##the results of normalization will need to be monitored until a more robust method is established ##UPDATE to BUG: ##'met_cnt_max = len(component_names)-1' changed to 'met_cnt_max = len(component_names)' and ##get_sampleNamesAndReplicateNumbersAndSampleTypes_experimentIDAndSampleNameAbbreviationAndSampleDescriptionAndTimePointAndDilution added ##so that the last component is included in the met/fragment spectra even when not all component names have "used_" ##set to true in isotopomer_MQResultsTable # component names, group names, fragment formula, and fragment mass if met_ids_I: component_names,component_group_names,\ precursor_formulas_O, precursor_masses_O,\ product_formulas_O, product_masses_O = [],[],[],[],[],[]; for met in met_ids_I: component_names_tmp,component_group_names_tmp,\ precursor_formulas_tmp, precursor_masses_tmp,\ product_formulas_tmp, product_masses_tmp = [],[],[],[],[],[]; component_names_tmp,component_group_names_tmp,\ precursor_formulas_tmp, precursor_masses_tmp,\ product_formulas_tmp, product_masses_tmp = \ self.get_componentsNamesAndOther_experimentIDAndSampleNameAndMSMethodTypeAndTimePointAndDilutionAndMetID( \ experiment_id_I,sna,ms_methodtype_I,tp,dil,met); if not(component_names_tmp): continue #no component information was found component_names.extend(component_names_tmp) component_group_names.extend(component_group_names_tmp) precursor_formulas_O.extend(precursor_formulas_tmp) precursor_masses_O.extend(precursor_masses_tmp) product_formulas_O.extend(product_formulas_tmp) product_masses_O.extend(product_masses_tmp) else: component_names,component_group_names,\ precursor_formulas_O, precursor_masses_O,\ product_formulas_O, product_masses_O = [],[],[],[],[],[]; component_names,component_group_names,\ precursor_formulas_O, precursor_masses_O,\ product_formulas_O, product_masses_O = \ self.get_componentsNamesAndOther_experimentIDAndSampleNameAndMSMethodTypeAndTimePointAndDilution( \ experiment_id_I,sna,ms_methodtype_I,tp,dil); if not(component_names): continue #no component information was found # extract unique met ids and precursor formula id met_ids_unique = []; met_ids = []; #precursor_formulas_unique = []; #product_formulas_unique = []; met_id = ''; met_id_old = ''; # algorithm works because lists are ordered by component_names for i,cn in enumerate(component_names): met_id = cn.split('.')[0]; met_ids.append(met_id); if met_id != met_id_old: met_id_old = met_id; met_ids_unique.append(met_id); #precursor_formulas_unique.append(precursor_formulas_O[i]); #product_formulas_unique.append(product_formulas_O[i]); # get precursor and productformulas for each unique met id: precursor_formulas_unique = []; product_formulas_unique = []; for met in met_ids_unique: precursor_formula,product_formula = None,None; precursor_formula,product_formula = self.get_precursorFormulaAndProductFormula_metID(met,'-','isotopomer_13C') precursor_formulas_unique.append(precursor_formula); product_formulas_unique.append(product_formula); # build precursor and spectrum for each met met_all_cnt = 0; met_cnt_max = len(component_names); # for use in a while loop for i,met in enumerate(met_ids_unique): print('Building precursor and product spectrum from MRMs for metabolite ' + met); # get filtrate samples precursorFiltrate_measured = {}; productFiltrate_measured = {}; #precursorFiltrate_measured[precursor_formulas_unique[i]] = None; #keep track of all met_ids precursorFiltrate = {}; productFiltrate = {}; met_cnt = met_all_cnt; # iterate through mets/fragments then sample names in order to calculate the average for each component_name (fragment/mass) while met_cnt < met_cnt_max and met==met_ids[met_cnt]: # get filtrate sample names sample_names = []; replicate_numbers = []; sample_description = 'Filtrate'; sample_names,replicate_numbers,sample_types = self.get_sampleNamesAndReplicateNumbersAndSampleTypes_experimentIDAndSampleNameAbbreviationAndSampleDescriptionAndComponentNameAndTimePointAndDilution(experiment_id_I,sna,sample_description,component_names[met_cnt],tp,dil); intensities = []; for sn_cnt,sn in enumerate(sample_names): # get intensities intensity = None; intensity = self.get_peakHeight_sampleNameAndComponentName(sn,component_names[met_cnt]); if not(intensity): continue intensities.append(intensity); n_replicates = len(intensities); intensities_average_filtrate = 0.0; intensities_var_filtrate = 0.0; # calculate average and CV of the intensities if (not(intensities)): intensities_average_filtrate = 0.0; elif n_replicates<2: intensities_average_filtrate = intensities[0]; else: #intensities_average_filtrate, intensities_var_filtrate = self.calculate.calculate_ave_var_R(intensities); intensities_average_filtrate = numpy.mean(numpy.array(intensities)); intensities_var_filtrate = numpy.var(numpy.array(intensities)); # append value to dictionary precursorFiltrate[(precursor_masses_O[met_cnt],product_masses_O[met_cnt])] = intensities_average_filtrate; productFiltrate[(precursor_masses_O[met_cnt],product_masses_O[met_cnt])] = intensities_average_filtrate; met_cnt += 1; precursorFiltrate_measured[precursor_formulas_unique[i]] = precursorFiltrate productFiltrate_measured[product_formulas_unique[i]] = productFiltrate # get broth samples sample_names = []; sample_description = 'Broth'; sample_names,replicate_numbers,sample_types = self.get_sampleNamesAndReplicateNumbersAndSampleTypes_experimentIDAndSampleNameAbbreviationAndSampleDescriptionAndTimePointAndDilution(experiment_id_I,sna,sample_description,tp,dil); #sample_names,replicate_numbers,sample_types = self.get_sampleNamesAndReplicateNumbersAndSampleTypes_experimentIDAndSampleNameAbbreviationAndSampleDescriptionAndComponentNameAndTimePointAndDilution(experiment_id_I,sna,sample_description,component_names[met_cnt],tp,dil); # iterate through sample names then mets/fragments in order to calculate the spectrum for each sample and component for sn_cnt,sn in enumerate(sample_names): print('Building precursor and product spectrum from MRMs for sample ' + sn); precursorPeakSpectrum_measured = {}; precursorPeakSpectrum_corrected = {}; productPeakSpectrum_measured = {}; productPeakSpectrum_corrected = {}; #precursorPeakSpectrum_measured[precursor_formulas_unique[i]] = None; #keep track of all met_ids #precursorPeakSpectrum_corrected[precursor_formulas_unique[i]] = None; #keep track of all met_ids precursorMeasured = {}; precursorCorrected = {}; productMeasured = {}; productCorrected = {}; met_cnt = met_all_cnt; while met_cnt < met_cnt_max and met==met_ids[met_cnt]: # get intensities intensity = None; intensity = self.get_peakHeight_sampleNameAndComponentName(sn,component_names[met_cnt]); if not(intensity): precursorMeasured[(precursor_masses_O[met_cnt],product_masses_O[met_cnt])] = 0.0; productMeasured[(precursor_masses_O[met_cnt],product_masses_O[met_cnt])] = 0.0; else: precursorMeasured[(precursor_masses_O[met_cnt],product_masses_O[met_cnt])] = intensity; productMeasured[(precursor_masses_O[met_cnt],product_masses_O[met_cnt])] = intensity; #if precursorFiltrate_measured[precursor_formulas_unique[i]][(precursor_masses_O[met_cnt],product_masses_O[met_cnt])] < 0.5*intensity: # corrected_intensity = intensity - precursorFiltrate_measured[precursor_formulas_unique[i]][(precursor_masses_O[met_cnt],product_masses_O[met_cnt])]; #else: corrected_intensity = 0.0; #precursorCorrected[(precursor_masses_O[met_cnt],product_masses_O[met_cnt])] = corrected_intensity; met_cnt += 1; precursorPeakSpectrum_measured[precursor_formulas_unique[i]] = precursorMeasured; productPeakSpectrum_measured[product_formulas_unique[i]] = productMeasured; # generate normalized spectrum for the precursor: precursorPeakSpectrum_measured, precursorPeakSpectrum_corrected, precursorPeakSpectrum_normalized \ = mids.build_precursorSpectrumFromMRMs(precursorPeakSpectrum_measured,precursorFiltrate_measured); peakSpectrum_stats_O,precursorPeakSpectrum_theoretical = mids.compare_peakSpectrum_normMax([precursorPeakSpectrum_normalized],True); # update data_stage01_isotopomer_normalized if precursorPeakSpectrum_theoretical[precursor_formulas_unique[i]]: for k,v in precursorPeakSpectrum_theoretical[precursor_formulas_unique[i]].items(): row1 = None; row1 = data_stage01_isotopomer_normalized(experiment_id_I,sn,sna,sample_types[sn_cnt],tp,dil,replicate_numbers[sn_cnt], met,precursor_formulas_unique[i],int(numpy.round(k)), precursorPeakSpectrum_measured[precursor_formulas_unique[i]][k],'cps', precursorPeakSpectrum_corrected[precursor_formulas_unique[i]][k],'cps', precursorPeakSpectrum_normalized[precursor_formulas_unique[i]][k],'normMax', v,peakSpectrum_stats_O[precursor_formulas_unique[i]][k]['absDev'],'MRM',True,None); self.session.add(row1); # generate normalized spectrum for the product: productPeakSpectrum_measured, productPeakSpectrum_corrected, productPeakSpectrum_normalized \ = mids.build_productSpectrumFromMRMs(productPeakSpectrum_measured,productFiltrate_measured); peakSpectrum_stats_O,productPeakSpectrum_theoretical = mids.compare_peakSpectrum_normMax([productPeakSpectrum_normalized],True); # update data_stage01_isotopomer_normalized if productPeakSpectrum_theoretical[product_formulas_unique[i]]: for k,v in productPeakSpectrum_theoretical[product_formulas_unique[i]].items(): row2 = None; row2 = data_stage01_isotopomer_normalized(experiment_id_I,sn,sna,sample_types[sn_cnt],tp,dil,replicate_numbers[sn_cnt], met,product_formulas_unique[i],int(numpy.round(k)), productPeakSpectrum_measured[product_formulas_unique[i]][k],'cps', productPeakSpectrum_corrected[product_formulas_unique[i]][k],'cps', productPeakSpectrum_normalized[product_formulas_unique[i]][k],'normMax', v,peakSpectrum_stats_O[product_formulas_unique[i]][k]['absDev'],'MRM',True,None); self.session.add(row2); met_all_cnt = met_cnt self.session.commit();