def plot_averageSpectrumNormSum(self,experiment_id_I, time_points_I = None, sample_name_abbreviations_I = None, met_ids_I = None, scan_types_I = None):
'''calculate the average normalized intensity for all samples and scan types'''
'''Assumptions:
only a single fragment:spectrum is used_ per sample name abbreviation, time-point, replicate, scan_type
(i.e. there are no multiple dilutions of the same precursor:spectrum that are used_)
'''
mids = mass_isotopomer_distributions();
print('plot_averagesNormSum...')
plot = matplot();
# get time points
if time_points_I:
time_points = time_points_I;
else:
time_points = [];
time_points = self.get_timePoint_experimentID_dataStage01AveragesNormSum(experiment_id_I);
for tp in time_points:
print('Plotting product and precursor for time-point ' + str(tp));
# get sample names and sample name abbreviations
if sample_name_abbreviations_I:
sample_abbreviations = sample_name_abbreviations_I;
sample_types_lst = ['Unknown' for x in sample_abbreviations];
else:
sample_abbreviations = [];
sample_types = ['Unknown'];
sample_types_lst = [];
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01AveragesNormSum(experiment_id_I,st,tp);
sample_abbreviations.extend(sample_abbreviations_tmp);
sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]);
for sna_cnt,sna in enumerate(sample_abbreviations):
print('Plotting product and precursor for sample name abbreviation ' + sna);
# get the scan_types
if scan_types_I:
scan_types = [];
scan_types_tmp = [];
scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
scan_types = [st for st in scan_types_tmp if st in scan_types_I];
else:
scan_types = [];
scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
for scan_type in scan_types:
print('Plotting product and precursor for scan type ' + scan_type)
# met_ids
if not met_ids_I:
met_ids = [];
met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01AveragesNormSum( \
experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type);
else:
met_ids = met_ids_I;
if not(met_ids): continue #no component information was found
for met in met_ids:
print('Plotting product and precursor for metabolite ' + met);
# fragments
fragment_formulas = [];
fragment_formulas = self.get_fragmentFormula_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanTypeAndMetID_dataStage01AveragesNormSum( \
experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met);
for frag in fragment_formulas:
print('Plotting product and precursor for fragment ' + frag);
# data
data_mat = [];
data_mat_cv = [];
data_masses = [];
data_mat,data_mat_cv,data_masses = self.get_spectrum_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanTypeAndMetIDAndFragmentFormula_dataStage01AveragesNormSum( \
experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met,frag);
data_stdev = [];
for i,d in enumerate(data_mat):
stdev = 0.0;
stderr = 0.0;
if data_mat_cv[i]:
stdev = data_mat[i]*data_mat_cv[i]/100;
data_stdev.append(stdev);
title = sna+'_'+met+'_'+frag;
plot.barPlot(title,data_masses,'intensity','m/z',data_mat,var_I=None,se_I=data_stdev,add_labels_I=True)
def execute_normalizeSpectrumFromReference_v1(self,experiment_id_I,sample_name_abbreviations_I = None,use_mrm_ref = True):
# 1. import used peak spectrum to normalized table after multiplying by measured
# scaling factor calculated from used MRM spectrum
# 2. be sure that the MRMs in the normalized table have been finalized
'''NOTES: Broken for the following reason:
cannot follow the forloop pattern used in buildSpectrumFromPeakData (i.e. starting with sample name)
must use the forloop pattern used in updateNormalizedSpectrum (i.e. time-point to dilutions to sample name abbreviations to scan types to mets)
buildSpectrumFromPeakData and updatePeakSpectrum methods process one product:spectrum from a single precursor at a time;
each precursor:product:spectrum is associated with only one sample name
However, because the entire range of precursor:product:spectrum for a given met can encompass multiple dilutions and therefore different
sample names, a more generic approach must be used
Please use current version'''
mids = mass_isotopomer_distributions();
# extract out the peakSpectrum
# get sample name for the experiment
print('execute_normalizeSpectrumFromReference...')
if sample_name_abbreviations_I:
sample_names = [];
sample_types = ['Unknown','QC'];
sample_types_lst = [];
for sna in sample_name_abbreviations_I:
for st in sample_types:
sample_names_tmp = [];
sample_names_tmp = self.get_sampleNames_experimentIDAndSampleTypeAndSampleNameAbbreviation_peakSpectrum(experiment_id_I,st,sna);
sample_names.extend(sample_names_tmp);
sample_types_lst.extend([st for i in range(len(sample_names_tmp))]);
else:
sample_names = [];
sample_types = ['Unknown','QC'];
sample_types_lst = [];
for st in sample_types:
sample_names_tmp = [];
sample_names_tmp = self.get_sampleNames_experimentIDAndSampleType_peakSpectrum(experiment_id_I,st);
sample_names.extend(sample_names_tmp);
sample_types_lst.extend([st for i in range(len(sample_names_tmp))]);
for sn_cnt,sn in enumerate(sample_names):
print('normalizing peak spectrum for sample ' + sn);
# get other information about the sample for later use
sample_name_abbreviation,time_point,dilution,replicate_numbers = None,None,None,None;
sample_name_abbreviation,time_point,dilution,replicate_numbers = self.get_sampleNameAbbreviationsAndOther_experimentIDAndSampleName_peakSpectrum(experiment_id_I,sn);
# get met_id and precursor_formula for each sample
scan_type = [];
scan_type = self.get_scanType_experimentIDAndSampleName_peakSpectrum(experiment_id_I,sn);
for scantype in scan_type:
print('normalizing spectrum for scan type ' + scantype);
# get met_id
med_id = [];
met_id = self.get_metID_experimentIDAndSampleNameAndScanType_peakSpectrum(experiment_id_I,sn,scantype);
for met in met_id:
print('normalizing peak spectrum for met_id ' + met);
# get precursor formula and mass
precursor_formula, precursor_mass = [], [];
precursor_formula, precursor_mass = self.get_precursorFormulaAndMass_experimentIDAndSampleNameAndMetIDAndScanType_peakSpectrum(experiment_id_I,sn,met,scantype);
peak_data_all = {};
scaling_factors_all = {};
for precursor_cnt, precursor in enumerate(precursor_formula):
peak_data_all[precursor] = None;
scaling_factors_all[precursor] = None;
print('normalizing peak spectrum for precursor ' + precursor);
precursor_str = re.sub('[+-]', '', precursor);
# get all product fragments for the met_id/precursor
product_formulas = [];
product_formulas = self.get_productFormulas_experimentIDAndSampleNameAndMetIDAndPrecursorFormulaAndScanType_peakSpectrum(experiment_id_I,sn,met,precursor,scantype);
# get the m+0 precursor_formula
precursor_formula_monoisotopic = self.get_precursorFormula_metID(met,'-','tuning');
precursor_monoisotopic_str = re.sub('[+-]', '', precursor_formula_monoisotopic);
precursor_monoisotpoic_mass = int(numpy.round(Formula(precursor_monoisotopic_str).isotope.mass));
# get peakSpectrum data
peak_data = {};
#Change to sna+rep+timepoint:peak_data = self.get_normalizedIntensity_experimentIDAndSampleNameAndMetIDAndPrecursorFormulaAndScanType_peakSpectrum(experiment_id_I,sn,met,precursor,scantype);
peak_data_all[precursor] = peak_data;
if scantype == 'ER':
scaling_factors_all[precursor] = 1.0; # there is no need to scale ER or other precursor ion scans
else:
if use_mrm_ref:
# get reference MRM spectrum scaling factor for the sample
#scaling_factor,scaling_factor_cv = None,None; # will need to incorporate propogation of error
#scaling_factor,scaling_factor_cv = self.get_normalizedIntensity_experimentIDAndSampleAbbreviationAndTimePointAndMetIDAndFragmentFormulaAndMassAndScanType_dataStage01Averages(experiment_id_I,sample_name_abbreviation,time_point,met,precursor_formula_monoisotopic,precursor_mass[precursor_cnt],'MRM');
scaling_factor = None; # does not require the propogation of error
scaling_factor = self.get_normalizedIntensity_experimentIDAndSampleAbbreviationAndTimePointAndReplicateNumberAndMetIDAndFragmentFormulaAndMassAndScanType_dataStage01Normalized(experiment_id_I,sample_name_abbreviation,time_point,replicate_numbers,met,precursor_formula_monoisotopic,precursor_mass[precursor_cnt],'MRM');
if scaling_factor: scaling_factors_all[precursor] = scaling_factor;
else:
scaling_factors_all[precursor] = 0.0;
## substitute with reference spectrum
#refspec = mids.report_fragmentSpectrum_normMax([precursor_formula_monoisotopic],True);
#scaling_factor = refspec[precursor_formula_monoisotopic][precursor_mass[precursor_cnt]];
#scaling_factors_all[precursor] = scaling_factor;
else:
# get reference ER spectrum scaling factor for the sample
scaling_factor = None;
scaling_factor = self.get_normalizedIntensity_experimentIDAndSampleAbbreviationAndTimePointAndReplicateNumberAndMetIDAndPrecursorFormulaAndMassAndScanType_peakSpectrum(experiment_id_I,sample_name_abbreviation,time_point,replicate_numbers,met,precursor_formula_monoisotopic,precursor_mass[precursor_cnt],'ER');
if scaling_factor: scaling_factors_all[precursor] = scaling_factor;
else:
scaling_factors_all[precursor] = 0.0;
## substitute with reference spectrum
#refspec = mids.report_fragmentSpectrum_normMax([precursor_formula_monoisotopic],True);
#scaling_factor = refspec[precursor_formula_monoisotopic][precursor_mass[precursor_cnt]];
#scaling_factors_all[precursor] = scaling_factor;
# normalize spectrum to reference MRM for each precursor (m+0,m+1,...)
peakSpectrum_normalized = mids.normalize_peakSpectrum_normMax(peak_data_all,scaling_factors_all);
peakSpectrum_stats,peakSpectrum_theoretical = mids.compare_peakSpectrum_normMax([peakSpectrum_normalized],True);
# update data_stage01_isotopomer_peakSpectrum
for frag,spec in peakSpectrum_theoretical.items():
if spec:
product_str = re.sub('[+-]', '', frag);
product_mass = Formula(product_str).isotope.mass;
for k,v in peakSpectrum_theoretical[frag].items():
if k in peakSpectrum_normalized[frag]:
row = None;
row = data_stage01_isotopomer_normalized(experiment_id_I,sn,sample_name_abbreviation,sample_types_lst[sn_cnt],time_point,dilution,replicate_numbers,
met,frag,int(numpy.round(k)),
None,'cps',None,'cps',
peakSpectrum_normalized[frag][k],'normMax',
v,peakSpectrum_stats[frag][k]['absDev'],scantype,True);
self.session.add(row);
self.session.commit();
def export_dataStage01IsotopomerAveragesNormSum_js(self,experiment_id_I, time_points_I = None, sample_name_abbreviations_I = None, met_ids_I = None, scan_types_I = None,data_dir_I="tmp",
single_plot_I = True):
'''Export data_stage01_isotopomer_averagesNormSum to js file'''
mids = mass_isotopomer_distributions();
print('plotting averagesNormSum...')
data_O = [];
data_dict_O = {};
# get time points
if time_points_I:
time_points = time_points_I;
else:
time_points = [];
time_points = self.get_timePoint_experimentID_dataStage01AveragesNormSum(experiment_id_I);
for tp in time_points:
print('Plotting product and precursor for time-point ' + str(tp));
# get sample names and sample name abbreviations
sample_abbreviations = [];
sample_types = ['Unknown'];
sample_types_lst = [];
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01AveragesNormSum(experiment_id_I,st,tp);
sample_abbreviations.extend(sample_abbreviations_tmp);
sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]);
if sample_name_abbreviations_I:
sample_abbreviations = [sna for sna in sample_abbreviations if sna in sample_name_abbreviations_I];
sample_types_lst = ['Unknown' for x in sample_abbreviations];
for sna_cnt,sna in enumerate(sample_abbreviations):
print('Plotting product and precursor for sample name abbreviation ' + sna);
data_dict_O[sna] = [];
# get the scan_types
if scan_types_I:
scan_types = [];
scan_types_tmp = [];
scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
scan_types = [st for st in scan_types_tmp if st in scan_types_I];
else:
scan_types = [];
scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
for scan_type in scan_types:
print('Plotting product and precursor for scan type ' + scan_type)
# met_ids
if not met_ids_I:
met_ids = [];
met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01AveragesNormSum( \
experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type);
else:
met_ids = met_ids_I;
if not(met_ids): continue #no component information was found
for met in met_ids:
print('Plotting product and precursor for metabolite ' + met);
# fragments
fragment_formulas = [];
fragment_formulas = self.get_fragmentFormula_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanTypeAndMetID_dataStage01AveragesNormSum( \
experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met);
for frag in fragment_formulas:
print('Plotting product and precursor for fragment ' + frag);
# data
data_mat = [];
data_mat_cv = [];
data_masses = [];
data_mat,data_mat_cv,data_masses = self.get_spectrum_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanTypeAndMetIDAndFragmentFormula_dataStage01AveragesNormSum( \
experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met,frag);
for i,d in enumerate(data_mat):
stdev = 0.0;
stderr = 0.0;
lb = None;
ub = None;
if data_mat_cv[i]:
stdev = data_mat[i]*data_mat_cv[i]/100;
lb = data_mat[i]-stdev;
ub = data_mat[i]+stdev;
fragment_id = mids.make_fragmentID(met,frag,data_masses[i]);
tmp = {
'experiment_id':experiment_id_I,
'sample_name_abbreviation':sna,
'sample_type':sample_types_lst[sna_cnt],
'time_point':tp,
'met_id':met,
'fragment_formula':frag,
'fragment_mass':data_masses[i],
'fragment_id':fragment_id,
'intensity_normalized_average':d,
'intensity_normalized_cv':data_mat_cv[i],
'intensity_normalized_lb':lb,
'intensity_normalized_ub':ub,
'intensity_normalized_units':"normSum",
'scan_type':scan_type,
'used_':True,
'comment_':None}
data_O.append(tmp);
data_dict_O[sna].append(tmp);
print('tabulating averagesNormSum...')
data_table_O = [];
# get time points
if time_points_I:
time_points = time_points_I;
else:
time_points = [];
time_points = self.get_timePoint_experimentID_dataStage01AveragesNormSum(experiment_id_I);
for tp in time_points:
print('Tabulating product and precursor for time-point ' + str(tp));
# get sample names and sample name abbreviations
sample_abbreviations = [];
sample_types = ['Unknown'];
sample_types_lst = [];
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01AveragesNormSum(experiment_id_I,st,tp);
sample_abbreviations.extend(sample_abbreviations_tmp);
sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]);
if sample_name_abbreviations_I:
sample_abbreviations = [sna for sna in sample_abbreviations if sna in sample_name_abbreviations_I];
sample_types_lst = ['Unknown' for x in sample_abbreviations];
for sna_cnt,sna in enumerate(sample_abbreviations):
print('Tabulating product and precursor for sample name abbreviation ' + sna);
# get the scan_types
if scan_types_I:
scan_types = [];
scan_types_tmp = [];
scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
scan_types = [st for st in scan_types_tmp if st in scan_types_I];
else:
scan_types = [];
scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
for scan_type in scan_types:
print('Tabulating product and precursor for scan type ' + scan_type)
# met_ids
if not met_ids_I:
met_ids = [];
met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01AveragesNormSum( \
experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type);
else:
met_ids = met_ids_I;
if not(met_ids): continue #no component information was found
for met in met_ids:
print('Tabulating product and precursor for metabolite ' + met);
# get rows
rows = [];
rows = self.get_rows_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanTypeAndMetID_dataStage01AveragesNormSum( \
experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met);
if not rows: continue;
for d in rows:
d['fragment_id'] = mids.make_fragmentID(d['met_id'],d['fragment_formula'],d['fragment_mass']);
data_table_O.extend(rows);
print('exporting averagesNormSum...');
# visualization parameters
data1_keys = ['sample_name_abbreviation',
'sample_type',
'met_id',
'time_point',
'fragment_formula',
'fragment_mass',
'scan_type',
'fragment_id'
];
data1_nestkeys = [
#'fragment_id',
'fragment_mass'
];
data1_keymap = {
#'xdata':'fragment_id',
'xdata':'fragment_mass',
'ydata':'intensity_normalized_average',
'tooltiplabel':'sample_name_abbreviation',
'serieslabel':'scan_type',
#'featureslabel':'fragment_id',
'featureslabel':'fragment_mass',
'ydatalb':'intensity_normalized_lb',
'ydataub':'intensity_normalized_ub'};
# initialize the ddt objects
dataobject_O = [];
parametersobject_O = [];
tile2datamap_O = {};
# make the tile parameter objects
formtileparameters_O = {'tileheader':'Filter menu','tiletype':'html','tileid':"filtermenu1",'rowid':"row1",'colid':"col1",
'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"};
formparameters_O = {'htmlid':'filtermenuform1',"htmltype":'form_01',"formsubmitbuttonidtext":{'id':'submit1','text':'submit'},"formresetbuttonidtext":{'id':'reset1','text':'reset'},"formupdatebuttonidtext":{'id':'update1','text':'update'}};
formtileparameters_O.update(formparameters_O);
dataobject_O.append({"data":data_O,"datakeys":data1_keys,"datanestkeys":data1_nestkeys});
parametersobject_O.append(formtileparameters_O);
tile2datamap_O.update({"filtermenu1":[0]});
if not single_plot_I:
rowcnt = 1;
colcnt = 1;
for cnt,sn in enumerate(sample_abbreviations):
svgtileid = "tilesvg"+str(cnt);
svgid = 'svg'+str(cnt);
iter=cnt+1; #start at 1
if (cnt % 2 == 0):
rowcnt = rowcnt+1;#even
colcnt = 1;
else:
colcnt = colcnt+1;
# make the svg object
svgparameters1_O = {"svgtype":'verticalbarschart2d_01',"svgkeymap":[data1_keymap],
#'svgid':'svg1',
'svgid':'svg'+str(cnt),
"svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 },
"svgwidth":350,"svgheight":250,"svgy1axislabel":"intensity (norm)",
"svgfilters":{'sample_name_abbreviation':[sn]}
};
svgtileparameters1_O = {'tileheader':'Isotopomer distribution','tiletype':'svg',
'tileid':svgtileid,
'rowid':"row"+str(rowcnt),
'colid':"col"+str(colcnt),
'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-6"};
svgtileparameters1_O.update(svgparameters1_O);
parametersobject_O.append(svgtileparameters1_O);
tile2datamap_O.update({svgtileid:[0]});
else:
cnt = 0;
svgtileid = "tilesvg"+str(cnt);
svgid = 'svg'+str(cnt);
rowcnt = 2;
colcnt = 1;
# make the svg object
svgparameters1_O = {"svgtype":'verticalbarschart2d_01',"svgkeymap":[data1_keymap],
'svgid':'svg'+str(cnt),
"svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 },
"svgwidth":500,"svgheight":350,"svgy1axislabel":"intensity (norm)",
};
svgtileparameters1_O = {'tileheader':'Isotopomer distribution','tiletype':'svg',
'tileid':svgtileid,
'rowid':"row"+str(rowcnt),
'colid':"col"+str(colcnt),
'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"};
svgtileparameters1_O.update(svgparameters1_O);
parametersobject_O.append(svgparameters1_O);
tile2datamap_O.update({svgtileid:[0]});
# make the table object
tableparameters1_O = {"tabletype":'responsivetable_01',
'tableid':'table1',
"tablefilters":None,
#"tableheaders":[],
"tableclass":"table table-condensed table-hover",
'tableformtileid':'tile1','tableresetbuttonid':'reset1','tablesubmitbuttonid':'submit1'};
tabletileparameters1_O = {'tileheader':'Isotopomer distribution','tiletype':'table','tileid':"tabletile1",
'rowid':"row100",
'colid':"col1",
'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"};
tabletileparameters1_O.update(tableparameters1_O);
dataobject_O.append({"data":data_table_O,"datakeys":data1_keys,"datanestkeys":data1_nestkeys});
parametersobject_O.append(tabletileparameters1_O);
tile2datamap_O.update({"tabletile1":[1]})
nsvgtable = ddt_container_filterMenuAndChart2dAndTable();
nsvgtable.make_filterMenuAndChart2dAndTable(
data_filtermenu=data_O,
data_filtermenu_keys=data1_keys,
data_filtermenu_nestkeys=data1_nestkeys,
data_filtermenu_keymap=data1_keymap,
data_svg_keys=data1_keys,
data_svg_nestkeys=data1_nestkeys,
data_svg_keymap=data1_keymap,
data_table_keys=None,
data_table_nestkeys=None,
data_table_keymap=None,
data_svg=data_dict_O,
data_table=None,
svgtype='verticalbarschart2d_01',
tabletype='responsivetable_01',
svgx1axislabel='',
svgy1axislabel='',
tablekeymap = [data1_keymap],
svgkeymap = [], #calculated on the fly
formtile2datamap=[0],
tabletile2datamap=[0],
svgtile2datamap=[], #calculated on the fly
svgfilters=None,
svgtileheader='Isotopomer distribution',
tablefilters=None,
tableheaders=None
);
if data_dir_I=='tmp':
filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js'
elif data_dir_I=='data_json':
data_json_O = nsvgtable.get_allObjects_js();
return data_json_O;
with open(filename_str,'w') as file:
file.write(nsvgtable.get_allObjects());
def execute_updatePeakSpectrum(self,experiment_id_I,sample_name_abbreviations_I = None):
'''re-calculate intensity_normalized from intensity_corrected and used'''
mids = mass_isotopomer_distributions();
# extract out the peakSpectrum
dataListUpdated = [];
# get sample names for the experiment
print('execute_updatePeakSpectrum...')
if sample_name_abbreviations_I:
sample_names = [];
sample_types = ['Unknown','QC'];
sample_types_lst = [];
for sna in sample_name_abbreviations_I:
for st in sample_types:
sample_names_tmp = [];
sample_names_tmp = self.get_sampleNames_experimentIDAndSampleTypeAndSampleNameAbbreviation_peakSpectrum(experiment_id_I,st,sna);
sample_names.extend(sample_names_tmp);
sample_types_lst.extend([st for i in range(len(sample_names_tmp))]);
else:
sample_names = [];
sample_types = ['Unknown','QC'];
sample_types_lst = [];
for st in sample_types:
sample_names_tmp = [];
sample_names_tmp = self.get_sampleNames_experimentIDAndSampleType_peakSpectrum(experiment_id_I,st);
sample_names.extend(sample_names_tmp);
sample_types_lst.extend([st for i in range(len(sample_names_tmp))]);
# create database table
for sn_cnt,sn in enumerate(sample_names):
print('updating peak spectrum for sample ' + sn);
# get other information about the sample for later use
sample_name_abbreviation,time_point,replicate_numbers = None,None,None;
sample_name_abbreviation,time_point,replicate_numbers = self.get_sampleNameAbbreviationsAndTimePointAndReplicateNumber_experimentIDAndSampleName_peakSpectrum(experiment_id_I,sn);
# get met_id and precursor_formula for each sample
scan_type = [];
scan_type = self.get_scanType_experimentIDAndSampleName_peakSpectrum(experiment_id_I,sn);
for scantype in scan_type:
print('building spectrum for scan type ' + scantype);
# get met_id and precursor formula for each sample
met_id, precursor_formula = [], [];
met_id, precursor_formula = self.get_metIDAndPrecursorFormula_experimentIDAndSampleNameAndScanType_peakSpectrum(experiment_id_I,sn,scantype);
for precursor_cnt, precursor in enumerate(precursor_formula):
print('updating peak spectrum for met_id/precursor ' + met_id[precursor_cnt] + '/' + precursor);
precursor_str = re.sub('[+-]', '', precursor);
precursor_mass = Formula(precursor_str).isotope.mass
# get all product fragments for the met_id/precursor
precursor_formulas_monoisotopic, product_formulas = [], [];
precursor_formulas_monoisotopic, product_formulas = self.get_precursorAndProductFormulas_metID(met_id[precursor_cnt],'-','tuning');
product_formulas.append(precursor_formulas_monoisotopic[0]); # add precursor to list of fragments
# get peak data for the sample/met_id/precursor_formula
peak_data = [];
peak_data = self.get_data_experimentIDAndSampleNameAndMetIDAndPrecursorFormulaAndScanType_peakSpectrum(experiment_id_I,sn,met_id[precursor_cnt],precursor,scantype);
peakSpectrum_corrected, peakSpectrum_normalized = mids.extract_peakList_normMax(\
peak_data, product_formulas,True);
peakSpectrum_stats,peakSpectrum_theoretical = mids.compare_peakSpectrum_normMax([peakSpectrum_normalized],True);
# update data_stage01_isotopomer_peakSpectrum
for frag,spec in peakSpectrum_theoretical.items():
if spec:
product_str = re.sub('[+-]', '', frag);
product_mass = Formula(product_str).isotope.mass;
for k,v in peakSpectrum_theoretical[frag].items():
dataListUpdated.append({'experiment_id':experiment_id_I,
'sample_name':sn,
'sample_name_abbreviation':sample_name_abbreviation,
'sample_type':sample_types_lst[sn_cnt],
'time_point':time_point,
'replicate_number':replicate_numbers,
'met_id':met_id[precursor_cnt],
'precursor_formula':precursor,
'precursor_mass':int(numpy.round(precursor_mass)),
'product_formula':frag,
'product_mass':int(numpy.round(k)),
'intensity_corrected':peakSpectrum_corrected[frag][k],
'intensity_corrected_units':'cps',
'intensity_normalized':peakSpectrum_normalized[frag][k],
'intensity_normalized_units':'normMax',
'intensity_theoretical':v,
'abs_devFromTheoretical':peakSpectrum_stats[frag][k]['absDev'],
'scan_type':scantype});
self.update_data_stage01_isotopomer_peakSpectrum(dataListUpdated);
def execute_buildSpectrumFromPeakData(self,experiment_id_I,ms_methodtype_I='isotopomer_13C',sample_name_abbreviations_I = None,met_ids_I = None):
'''Build spectrum from raw peak data'''
'''Assumptions:
Only 1 precursur:spectrum per sample name and
only 1 precursor:spectrum per dilution
(i.e. the best/most representative precursor:spectrum was chose from the
available EPI scans and dilutions of that particular precursor)
'''
mids = mass_isotopomer_distributions();
# extract out the peakSpectrum
# get sample names for the experiment
print('execute_buildSpectrumFromPeakData...')
if sample_name_abbreviations_I:
sample_names = [];
sample_types = ['Unknown','QC'];
sample_types_lst = [];
for sna in sample_name_abbreviations_I:
for st in sample_types:
sample_names_tmp = [];
sample_names_tmp = self.get_sampleNames_experimentIDAndSampleTypeAndSampleNameAbbreviation_peakData(experiment_id_I,st,sna);
sample_names.extend(sample_names_tmp);
sample_types_lst.extend([st for i in range(len(sample_names_tmp))]);
else:
sample_names = [];
sample_types = ['Unknown','QC'];
sample_types_lst = [];
for st in sample_types:
sample_names_tmp = [];
sample_names_tmp = self.get_sampleNames_experimentIDAndSampleType_peakData(experiment_id_I,st);
sample_names.extend(sample_names_tmp);
sample_types_lst.extend([st for i in range(len(sample_names_tmp))]);
# create database table
for sn_cnt,sn in enumerate(sample_names):
print('building spectrum for sample ' + sn);
# get other information about the sample for later use
sample_name_abbreviation,time_point,replicate_numbers = None,None,None;
sample_name_abbreviation,time_point,replicate_numbers = self.get_sampleNameAbbreviationsAndOther_experimentIDAndSampleName_peakData(experiment_id_I,sn);
# get met_id and precursor_formula for each sample
scan_type = [];
scan_type = self.get_scanType_experimentIDAndSampleName_peakData(experiment_id_I,sn);
for scantype in scan_type:
print('building spectrum for scan type ' + scantype);
# get met_id and precursor formula for each sample
if met_ids_I:
met_id, precursor_formula = [], [];
for met in met_ids_I:
met_id_tmp, precursor_formula_tmp = [], []
met_id_tmp, precursor_formula_tmp = self.get_metIDAndPrecursorFormula_experimentIDAndSampleNameAndScanTypeAndMetID_peakData(experiment_id_I,sn,scantype,met);
met_id.extend(met_id_tmp);
precursor_formula.extend(precursor_formula_tmp);
else:
met_id, precursor_formula = [], [];
met_id, precursor_formula = self.get_metIDAndPrecursorFormula_experimentIDAndSampleNameAndScanType_peakData(experiment_id_I,sn,scantype);
for precursor_cnt, precursor in enumerate(precursor_formula):
print('building spectrum for met_id/precursor ' + met_id[precursor_cnt] + '/' + precursor);
precursor_str = re.sub('[+-]', '', precursor);
precursor_mass = Formula(precursor_str).isotope.mass
# get all product fragments for the met_id/precursor
precursor_formulas_monoisotopic, product_formulas = [], [];
precursor_formulas_monoisotopic, product_formulas = self.get_precursorAndProductFormulas_metID(met_id[precursor_cnt],'-','tuning');
product_formulas.append(precursor_formulas_monoisotopic[0]); # add precursor to list of fragments
# get peak data for the sample/met_id/precursor_formula
peak_data = [];
peak_data = self.get_data_experimentIDAndSampleNameAndMetIDAndPrecursorFormulaAndScanType_peakData(experiment_id_I,sn,met_id[precursor_cnt],precursor,scantype);
peakSpectrum_measured,\
peakSpectrum_corrected, peakSpectrum_normalized = mids.extract_peakData_normMax(\
peak_data, product_formulas, 0.3, True);
peakSpectrum_stats,peakSpectrum_theoretical = mids.compare_peakSpectrum_normMax([peakSpectrum_normalized],True);
# update data_stage01_isotopomer_normalized
for frag,spec in peakSpectrum_theoretical.items():
if spec:
product_str = re.sub('[+-]', '', frag);
product_mass = Formula(product_str).isotope.mass;
for k,v in peakSpectrum_theoretical[frag].items():
row1 = None;
row1 = data_stage01_isotopomer_peakSpectrum(experiment_id_I,sn,sample_name_abbreviation,
sample_types_lst[sn_cnt],time_point,replicate_numbers,
met_id[precursor_cnt],precursor,int(numpy.round(precursor_mass)),
frag,int(numpy.round(k)),
peakSpectrum_measured[frag][k],'cps',
peakSpectrum_corrected[frag][k],'cps',
peakSpectrum_normalized[frag][k],'normMax',
v,peakSpectrum_stats[frag][k]['absDev'],scantype,True,None);
self.session.add(row1);
self.session.commit();
def export_dataStage01IsotopomerNormalized_js(self,experiment_id_I,sample_names_I=[],sample_name_abbreviations_I=[],time_points_I=[],scan_types_I=[],met_ids_I=[],data_dir_I="tmp",
single_plot_I = True):
"""Export data_stage01_isotopomer_normalized to js file"""
mids = mass_isotopomer_distributions();
# get the data
data_O = [];
data_dict_O = {};
sample_names_O = [];
# get time points
if time_points_I:
time_points = time_points_I;
else:
time_points = self.get_timePoint_experimentID_dataStage01Normalized(experiment_id_I);
for tp in time_points:
print('Plotting precursor and product spectrum from isotopomer normalized for time-point ' + str(tp));
if sample_names_I:
sample_abbreviations = [];
sample_types = ['Unknown','QC'];
sample_types_lst = [];
for sn in sample_names_I:
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndSampleName_dataStage01Normalized(experiment_id_I,st,tp,sn);
sample_abbreviations.extend(sample_abbreviations_tmp);
sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]);
elif sample_name_abbreviations_I:
sample_abbreviations = [];
sample_types = ['Unknown','QC'];
sample_types_lst = [];
for sn in sample_name_abbreviations_I:
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndSampleNameAbbreviation_dataStage01Normalized(experiment_id_I,st,tp,sn);
sample_abbreviations.extend(sample_abbreviations_tmp);
sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]);
# query sample types from sample name abbreviations and time-point from data_stage01_isotopomer_normalized
else:
# get sample names and sample name abbreviations
sample_abbreviations = [];
sample_types = ['Unknown','QC'];
sample_types_lst = [];
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01Normalized(experiment_id_I,st,tp);
sample_abbreviations.extend(sample_abbreviations_tmp);
sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]);
for sna_cnt,sna in enumerate(sample_abbreviations):
print('Plotting precursor and product spectrum from isotopomer normalized for sample name abbreviation ' + sna);
# get the scan_types
if scan_types_I:
scan_types = [];
scan_types_tmp = [];
scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01Normalized(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
scan_types = [st for st in scan_types_tmp if st in scan_types_I];
else:
scan_types = [];
scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01Normalized(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
for scan_type in scan_types:
print('Plotting precursor and product spectrum for scan type ' + scan_type)
# met_ids
if not met_ids_I:
met_ids = [];
met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01Normalized( \
experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type);
else:
met_ids = met_ids_I;
if not(met_ids): continue #no component information was found
for met in met_ids:
print('Plotting precursor and product spectrum for metabolite ' + met);
if sample_names_I:
sample_names = sample_names_I;
else:
sample_names,replicate_numbers,sample_types = [],[],[];
sample_names,replicate_numbers,sample_types = self.get_sampleNamesAndReplicateNumbersAndSampleTypes_experimentIDAndSampleNameAbbreviationAndMetIDAndTimePointAndScanType_dataStage01Normalized( \
experiment_id_I,sna,met,tp,scan_type);
if not(replicate_numbers): continue; #no replicates found
for rep_cnt,rep in enumerate(replicate_numbers):
#if not(sample_names): continue; #no replicates found
#for sn_cnt,sn in enumerate(sample_names):
print('Plotting precursor and product spectrum for replicate_number ' + str(rep));
#print('Plotting precursor and product spectrum for sample_name ' + sn);
#sample_names_O.append(sn);
#get data
peakData_I = {};
peakData_I = self.get_dataNormalized_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetIDAndReplicateNumber_dataStage01Normalized( \
experiment_id_I,sna,tp,scan_type,met,rep);
#peakData_I = self.get_dataNormalized_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetIDAndSampleName_dataStage01Normalized( \
# experiment_id_I,sna,tp,scan_type,met,sn);
if peakData_I:
data_sample_names_tmp=[];
fragment_formulas = list(peakData_I.keys());
peakSpectrum_corrected, peakSpectrum_normalized = mids.extract_peakList_normMax(\
peakData_I, fragment_formulas, True);
for fragment_formula in fragment_formulas:
for fragment_mass,intensity_normalized in peakSpectrum_normalized[fragment_formula].items():
sample_name = sna + "_" + str(rep);
sample_names_O.append(sample_name);
fragment_id = mids.make_fragmentID(met,fragment_formula,fragment_mass);
intensity = 0.0;
if intensity_normalized:
intensity = intensity_normalized;
data_tmp = {
'experiment_id':experiment_id_I,
'sample_name':sample_name,
#'sample_name':sn,
'sample_name_abbreviation':sna,
'sample_type':sample_types_lst[sna_cnt],
'time_point':tp,
#'dilution':dil,
'replicate_number':rep,
'met_id':met,
'fragment_formula':fragment_formula,
'fragment_mass':fragment_mass,
'intensity_normalized':intensity,
'intensity_normalized_units':"normMax",
'scan_type':scan_type,
'fragment_id':fragment_id};
data_O.append(data_tmp);
if not sample_name in data_dict_O.keys():
data_dict_O[sample_name] = [];
data_dict_O[sample_name].append(data_tmp);
else:
data_dict_O[sample_name].append(data_tmp);
# record the unique sample names:
sample_names_unique = list(set(sample_names_O));
sample_names_unique.sort();
# get the table data
data_table_O = [];
if time_points_I:
time_points = time_points_I;
else:
time_points = self.get_timePoint_experimentID_dataStage01Normalized(experiment_id_I);
for tp in time_points:
print('Tabulating precursor and product spectrum from isotopomer normalized for time-point ' + str(tp));
dataListUpdated = [];
# get dilutions
dilutions = [];
dilutions = self.get_sampleDilution_experimentIDAndTimePoint_dataStage01Normalized(experiment_id_I,tp);
for dil in dilutions:
print('Tabulating precursor and product spectrum from isotopomer normalized for dilution ' + str(dil));
if sample_names_I:
sample_abbreviations = [];
sample_types = ['Unknown','QC'];
for sn in sample_names_I:
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndDilutionAndSampleName_dataStage01Normalized(experiment_id_I,st,tp,dil,sn);
sample_abbreviations.extend(sample_abbreviations_tmp);
elif sample_name_abbreviations_I:
sample_abbreviations = sample_name_abbreviations_I;
else:
# get sample names and sample name abbreviations
sample_abbreviations = [];
sample_types = ['Unknown','QC'];
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndDilution_dataStage01Normalized(experiment_id_I,st,tp,dil);
sample_abbreviations.extend(sample_abbreviations_tmp);
for sna_cnt,sna in enumerate(sample_abbreviations):
print('Tabulating precursor and product spectrum from isotopomer normalized for sample name abbreviation ' + sna);
# get the scan_types
if scan_types_I:
scan_types = scan_types_I;
else:
scan_types = [];
scan_types = self.get_scanTypes_experimentIDAndTimePointAndDilutionAndSampleAbbreviations_dataStage01Normalized(experiment_id_I,tp,dil,sna);
for scan_type in scan_types:
print('Tabulting precursor and product spectrum for scan type ' + scan_type)
# met_ids
if not met_ids_I:
met_ids = [];
met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndDilutionAndScanType_dataStage01Normalized( \
experiment_id_I,sna,tp,dil,scan_type);
else:
met_ids = met_ids_I;
if not(met_ids): continue #no component information was found
for met in met_ids:
# get the data
data = [];
data = self.get_rows_experimentIDAndSampleAbbreviationAndTimePointAndDilutionAndScanTypeAndMetID_dataStage01Normalized( \
experiment_id_I,sna,tp,dil,scan_type,met);
for d in data:
d['sample_name'] = d['sample_name_abbreviation']+"_"+str(d['replicate_number']);
d['fragment_id'] = mids.make_fragmentID(d['met_id'],d['fragment_formula'],d['fragment_mass']);
data_table_O.extend(data);
# visualization parameters
data1_keys = ['sample_name','sample_name_abbreviation',
'sample_type',
'met_id','time_point','fragment_formula','fragment_mass','scan_type','fragment_id'];
data1_nestkeys = [
#'fragment_id',
'fragment_mass'
];
data1_keymap = {
#'xdata':'fragment_id',
'xdata':'fragment_mass',
'ydata':'intensity_normalized',
'serieslabel':'scan_type',
#'serieslabel':'sample_name', #single plot
#'featureslabel':'fragment_id',
'featureslabel':'fragment_mass',
'tooltipdata':'sample_name_abbreviation',
'ydatalb':None,
'ydataub':None};
# initialize the ddt objects
dataobject_O = [];
parametersobject_O = [];
tile2datamap_O = {};
# make the tile parameter objects
formtileparameters_O = {'tileheader':'Filter menu','tiletype':'html','tileid':"filtermenu1",'rowid':"row1",'colid':"col1",
'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"};
formparameters_O = {'htmlid':'filtermenuform1',"htmltype":'form_01',"formsubmitbuttonidtext":{'id':'submit1','text':'submit'},"formresetbuttonidtext":{'id':'reset1','text':'reset'},"formupdatebuttonidtext":{'id':'update1','text':'update'}};
formtileparameters_O.update(formparameters_O);
dataobject_O.append({"data":data_O,"datakeys":data1_keys,"datanestkeys":data1_nestkeys});
parametersobject_O.append(formtileparameters_O);
tile2datamap_O.update({"filtermenu1":[0]});
if not single_plot_I:
rowcnt = 1;
colcnt = 1;
cnt = 0;
for sn in sample_names_unique:
svgtileid = "tilesvg"+str(cnt);
svgid = 'svg'+str(cnt);
iter=cnt+1; #start at 1
if (cnt % 2 == 0):
rowcnt = rowcnt+1;#even
colcnt = 1;
else:
colcnt = colcnt+1;
# make the svg object
svgparameters1_O = {"svgtype":'verticalbarschart2d_01',"svgkeymap":[data1_keymap],
#'svgid':'svg1',
'svgid':'svg'+str(cnt),
"svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 },
"svgwidth":350,"svgheight":250,"svgy1axislabel":"intensity (norm)",
};
svgtileparameters1_O = {'tileheader':sn,'tiletype':'svg',
#'tileid':"tile2",
'tileid':svgtileid,
#'rowid':"row1",
'rowid':"row"+str(rowcnt),
#'colid':"col1",
'colid':"col"+str(colcnt),
'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-6"};
svgtileparameters1_O.update(svgparameters1_O);
parametersobject_O.append(svgtileparameters1_O);
tile2datamap_O.update({svgtileid:[iter]});
dataobject_O.append({"data":data_dict_O[sn],"datakeys":data1_keys,"datanestkeys":data1_nestkeys});
cnt+=1;
else:
cnt = 0;
svgtileid = "tilesvg"+str(cnt);
svgid = 'svg'+str(cnt);
rowcnt = 2;
colcnt = 1;
# make the svg object
svgparameters1_O = {"svgtype":'verticalbarschart2d_01',"svgkeymap":[data1_keymap],
#'svgid':'svg1',
'svgid':'svg'+str(cnt),
"svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 },
"svgwidth":500,"svgheight":350,"svgy1axislabel":"intensity (norm)",
};
svgtileparameters1_O = {'tileheader':'Isotopomer distribution','tiletype':'svg',
#'tileid':"tile2",
'tileid':svgtileid,
#'rowid':"row1",
#'colid':"col1",
'rowid':"row"+str(rowcnt),
'colid':"col"+str(colcnt),
'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"};
svgtileparameters1_O.update(svgparameters1_O);
parametersobject_O.append(svgparameters1_O);
tile2datamap_O.update({svgtileid:[0]});
iter+=1;
# make the table object
tableparameters1_O = {"tabletype":'responsivetable_01',
'tableid':'table1',
"tablefilters":None,
#"tableheaders":[],
"tableclass":"table table-condensed table-hover",
'tableformtileid':'tile1','tableresetbuttonid':'reset1','tablesubmitbuttonid':'submit1'};
tabletileparameters1_O = {'tileheader':'Isotopomer distribution','tiletype':'table','tileid':"tabletile1",
'rowid':"row"+str(rowcnt+1),
'colid':"col1",
'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"};
tabletileparameters1_O.update(tableparameters1_O);
#dataobject_O.append({"data":data_table_O,"datakeys":data1_keys,"datanestkeys":data1_nestkeys});
parametersobject_O.append(tabletileparameters1_O);
tile2datamap_O.update({"tabletile1":[0]})
#filtermenunsvgtable = ddt_container_filterMenuNSvgTable();
#filtermenunsvgtable.make_filterMenuNSvgTable(
# data_filtermenu=data_O,
# data_filtermenu_keys=[data1_keymap],
# data_filtermenu_nestkeys=data1_nestkeys,
# data_filtermenu_keymap=[data1_keymap],
# data_svg_keys=[data1_keymap],
# data_svg_nestkeys=data1_nestkeys,
# data_svg_keymap=[data1_keymap],
# data_table_keys=data1_keys,
# data_table_nestkeys=data1_nestkeys,
# data_table_keymap=[data1_keymap],
# data_svg=data_dict_O,
# data_table=data_table_O,
# svgtype='verticalbarschart2d_01',
# tabletype='responsivetable_01',
# svgx1axislabel=None,
# svgy1axislabel="intensity (norm)",
# single_plot_I=True,
# formtile2datamap=[0],
# tabletile2datamap=[1],
# svgtile2datamap=None,
# svgfilters=None,
# svgtileheader='Isotopomer distribution',
# tablefilters=None,
# tableheaders=None
# );
# dump the data to a json file
ddtutilities = ddt_container(parameters_I = parametersobject_O,data_I = dataobject_O,tile2datamap_I = tile2datamap_O,filtermenu_I = None);
if data_dir_I=='tmp':
filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js'
elif data_dir_I=='data_json':
data_json_O = ddtutilities.get_allObjects_js();
return data_json_O;
with open(filename_str,'w') as file:
file.write(ddtutilities.get_allObjects());
def plot_normalizedSpectrumNormSum(self,experiment_id_I, sample_names_I = None, sample_name_abbreviations_I = None, met_ids_I = None, scan_types_I = None):
'''calculate the average normalized intensity for all samples and scan types'''
'''Assumptions:
only a single fragment:spectrum is used_ per sample name abbreviation, time-point, replicate, scan_type
(i.e. there are no multiple dilutions of the same precursor:spectrum that are used_)
'''
mids = mass_isotopomer_distributions();
print('plot_normalizedSpectrumNormSum...')
plot = matplot();
# get time points
time_points = self.get_timePoint_experimentID_dataStage01Normalized(experiment_id_I);
for tp in time_points:
print('Plotting precursor and product spectrum from isotopomer normalized for time-point ' + str(tp));
if sample_names_I:
sample_abbreviations = [];
sample_types = ['Unknown','QC'];
sample_types_lst = [];
for sn in sample_names_I:
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndSampleName_dataStage01Normalized(experiment_id_I,st,tp,sn);
sample_abbreviations.extend(sample_abbreviations_tmp);
sample_types_lst.extend([st for i in range(len(sample_names_tmp))]);
elif sample_name_abbreviations_I:
sample_abbreviations = sample_name_abbreviations_I;
sample_types_lst = ['Unknown' for x in sample_abbreviations];
# query sample types from sample name abbreviations and time-point from data_stage01_isotopomer_normalized
else:
# get sample names and sample name abbreviations
sample_abbreviations = [];
sample_types = ['Unknown','QC'];
sample_types_lst = [];
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01Normalized(experiment_id_I,st,tp);
sample_abbreviations.extend(sample_abbreviations_tmp);
sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]);
for sna_cnt,sna in enumerate(sample_abbreviations):
print('Plotting precursor and product spectrum from isotopomer normalized for sample name abbreviation ' + sna);
# get the scan_types
if scan_types_I:
scan_types = [];
scan_types_tmp = [];
scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01Normalized(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
scan_types = [st for st in scan_types_tmp if st in scan_types_I];
else:
scan_types = [];
scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01Normalized(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
for scan_type in scan_types:
print('Plotting precursor and product spectrum for scan type ' + scan_type)
# met_ids
if not met_ids_I:
met_ids = [];
met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01Normalized( \
experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type);
else:
met_ids = met_ids_I;
if not(met_ids): continue #no component information was found
for met in met_ids:
print('Plotting precursor and product spectrum for metabolite ' + met);
# get replicates
replicate_numbers = [];
replicate_numbers = self.get_replicateNumbers_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetID_dataStage01Normalized( \
experiment_id_I,sna,tp,scan_type,met);
peakSpectrum_normalized_lst = [];
for rep in replicate_numbers:
print('Plotting precursor and product spectrum for replicate_number ' + str(rep));
#get data
peakData_I = {};
peakData_I = self.get_dataNormalized_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetIDAndReplicateNumber_dataStage01Normalized( \
experiment_id_I,sna,tp,scan_type,met,rep);
fragment_formulas = list(peakData_I.keys());
peakSpectrum_corrected, peakSpectrum_normalized = mids.extract_peakList_normSum(\
peakData_I, fragment_formulas, True);
peakSpectrum_normalized_lst.append(peakSpectrum_normalized);
# plot spectrum data for all replicates and fragments
fragment_formulas_unique = list(set(fragment_formulas_lst));
for fragment in fragment_formulas_unique:
panelLabels = [];
xticklabels = [];
mean = [];
xlabel = 'm/z'
ylabel = 'intensity'
for rep,spectrum in enumerate(peakSpectrum_normalized_lst):
panelLabels_tmp = sna+'_'+met+'_'+fragment+'_'+str(rep+1)
xticklabels_tmp = [];
mean_tmp = [];
for mass,intensity in spectrum[fragment].items():
intensity_tmp = intensity;
if not intensity_tmp: intensity_tmp=0.0
mean_tmp.append(intensity_tmp);
xticklabels_tmp.append(mass);
panelLabels.append(panelLabels_tmp);
xticklabels.append(xticklabels_tmp);
mean.append(mean_tmp);
plot.multiPanelBarPlot('',xticklabels,xlabel,ylabel,panelLabels,mean);
def execute_analyzeAveragesNormSum(self,experiment_id_I, sample_names_I = None, sample_name_abbreviations_I = None, met_ids_I = None, scan_types_I = None):
'''calculate the average normalized intensity for all samples and scan types'''
'''Assumptions:
only a single fragment:spectrum is used_ per sample name abbreviation, time-point, replicate, scan_type
(i.e. there are no multiple dilutions of the same precursor:spectrum that are used_)
'''
mids = mass_isotopomer_distributions();
print('execute_analyzeAveragesNormSum...')
# get time points
time_points = self.get_timePoint_experimentID_dataStage01Normalized(experiment_id_I);
for tp in time_points:
print('Calculating average precursor and product spectrum from isotopomer normalized for time-point ' + str(tp));
if sample_names_I:
sample_abbreviations = [];
sample_types = ['Unknown','QC'];
sample_types_lst = [];
for sn in sample_names_I:
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndSampleName_dataStage01Normalized(experiment_id_I,st,tp,sn);
sample_abbreviations.extend(sample_abbreviations_tmp);
sample_types_lst.extend([st for i in range(len(sample_names_tmp))]);
elif sample_name_abbreviations_I:
sample_abbreviations = sample_name_abbreviations_I;
sample_types_lst = ['Unknown' for x in sample_abbreviations];
# query sample types from sample name abbreviations and time-point from data_stage01_isotopomer_normalized
else:
# get sample names and sample name abbreviations
sample_abbreviations = [];
sample_types = ['Unknown','QC'];
sample_types_lst = [];
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01Normalized(experiment_id_I,st,tp);
sample_abbreviations.extend(sample_abbreviations_tmp);
sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]);
for sna_cnt,sna in enumerate(sample_abbreviations):
print('Calculating average precursor and product spectrum from isotopomer normalized for sample name abbreviation ' + sna);
# get the scan_types
if scan_types_I:
scan_types = [];
scan_types_tmp = [];
scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01Normalized(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
scan_types = [st for st in scan_types_tmp if st in scan_types_I];
else:
scan_types = [];
scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01Normalized(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
for scan_type in scan_types:
print('Calculating average precursor and product spectrum for scan type ' + scan_type)
# met_ids
if not met_ids_I:
met_ids = [];
met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01Normalized( \
experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type);
else:
met_ids = met_ids_I;
if not(met_ids): continue #no component information was found
for met in met_ids:
print('Calculating average precursor and product spectrum for metabolite ' + met);
# get replicates
replicate_numbers = [];
replicate_numbers = self.get_replicateNumbers_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetID_dataStage01Normalized( \
experiment_id_I,sna,tp,scan_type,met);
peakSpectrum_normalized_lst = [];
for rep in replicate_numbers:
print('Calculating average precursor and product spectrum for replicate_number ' + str(rep));
#get data
peakData_I = {};
peakData_I = self.get_dataNormalized_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetIDAndReplicateNumber_dataStage01Normalized( \
experiment_id_I,sna,tp,scan_type,met,rep);
fragment_formulas = list(peakData_I.keys());
peakSpectrum_corrected, peakSpectrum_normalized = mids.extract_peakList_normSum(\
peakData_I, fragment_formulas, True);
peakSpectrum_normalized_lst.append(peakSpectrum_normalized);
peakSpectrum_stats,peakSpectrum_theoretical = mids.compare_peakSpectrum_normSum(peakSpectrum_normalized_lst,True);
# update data_stage01_isotopomer_normalized
for frag,spec in peakSpectrum_theoretical.items():
if spec:
fragment_str = re.sub('[+-]', '', frag);
fragment_mass = Formula(fragment_str).isotope.mass;
for k,v in peakSpectrum_theoretical[frag].items():
if v and k in peakSpectrum_stats[frag]:
if peakSpectrum_stats[frag][k]['mean']> 0.0: intensities_cv = peakSpectrum_stats[frag][k]['stdDev']/peakSpectrum_stats[frag][k]['mean']*100;
else: intensities_cv = 0.0;
row = [];
row = data_stage01_isotopomer_averagesNormSum(experiment_id_I, sna, sample_types_lst[sna_cnt], tp, met,frag, k,
peakSpectrum_stats[frag][k]['n'], peakSpectrum_stats[frag][k]['mean'], intensities_cv,
'normSum', v, peakSpectrum_stats[frag][k]['absDev'], scan_type, True);
elif v and k not in peakSpectrum_stats[frag]:
intensities_cv = None;
row = [];
row = data_stage01_isotopomer_averagesNormSum(experiment_id_I, sna, sample_types_lst[sna_cnt], tp, met,frag, k,
None, None, intensities_cv,
'normSum', v, None, scan_type, True);
elif not v and k in peakSpectrum_stats[frag]:
if peakSpectrum_stats[frag][k]['mean']> 0.0: intensities_cv = peakSpectrum_stats[frag][k]['stdDev']/peakSpectrum_stats[frag][k]['mean']*100;
else: intensities_cv = 0.0;
row = [];
row = data_stage01_isotopomer_averagesNormSum(experiment_id_I, sna, sample_types_lst[sna_cnt], tp, met,frag, k,
peakSpectrum_stats[frag][k]['n'], peakSpectrum_stats[frag][k]['mean'], intensities_cv,
'normSum', None, peakSpectrum_stats[frag][k]['absDev'], scan_type, True);
self.session.add(row);
self.session.commit();
def execute_analyzeAverages(self,experiment_id_I, sample_names_I = None, sample_name_abbreviations_I = None, met_ids_I = None, scan_types_I = None):
'''calculate the average normalized intensity for MRM samples'''
'''Assumptions:
only a single fragment:spectrum is used_ per sample name abbreviation, time-point, replicate, scan_type
(i.e. there are no multiple dilutions of the same precursor:spectrum that are used_)
'''
mids = mass_isotopomer_distributions();
print('execute_analyzeAverages...')
# get time points
time_points = self.get_timePoint_experimentID_dataStage01Normalized(experiment_id_I);
for tp in time_points:
print('Calculating average precursor and product spectrum from isotopomer normalized for time-point ' + str(tp));
if sample_names_I:
sample_abbreviations = [];
sample_types = ['Unknown','QC'];
sample_types_lst = [];
for sn in sample_names_I:
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndSampleName_dataStage01Normalized(experiment_id_I,st,tp,sn);
sample_abbreviations.extend(sample_abbreviations_tmp);
sample_types_lst.extend([st for i in range(len(sample_names_tmp))]);
elif sample_name_abbreviations_I:
sample_abbreviations = sample_name_abbreviations_I;
sample_types_lst = ['Unknown' for x in sample_abbreviations];
# query sample types from sample name abbreviations and time-point from data_stage01_isotopomer_normalized
else:
# get sample names and sample name abbreviations
sample_abbreviations = [];
sample_types = ['Unknown','QC'];
sample_types_lst = [];
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01Normalized(experiment_id_I,st,tp);
sample_abbreviations.extend(sample_abbreviations_tmp);
sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]);
for sna_cnt,sna in enumerate(sample_abbreviations):
print('Calculating average precursor and product spectrum from isotopomer normalized for sample name abbreviation ' + sna);
# get the scan_types
if scan_types_I:
scan_types = [];
scan_types_tmp = [];
scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01Normalized(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
scan_types = [st for st in scan_types_tmp if st in scan_types_I];
else:
scan_types = [];
scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01Normalized(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
for scan_type in scan_types:
print('Calculating average precursor and product spectrum for scan type ' + scan_type)
# met_ids
if not met_ids_I:
met_ids = [];
met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01Normalized( \
experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type);
else:
met_ids = met_ids_I;
if not(met_ids): continue #no component information was found
for met in met_ids:
print('Calculating average precursor and product spectrum for metabolite ' + met);
## get fragment formulas and masses
#fragment_formulas, fragment_masses = [],[];
#fragment_formulas,fragment_masses = self.get_fragmentFormulasAndMass_experimentIDAndSampleAbbreviationAndTimePointAndAndSampleTypeAndScanTypeAndMetID_dataStage01Normalized(experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met);
#for mass_cnt,mass in enumerate(fragment_masses):
# print 'Calculating average precursor and product spectrum for fragment/mass ' + fragment_formulas[mass_cnt] + '/' + str(mass);
# # get data
# intensities = [];
# intensities = self.get_normalizedIntensity_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndMetIDAndFragmentFormulaAndMassAndScanType_dataStage01Normalized(experiment_id_I,sna,tp,sample_types_lst[sna_cnt],met,fragment_formulas[mass_cnt],mass,scan_type);
# # calculate the average and cv
# n_replicates = len(intensities);
# intensities_average = 0.0;
# intensities_var = 0.0;
# intensities_cv = 0.0;
# # calculate average and CV of intensities
# if (not(intensities)):
# #continue
# intensities_average = 0.0;
# intensities_var = 0.0;
# intensities_cv = 0.0;
# elif n_replicates<2: # require at least 2 replicates
# #continue
# intensities_average = 0.0;
# intensities_var = 0.0;
# intensities_cv = 0.0;
# else:
# intensities_average = numpy.mean(numpy.array(intensities));
# intensities_var = numpy.var(numpy.array(intensities));
# if (intensities_average <= 0.0): intensities_cv = 0.0;
# else: intensities_cv = sqrt(intensities_var)/intensities_average*100;
# # calculate the theoretical spectrum for the pecursor/mass
# peakSpectrum_theoretical = mids.report_fragmentSpectrum_normMax([fragment_formulas[mass_cnt]],True);
# # calculate the absolute deviation from the theoretical
# intensity_theoretical = peakSpectrum_theoretical[fragment_formulas[mass_cnt]][mass];
# if intensity_theoretical > 0.0:abs_devFromTheoretical = abs(intensity_theoretical-intensities_average)/intensity_theoretical*100;
# else: abs_devFromTheoretical = None;
# # add to data_stage01_isotopomer_averages
# row = [];
# row = data_stage01_isotopomer_averages(experiment_id_I, sna, sample_types_lst[sna_cnt], tp, met,fragment_formulas[mass_cnt], mass,
# n_replicates, intensities_average, intensities_cv,
# 'normMax', intensity_theoretical, abs_devFromTheoretical, scan_type, True)
# self.session.add(row);
# get replicates
replicate_numbers = [];
replicate_numbers = self.get_replicateNumbers_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetID_dataStage01Normalized( \
experiment_id_I,sna,tp,scan_type,met);
peakSpectrum_normalized_lst = [];
for rep in replicate_numbers:
print('Calculating average precursor and product spectrum for replicate_number ' + str(rep));
#get data
peakData_I = {};
peakData_I = self.get_dataNormalized_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetIDAndReplicateNumber_dataStage01Normalized( \
experiment_id_I,sna,tp,scan_type,met,rep);
fragment_formulas = list(peakData_I.keys());
peakSpectrum_corrected, peakSpectrum_normalized = mids.extract_peakList_normMax(\
peakData_I, fragment_formulas, True);
peakSpectrum_normalized_lst.append(peakSpectrum_normalized);
peakSpectrum_stats,peakSpectrum_theoretical = mids.compare_peakSpectrum_normMax(peakSpectrum_normalized_lst,True);
# update data_stage01_isotopomer_averages
for frag,spec in peakSpectrum_theoretical.items():
if spec:
fragment_str = re.sub('[+-]', '', frag);
fragment_mass = Formula(fragment_str).isotope.mass;
for k,v in peakSpectrum_theoretical[frag].items():
if v and k in peakSpectrum_stats[frag]:
if peakSpectrum_stats[frag][k]['mean']> 0.0: intensities_cv = peakSpectrum_stats[frag][k]['stdDev']/peakSpectrum_stats[frag][k]['mean']*100;
else: intensities_cv = 0.0;
row = [];
row = data_stage01_isotopomer_averages(experiment_id_I, sna, sample_types_lst[sna_cnt], tp, met,frag, k,
peakSpectrum_stats[frag][k]['n'], peakSpectrum_stats[frag][k]['mean'], intensities_cv,
'normMax', v, peakSpectrum_stats[frag][k]['absDev'], scan_type, True);
elif v and k not in peakSpectrum_stats[frag]:
intensities_cv = None;
row = [];
row = data_stage01_isotopomer_averages(experiment_id_I, sna, sample_types_lst[sna_cnt], tp, met,frag, k,
None, None, intensities_cv,
'normMax', v, None, scan_type, True);
elif not v and k in peakSpectrum_stats[frag]:
if peakSpectrum_stats[frag][k]['mean']> 0.0: intensities_cv = peakSpectrum_stats[frag][k]['stdDev']/peakSpectrum_stats[frag][k]['mean']*100;
else: intensities_cv = 0.0;
row = [];
row = data_stage01_isotopomer_averages(experiment_id_I, sna, sample_types_lst[sna_cnt], tp, met,frag, k,
peakSpectrum_stats[frag][k]['n'], peakSpectrum_stats[frag][k]['mean'], intensities_cv,
'normMax', None, peakSpectrum_stats[frag][k]['absDev'], scan_type, True);
self.session.add(row);
self.session.commit();
def execute_recombineNormalizedSpectrum(self,experiment_id_I, sample_names_I = None, sample_name_abbreviations_I = None, met_ids_I = None):
'''recombine intensity_normalized from a lower and higher dilution'''
'''Assumptions:
only a single fragment:spectrum is used_ per sample name abbreviation, time-point, replicate, scan_type
(i.e. there are no multiple dilutions of the same precursor:spectrum that are used_)
'''
mids = mass_isotopomer_distributions();
print('execute_recombineNormalizedSpectrum...')
# get time points
time_points = self.get_timePoint_experimentIDAndComment_dataStage01Normalized(experiment_id_I,'Recombine');
for tp in time_points:
print('recombining spectrum for time-point ' + str(tp));
dataListUpdated = [];
if sample_names_I:
sample_abbreviations = [];
sample_types = ['Unknown','QC'];
for sn in sample_names_I:
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndSampleNameAndComment_dataStage01Normalized(experiment_id_I,st,tp,sn,'Recombine');
sample_abbreviations.extend(sample_abbreviations_tmp);
elif sample_name_abbreviations_I:
sample_abbreviations = sample_name_abbreviations_I;
else:
# get sample names and sample name abbreviations
sample_abbreviations = [];
sample_types = ['Unknown','QC'];
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndComment_dataStage01Normalized(experiment_id_I,st,tp,'Recombine');
sample_abbreviations.extend(sample_abbreviations_tmp);
for sna_cnt,sna in enumerate(sample_abbreviations):
print('recombining spectrum for sample name abbreviation ' + sna);
# get the scan_types
scan_types = [];
scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndComment_dataStage01Normalized(experiment_id_I,tp,sna,'Recombine');
for scan_type in scan_types:
print('recombining spectrum for scan type ' + scan_type)
# met_ids
if not met_ids_I:
met_ids = [];
met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndComment_dataStage01Normalized( \
experiment_id_I,sna,tp,scan_type,'Recombine');
else:
met_ids = met_ids_I;
if not(met_ids): continue #no component information was found
for met in met_ids:
print('recombining spectrum for metabolite ' + met);
# get replicates
replicate_numbers = [];
replicate_numbers = self.get_replicateNumbers_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetID_dataStage01Normalized( \
experiment_id_I,sna,tp,scan_type,met);
for rep in replicate_numbers:
print('recombining spectrum for replicate_number ' + str(rep));
#get data
peakData_I = {};
peakData_I = self.get_data_experimentIDAndSampleAbbreviationAndTimePointAndScanTypeAndMetIDAndReplicateNumber_dataStage01Normalized( \
experiment_id_I,sna,tp,scan_type,met,rep);
peakData_O,peakData_O_false,peakData_intensities_O = mids.recombine_dilutionsMRMs(peakData_I);
peakSpectrum_stats = mids.compare_peakSpectrum_normMax([peakData_intensities_O]);
# update data_stage01_isotopomer_normalized
for frag,spec in peakSpectrum_stats.items():
if spec:
fragment_str = re.sub('[+-]', '', frag);
fragment_mass = Formula(fragment_str).isotope.mass;
for k,v in peakSpectrum_stats[frag].items():
if int(numpy.round(k)) in peakData_O[frag]:
dataListUpdated.append({'experiment_id':experiment_id_I,
'sample_name_abbreviation':sna,
'time_point':tp,
'dilution':peakData_O[frag][int(numpy.round(k))]['dilution'],
'replicate_number':rep,
'met_id':met,
'fragment_formula':frag,
'fragment_mass':int(numpy.round(k)),
'intensity_normalized':peakData_O[frag][int(numpy.round(k))]['intensity'],
'intensity_normalized_units':'normMax',
'abs_devFromTheoretical':v['absDev'],
'scan_type':scan_type,
'used_':peakData_O[frag][int(numpy.round(k))]['used_'],
'comment_':peakData_O[frag][int(numpy.round(k))]['comment_']});
# update data_stage01_isotopomer_normalized (rows changed to false)
for frag,spec in peakData_O_false.items():
if spec:
fragment_str = re.sub('[+-]', '', frag);
fragment_mass = Formula(fragment_str).isotope.mass;
for k,v in peakData_O_false[frag].items():
if v:
dataListUpdated.append({'experiment_id':experiment_id_I,
'sample_name_abbreviation':sna,
'time_point':tp,
'dilution':v['dilution'],
'replicate_number':rep,
'met_id':met,
'fragment_formula':frag,
'fragment_mass':int(numpy.round(k)),
'intensity_normalized':v['intensity'],
'intensity_normalized_units':'normMax',
'abs_devFromTheoretical':None,
'scan_type':scan_type,
'used_':v['used_'],
'comment_':v['comment_']});
self.update_data_stage01_isotopomer_normalized(dataListUpdated);
def execute_updateNormalizedSpectrum(self,experiment_id_I, sample_names_I = None, sample_name_abbreviations_I = None, met_ids_I = None, scan_types_I = None):
'''re-calculate intensity_normalized from intensity_corrected and used'''
mids = mass_isotopomer_distributions();
print('execute_updateNormalizedSpectrum...')
# get time points
time_points = self.get_timePoint_experimentID_dataStage01Normalized(experiment_id_I);
for tp in time_points:
print('Building precursor and product spectrum from isotopomer normalized for time-point ' + str(tp));
dataListUpdated = [];
# get dilutions
dilutions = [];
dilutions = self.get_sampleDilution_experimentIDAndTimePoint_dataStage01Normalized(experiment_id_I,tp);
for dil in dilutions:
print('Building precursor and product spectrum from isotopomer normalized for dilution ' + str(dil));
if sample_names_I:
sample_abbreviations = [];
sample_types = ['Unknown','QC'];
for sn in sample_names_I:
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndDilutionAndSampleName_dataStage01Normalized(experiment_id_I,st,tp,dil,sn);
sample_abbreviations.extend(sample_abbreviations_tmp);
elif sample_name_abbreviations_I:
sample_abbreviations = sample_name_abbreviations_I;
else:
# get sample names and sample name abbreviations
sample_abbreviations = [];
sample_types = ['Unknown','QC'];
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndDilution_dataStage01Normalized(experiment_id_I,st,tp,dil);
sample_abbreviations.extend(sample_abbreviations_tmp);
for sna_cnt,sna in enumerate(sample_abbreviations):
print('Building precursor and product spectrum from isotopomer normalized for sample name abbreviation ' + sna);
# get the scan_types
if scan_types_I:
scan_types = scan_types_I;
else:
scan_types = [];
scan_types = self.get_scanTypes_experimentIDAndTimePointAndDilutionAndSampleAbbreviations_dataStage01Normalized(experiment_id_I,tp,dil,sna);
for scan_type in scan_types:
print('Building precursor and product spectrum for scan type ' + scan_type)
# met_ids
if not met_ids_I:
met_ids = [];
met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndDilutionAndScanType_dataStage01Normalized( \
experiment_id_I,sna,tp,dil,scan_type);
else:
met_ids = met_ids_I;
if not(met_ids): continue #no component information was found
for met in met_ids:
print('Building precursor and product spectrum from isotopomer normalized for metabolite ' + met);
# get sample names
sample_names = [];
sample_names,replicate_numbers,sample_types = self.get_sampleNamesAndReplicateNumbersAndSampleTypes_experimentIDAndSampleNameAbbreviationAndMetIDAndTimePointAndDilutionAndScanType_dataStage01Normalized(experiment_id_I,sna,met,tp,dil,scan_type);
# iterate through sample names then mets/fragments in order to calculate the spectrum for each sample and component
for sn_cnt,sn in enumerate(sample_names):
print('Building precursor and product spectrum from isotopomer normalized for sample ' + sn);
# get peak data for the sample/met_id/scan_type
peak_data = [];
peak_data = self.get_data_experimentIDAndSampleNameAndMetIDAndAndScanType_normalized(experiment_id_I,sn,met,scan_type);
fragment_formulas = list(peak_data.keys());
peakSpectrum_corrected, peakSpectrum_normalized = mids.extract_peakList_normMax(\
peak_data, fragment_formulas, True);
peakSpectrum_stats,peakSpectrum_theoretical = mids.compare_peakSpectrum_normMax([peakSpectrum_normalized],True);
# update data_stage01_isotopomer_normalized
for frag,spec in peakSpectrum_theoretical.items():
if spec:
fragment_str = re.sub('[+-]', '', frag);
fragment_mass = Formula(fragment_str).isotope.mass;
for k,v in peakSpectrum_theoretical[frag].items():
dataListUpdated.append({'experiment_id':experiment_id_I,
'sample_name':sn,
'sample_name_abbreviation':sna,
'sample_type':sample_types[sn_cnt],
'time_point':tp,
'dilution':dil,
'replicate_number':replicate_numbers[sn_cnt],
'met_id':met,
'fragment_formula':frag,
'fragment_mass':int(numpy.round(k)),
'intensity_corrected':peakSpectrum_corrected[frag][k],
'intensity_corrected_units':'cps',
'intensity_normalized':peakSpectrum_normalized[frag][k],
'intensity_normalized_units':'normMax',
'intensity_theoretical':v,
'abs_devFromTheoretical':peakSpectrum_stats[frag][k]['absDev'],
'scan_type':scan_type});
self.update_data_stage01_isotopomer_normalized(dataListUpdated);
def execute_buildSpectrumFromMRMs(self,experiment_id_I,ms_methodtype_I='isotopomer_13C',sample_name_abbreviations_I=[],sample_names_I=[],met_ids_I=[]):
'''Extract peak spectrum for each fragment from MRMs'''
# Input:
# experiment_id
# sample_names = (optional) list of specific samples
# Output:
# sample_name
# sample_id
# component_group_name
# component_name
# calculated_concentration
# calculated_concentration_units
# used_
# assumptions:
# 1. there is only spectrum of MRMs for each components
mids = mass_isotopomer_distributions();
print('build_precursorSpectrumFromMRMs...')
# get time points
time_points = self.get_timePoint_experimentID(experiment_id_I);
for tp in time_points:
print('Building precursor and product spectrum from MRMs for time-point ' + str(tp));
# get dilutions
dilutions = self.get_sampleDilution_experimentIDAndTimePoint(experiment_id_I,tp);
for dil in dilutions:
print('Building precursor and product spectrum from MRMs for dilution ' + str(dil));
if sample_names_I:
sample_abbreviations = [];
for sn in sample_names_I:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleName(experiment_id_I,sn);
sample_abbreviations.extend(sample_abbreviations_tmp);
elif sample_name_abbreviations_I:
sample_abbreviations = sample_name_abbreviations_I;
else:
# get sample names and sample name short
sample_abbreviations = [];
sample_types = ['Unknown','QC'];
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePointAndDilution(experiment_id_I,st,tp,dil);
sample_abbreviations.extend(sample_abbreviations_tmp);
for sna_cnt,sna in enumerate(sample_abbreviations):
print('Building precursor and product spectrum from MRMs for sample name abbreviation ' + sna);
##BUG alert:
##there is a potential bug whereby if the entire spectra per compound is not returned
##e.g. one of the samples is not returned because "used_" is set to false in isotopomer_MQResultsTable
##the spectra could potentially be shifted
##get_componentsNamesAndOther_experimentIDAndSampleNameAndMSMethodTypeAndTimePointAndDilution has been adjusted to
##return all component_names even if the sample from which it came is not "used_" as a temporary fix
##this works for C12 validation experiments, but
##the results of normalization will need to be monitored until a more robust method is established
##UPDATE to BUG:
##'met_cnt_max = len(component_names)-1' changed to 'met_cnt_max = len(component_names)' and
##get_sampleNamesAndReplicateNumbersAndSampleTypes_experimentIDAndSampleNameAbbreviationAndSampleDescriptionAndTimePointAndDilution added
##so that the last component is included in the met/fragment spectra even when not all component names have "used_"
##set to true in isotopomer_MQResultsTable
# component names, group names, fragment formula, and fragment mass
if met_ids_I:
component_names,component_group_names,\
precursor_formulas_O, precursor_masses_O,\
product_formulas_O, product_masses_O = [],[],[],[],[],[];
for met in met_ids_I:
component_names_tmp,component_group_names_tmp,\
precursor_formulas_tmp, precursor_masses_tmp,\
product_formulas_tmp, product_masses_tmp = [],[],[],[],[],[];
component_names_tmp,component_group_names_tmp,\
precursor_formulas_tmp, precursor_masses_tmp,\
product_formulas_tmp, product_masses_tmp = \
self.get_componentsNamesAndOther_experimentIDAndSampleNameAndMSMethodTypeAndTimePointAndDilutionAndMetID( \
experiment_id_I,sna,ms_methodtype_I,tp,dil,met);
if not(component_names_tmp): continue #no component information was found
component_names.extend(component_names_tmp)
component_group_names.extend(component_group_names_tmp)
precursor_formulas_O.extend(precursor_formulas_tmp)
precursor_masses_O.extend(precursor_masses_tmp)
product_formulas_O.extend(product_formulas_tmp)
product_masses_O.extend(product_masses_tmp)
else:
component_names,component_group_names,\
precursor_formulas_O, precursor_masses_O,\
product_formulas_O, product_masses_O = [],[],[],[],[],[];
component_names,component_group_names,\
precursor_formulas_O, precursor_masses_O,\
product_formulas_O, product_masses_O = \
self.get_componentsNamesAndOther_experimentIDAndSampleNameAndMSMethodTypeAndTimePointAndDilution( \
experiment_id_I,sna,ms_methodtype_I,tp,dil);
if not(component_names): continue #no component information was found
# extract unique met ids and precursor formula id
met_ids_unique = [];
met_ids = [];
#precursor_formulas_unique = [];
#product_formulas_unique = [];
met_id = '';
met_id_old = '';
# algorithm works because lists are ordered by component_names
for i,cn in enumerate(component_names):
met_id = cn.split('.')[0];
met_ids.append(met_id);
if met_id != met_id_old:
met_id_old = met_id;
met_ids_unique.append(met_id);
#precursor_formulas_unique.append(precursor_formulas_O[i]);
#product_formulas_unique.append(product_formulas_O[i]);
# get precursor and productformulas for each unique met id:
precursor_formulas_unique = [];
product_formulas_unique = [];
for met in met_ids_unique:
precursor_formula,product_formula = None,None;
precursor_formula,product_formula = self.get_precursorFormulaAndProductFormula_metID(met,'-','isotopomer_13C')
precursor_formulas_unique.append(precursor_formula);
product_formulas_unique.append(product_formula);
# build precursor and spectrum for each met
met_all_cnt = 0;
met_cnt_max = len(component_names); # for use in a while loop
for i,met in enumerate(met_ids_unique):
print('Building precursor and product spectrum from MRMs for metabolite ' + met);
# get filtrate samples
precursorFiltrate_measured = {};
productFiltrate_measured = {};
#precursorFiltrate_measured[precursor_formulas_unique[i]] = None; #keep track of all met_ids
precursorFiltrate = {};
productFiltrate = {};
met_cnt = met_all_cnt;
# iterate through mets/fragments then sample names in order to calculate the average for each component_name (fragment/mass)
while met_cnt < met_cnt_max and met==met_ids[met_cnt]:
# get filtrate sample names
sample_names = [];
replicate_numbers = [];
sample_description = 'Filtrate';
sample_names,replicate_numbers,sample_types = self.get_sampleNamesAndReplicateNumbersAndSampleTypes_experimentIDAndSampleNameAbbreviationAndSampleDescriptionAndComponentNameAndTimePointAndDilution(experiment_id_I,sna,sample_description,component_names[met_cnt],tp,dil);
intensities = [];
for sn_cnt,sn in enumerate(sample_names):
# get intensities
intensity = None;
intensity = self.get_peakHeight_sampleNameAndComponentName(sn,component_names[met_cnt]);
if not(intensity): continue
intensities.append(intensity);
n_replicates = len(intensities);
intensities_average_filtrate = 0.0;
intensities_var_filtrate = 0.0;
# calculate average and CV of the intensities
if (not(intensities)): intensities_average_filtrate = 0.0;
elif n_replicates<2: intensities_average_filtrate = intensities[0];
else:
#intensities_average_filtrate, intensities_var_filtrate = self.calculate.calculate_ave_var_R(intensities);
intensities_average_filtrate = numpy.mean(numpy.array(intensities));
intensities_var_filtrate = numpy.var(numpy.array(intensities));
# append value to dictionary
precursorFiltrate[(precursor_masses_O[met_cnt],product_masses_O[met_cnt])] = intensities_average_filtrate;
productFiltrate[(precursor_masses_O[met_cnt],product_masses_O[met_cnt])] = intensities_average_filtrate;
met_cnt += 1;
precursorFiltrate_measured[precursor_formulas_unique[i]] = precursorFiltrate
productFiltrate_measured[product_formulas_unique[i]] = productFiltrate
# get broth samples
sample_names = [];
sample_description = 'Broth';
sample_names,replicate_numbers,sample_types = self.get_sampleNamesAndReplicateNumbersAndSampleTypes_experimentIDAndSampleNameAbbreviationAndSampleDescriptionAndTimePointAndDilution(experiment_id_I,sna,sample_description,tp,dil);
#sample_names,replicate_numbers,sample_types = self.get_sampleNamesAndReplicateNumbersAndSampleTypes_experimentIDAndSampleNameAbbreviationAndSampleDescriptionAndComponentNameAndTimePointAndDilution(experiment_id_I,sna,sample_description,component_names[met_cnt],tp,dil);
# iterate through sample names then mets/fragments in order to calculate the spectrum for each sample and component
for sn_cnt,sn in enumerate(sample_names):
print('Building precursor and product spectrum from MRMs for sample ' + sn);
precursorPeakSpectrum_measured = {};
precursorPeakSpectrum_corrected = {};
productPeakSpectrum_measured = {};
productPeakSpectrum_corrected = {};
#precursorPeakSpectrum_measured[precursor_formulas_unique[i]] = None; #keep track of all met_ids
#precursorPeakSpectrum_corrected[precursor_formulas_unique[i]] = None; #keep track of all met_ids
precursorMeasured = {};
precursorCorrected = {};
productMeasured = {};
productCorrected = {};
met_cnt = met_all_cnt;
while met_cnt < met_cnt_max and met==met_ids[met_cnt]:
# get intensities
intensity = None;
intensity = self.get_peakHeight_sampleNameAndComponentName(sn,component_names[met_cnt]);
if not(intensity):
precursorMeasured[(precursor_masses_O[met_cnt],product_masses_O[met_cnt])] = 0.0;
productMeasured[(precursor_masses_O[met_cnt],product_masses_O[met_cnt])] = 0.0;
else:
precursorMeasured[(precursor_masses_O[met_cnt],product_masses_O[met_cnt])] = intensity;
productMeasured[(precursor_masses_O[met_cnt],product_masses_O[met_cnt])] = intensity;
#if precursorFiltrate_measured[precursor_formulas_unique[i]][(precursor_masses_O[met_cnt],product_masses_O[met_cnt])] < 0.5*intensity:
# corrected_intensity = intensity - precursorFiltrate_measured[precursor_formulas_unique[i]][(precursor_masses_O[met_cnt],product_masses_O[met_cnt])];
#else: corrected_intensity = 0.0;
#precursorCorrected[(precursor_masses_O[met_cnt],product_masses_O[met_cnt])] = corrected_intensity;
met_cnt += 1;
precursorPeakSpectrum_measured[precursor_formulas_unique[i]] = precursorMeasured;
productPeakSpectrum_measured[product_formulas_unique[i]] = productMeasured;
# generate normalized spectrum for the precursor:
precursorPeakSpectrum_measured, precursorPeakSpectrum_corrected, precursorPeakSpectrum_normalized \
= mids.build_precursorSpectrumFromMRMs(precursorPeakSpectrum_measured,precursorFiltrate_measured);
peakSpectrum_stats_O,precursorPeakSpectrum_theoretical = mids.compare_peakSpectrum_normMax([precursorPeakSpectrum_normalized],True);
# update data_stage01_isotopomer_normalized
if precursorPeakSpectrum_theoretical[precursor_formulas_unique[i]]:
for k,v in precursorPeakSpectrum_theoretical[precursor_formulas_unique[i]].items():
row1 = None;
row1 = data_stage01_isotopomer_normalized(experiment_id_I,sn,sna,sample_types[sn_cnt],tp,dil,replicate_numbers[sn_cnt],
met,precursor_formulas_unique[i],int(numpy.round(k)),
precursorPeakSpectrum_measured[precursor_formulas_unique[i]][k],'cps',
precursorPeakSpectrum_corrected[precursor_formulas_unique[i]][k],'cps',
precursorPeakSpectrum_normalized[precursor_formulas_unique[i]][k],'normMax',
v,peakSpectrum_stats_O[precursor_formulas_unique[i]][k]['absDev'],'MRM',True,None);
self.session.add(row1);
# generate normalized spectrum for the product:
productPeakSpectrum_measured, productPeakSpectrum_corrected, productPeakSpectrum_normalized \
= mids.build_productSpectrumFromMRMs(productPeakSpectrum_measured,productFiltrate_measured);
peakSpectrum_stats_O,productPeakSpectrum_theoretical = mids.compare_peakSpectrum_normMax([productPeakSpectrum_normalized],True);
# update data_stage01_isotopomer_normalized
if productPeakSpectrum_theoretical[product_formulas_unique[i]]:
for k,v in productPeakSpectrum_theoretical[product_formulas_unique[i]].items():
row2 = None;
row2 = data_stage01_isotopomer_normalized(experiment_id_I,sn,sna,sample_types[sn_cnt],tp,dil,replicate_numbers[sn_cnt],
met,product_formulas_unique[i],int(numpy.round(k)),
productPeakSpectrum_measured[product_formulas_unique[i]][k],'cps',
productPeakSpectrum_corrected[product_formulas_unique[i]][k],'cps',
productPeakSpectrum_normalized[product_formulas_unique[i]][k],'normMax',
v,peakSpectrum_stats_O[product_formulas_unique[i]][k]['absDev'],'MRM',True,None);
self.session.add(row2);
met_all_cnt = met_cnt
self.session.commit();