def export_dataStage01PhysiologyRatesAverages_js(self,analysis_id_I,data_dir_I="tmp"):
"""Export data_stage01_physiology_ratesAverages to js file"""
# get the analysis information
experiment_ids,sample_name_abbreviations = [],[];
experiment_ids,sample_name_abbreviations = self.get_experimentIDAndSampleName_analysisID_dataStage01PhysiologyAnalysis(analysis_id_I);
data_O = [];
for sna_cnt,sna in enumerate(sample_name_abbreviations):
data_tmp = [];
data_tmp = self.get_rows_experimentIDAndSampleNameAbbreviation_dataStage01PhysiologyRatesAverages(experiment_ids[sna_cnt],sna);
if data_tmp: data_O.extend(data_tmp);
# visualization parameters
data1_keys = ['experiment_id','sample_name_abbreviation', 'met_id','rate_units'];
data1_nestkeys = ['met_id'];
data1_keymap = {'xdata':'met_id','ydata':'rate_average',
'serieslabel':'sample_name_abbreviation','featureslabel':'met_id',
'ydatalb':'rate_lb','ydataub':'rate_ub'};
nsvgtable = ddt_container_filterMenuAndChart2dAndTable();
nsvgtable.make_filterMenuAndChart2dAndTable(
data_filtermenu=data_O,
data_filtermenu_keys=data1_keys,
data_filtermenu_nestkeys=data1_nestkeys,
data_filtermenu_keymap=data1_keymap,
data_svg_keys=None,
data_svg_nestkeys=None,
data_svg_keymap=None,
data_table_keys=None,
data_table_nestkeys=None,
data_table_keymap=None,
data_svg=None,
data_table=None,
svgtype='verticalbarschart2d_01',
tabletype='responsivetable_01',
svgx1axislabel='met_id',
svgy1axislabel='Rate',
tablekeymap = [data1_keymap],
svgkeymap = [data1_keymap], #calculated on the fly
formtile2datamap=[0],
tabletile2datamap=[0],
svgtile2datamap=[0], #calculated on the fly
svgfilters=None,
svgtileheader='Physiological data',
tablefilters=None,
tableheaders=None,
svgparameters_I= {
"svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 },
"svgwidth":500,"svgheight":350,
'colclass':"col-sm-8"
}
);
if data_dir_I=='tmp':
filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js'
elif data_dir_I=='data_json':
data_json_O = nsvgtable.get_allObjects_js();
return data_json_O;
with open(filename_str,'w') as file:
file.write(nsvgtable.get_allObjects());
def export_dataStage01IsotopomerAveragesNormSum_js(self,experiment_id_I, time_points_I = None, sample_name_abbreviations_I = None, met_ids_I = None, scan_types_I = None,data_dir_I="tmp",
single_plot_I = True):
'''Export data_stage01_isotopomer_averagesNormSum to js file'''
mids = mass_isotopomer_distributions();
print('plotting averagesNormSum...')
data_O = [];
data_dict_O = {};
# get time points
if time_points_I:
time_points = time_points_I;
else:
time_points = [];
time_points = self.get_timePoint_experimentID_dataStage01AveragesNormSum(experiment_id_I);
for tp in time_points:
print('Plotting product and precursor for time-point ' + str(tp));
# get sample names and sample name abbreviations
sample_abbreviations = [];
sample_types = ['Unknown'];
sample_types_lst = [];
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01AveragesNormSum(experiment_id_I,st,tp);
sample_abbreviations.extend(sample_abbreviations_tmp);
sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]);
if sample_name_abbreviations_I:
sample_abbreviations = [sna for sna in sample_abbreviations if sna in sample_name_abbreviations_I];
sample_types_lst = ['Unknown' for x in sample_abbreviations];
for sna_cnt,sna in enumerate(sample_abbreviations):
print('Plotting product and precursor for sample name abbreviation ' + sna);
data_dict_O[sna] = [];
# get the scan_types
if scan_types_I:
scan_types = [];
scan_types_tmp = [];
scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
scan_types = [st for st in scan_types_tmp if st in scan_types_I];
else:
scan_types = [];
scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
for scan_type in scan_types:
print('Plotting product and precursor for scan type ' + scan_type)
# met_ids
if not met_ids_I:
met_ids = [];
met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01AveragesNormSum( \
experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type);
else:
met_ids = met_ids_I;
if not(met_ids): continue #no component information was found
for met in met_ids:
print('Plotting product and precursor for metabolite ' + met);
# fragments
fragment_formulas = [];
fragment_formulas = self.get_fragmentFormula_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanTypeAndMetID_dataStage01AveragesNormSum( \
experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met);
for frag in fragment_formulas:
print('Plotting product and precursor for fragment ' + frag);
# data
data_mat = [];
data_mat_cv = [];
data_masses = [];
data_mat,data_mat_cv,data_masses = self.get_spectrum_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanTypeAndMetIDAndFragmentFormula_dataStage01AveragesNormSum( \
experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met,frag);
for i,d in enumerate(data_mat):
stdev = 0.0;
stderr = 0.0;
lb = None;
ub = None;
if data_mat_cv[i]:
stdev = data_mat[i]*data_mat_cv[i]/100;
lb = data_mat[i]-stdev;
ub = data_mat[i]+stdev;
fragment_id = mids.make_fragmentID(met,frag,data_masses[i]);
tmp = {
'experiment_id':experiment_id_I,
'sample_name_abbreviation':sna,
'sample_type':sample_types_lst[sna_cnt],
'time_point':tp,
'met_id':met,
'fragment_formula':frag,
'fragment_mass':data_masses[i],
'fragment_id':fragment_id,
'intensity_normalized_average':d,
'intensity_normalized_cv':data_mat_cv[i],
'intensity_normalized_lb':lb,
'intensity_normalized_ub':ub,
'intensity_normalized_units':"normSum",
'scan_type':scan_type,
'used_':True,
'comment_':None}
data_O.append(tmp);
data_dict_O[sna].append(tmp);
print('tabulating averagesNormSum...')
data_table_O = [];
# get time points
if time_points_I:
time_points = time_points_I;
else:
time_points = [];
time_points = self.get_timePoint_experimentID_dataStage01AveragesNormSum(experiment_id_I);
for tp in time_points:
print('Tabulating product and precursor for time-point ' + str(tp));
# get sample names and sample name abbreviations
sample_abbreviations = [];
sample_types = ['Unknown'];
sample_types_lst = [];
for st in sample_types:
sample_abbreviations_tmp = [];
sample_abbreviations_tmp = self.get_sampleNameAbbreviations_experimentIDAndSampleTypeAndTimePoint_dataStage01AveragesNormSum(experiment_id_I,st,tp);
sample_abbreviations.extend(sample_abbreviations_tmp);
sample_types_lst.extend([st for i in range(len(sample_abbreviations_tmp))]);
if sample_name_abbreviations_I:
sample_abbreviations = [sna for sna in sample_abbreviations if sna in sample_name_abbreviations_I];
sample_types_lst = ['Unknown' for x in sample_abbreviations];
for sna_cnt,sna in enumerate(sample_abbreviations):
print('Tabulating product and precursor for sample name abbreviation ' + sna);
# get the scan_types
if scan_types_I:
scan_types = [];
scan_types_tmp = [];
scan_types_tmp = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
scan_types = [st for st in scan_types_tmp if st in scan_types_I];
else:
scan_types = [];
scan_types = self.get_scanTypes_experimentIDAndTimePointAndSampleAbbreviationsAndSampleType_dataStage01AveragesNormSum(experiment_id_I,tp,sna,sample_types_lst[sna_cnt]);
for scan_type in scan_types:
print('Tabulating product and precursor for scan type ' + scan_type)
# met_ids
if not met_ids_I:
met_ids = [];
met_ids = self.get_metIDs_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanType_dataStage01AveragesNormSum( \
experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type);
else:
met_ids = met_ids_I;
if not(met_ids): continue #no component information was found
for met in met_ids:
print('Tabulating product and precursor for metabolite ' + met);
# get rows
rows = [];
rows = self.get_rows_experimentIDAndSampleAbbreviationAndTimePointAndSampleTypeAndScanTypeAndMetID_dataStage01AveragesNormSum( \
experiment_id_I,sna,tp,sample_types_lst[sna_cnt],scan_type,met);
if not rows: continue;
for d in rows:
d['fragment_id'] = mids.make_fragmentID(d['met_id'],d['fragment_formula'],d['fragment_mass']);
data_table_O.extend(rows);
print('exporting averagesNormSum...');
# visualization parameters
data1_keys = ['sample_name_abbreviation',
'sample_type',
'met_id',
'time_point',
'fragment_formula',
'fragment_mass',
'scan_type',
'fragment_id'
];
data1_nestkeys = [
#'fragment_id',
'fragment_mass'
];
data1_keymap = {
#'xdata':'fragment_id',
'xdata':'fragment_mass',
'ydata':'intensity_normalized_average',
'tooltiplabel':'sample_name_abbreviation',
'serieslabel':'scan_type',
#'featureslabel':'fragment_id',
'featureslabel':'fragment_mass',
'ydatalb':'intensity_normalized_lb',
'ydataub':'intensity_normalized_ub'};
# initialize the ddt objects
dataobject_O = [];
parametersobject_O = [];
tile2datamap_O = {};
# make the tile parameter objects
formtileparameters_O = {'tileheader':'Filter menu','tiletype':'html','tileid':"filtermenu1",'rowid':"row1",'colid':"col1",
'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"};
formparameters_O = {'htmlid':'filtermenuform1',"htmltype":'form_01',"formsubmitbuttonidtext":{'id':'submit1','text':'submit'},"formresetbuttonidtext":{'id':'reset1','text':'reset'},"formupdatebuttonidtext":{'id':'update1','text':'update'}};
formtileparameters_O.update(formparameters_O);
dataobject_O.append({"data":data_O,"datakeys":data1_keys,"datanestkeys":data1_nestkeys});
parametersobject_O.append(formtileparameters_O);
tile2datamap_O.update({"filtermenu1":[0]});
if not single_plot_I:
rowcnt = 1;
colcnt = 1;
for cnt,sn in enumerate(sample_abbreviations):
svgtileid = "tilesvg"+str(cnt);
svgid = 'svg'+str(cnt);
iter=cnt+1; #start at 1
if (cnt % 2 == 0):
rowcnt = rowcnt+1;#even
colcnt = 1;
else:
colcnt = colcnt+1;
# make the svg object
svgparameters1_O = {"svgtype":'verticalbarschart2d_01',"svgkeymap":[data1_keymap],
#'svgid':'svg1',
'svgid':'svg'+str(cnt),
"svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 },
"svgwidth":350,"svgheight":250,"svgy1axislabel":"intensity (norm)",
"svgfilters":{'sample_name_abbreviation':[sn]}
};
svgtileparameters1_O = {'tileheader':'Isotopomer distribution','tiletype':'svg',
'tileid':svgtileid,
'rowid':"row"+str(rowcnt),
'colid':"col"+str(colcnt),
'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-6"};
svgtileparameters1_O.update(svgparameters1_O);
parametersobject_O.append(svgtileparameters1_O);
tile2datamap_O.update({svgtileid:[0]});
else:
cnt = 0;
svgtileid = "tilesvg"+str(cnt);
svgid = 'svg'+str(cnt);
rowcnt = 2;
colcnt = 1;
# make the svg object
svgparameters1_O = {"svgtype":'verticalbarschart2d_01',"svgkeymap":[data1_keymap],
'svgid':'svg'+str(cnt),
"svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 },
"svgwidth":500,"svgheight":350,"svgy1axislabel":"intensity (norm)",
};
svgtileparameters1_O = {'tileheader':'Isotopomer distribution','tiletype':'svg',
'tileid':svgtileid,
'rowid':"row"+str(rowcnt),
'colid':"col"+str(colcnt),
'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"};
svgtileparameters1_O.update(svgparameters1_O);
parametersobject_O.append(svgparameters1_O);
tile2datamap_O.update({svgtileid:[0]});
# make the table object
tableparameters1_O = {"tabletype":'responsivetable_01',
'tableid':'table1',
"tablefilters":None,
#"tableheaders":[],
"tableclass":"table table-condensed table-hover",
'tableformtileid':'tile1','tableresetbuttonid':'reset1','tablesubmitbuttonid':'submit1'};
tabletileparameters1_O = {'tileheader':'Isotopomer distribution','tiletype':'table','tileid':"tabletile1",
'rowid':"row100",
'colid':"col1",
'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"};
tabletileparameters1_O.update(tableparameters1_O);
dataobject_O.append({"data":data_table_O,"datakeys":data1_keys,"datanestkeys":data1_nestkeys});
parametersobject_O.append(tabletileparameters1_O);
tile2datamap_O.update({"tabletile1":[1]})
nsvgtable = ddt_container_filterMenuAndChart2dAndTable();
nsvgtable.make_filterMenuAndChart2dAndTable(
data_filtermenu=data_O,
data_filtermenu_keys=data1_keys,
data_filtermenu_nestkeys=data1_nestkeys,
data_filtermenu_keymap=data1_keymap,
data_svg_keys=data1_keys,
data_svg_nestkeys=data1_nestkeys,
data_svg_keymap=data1_keymap,
data_table_keys=None,
data_table_nestkeys=None,
data_table_keymap=None,
data_svg=data_dict_O,
data_table=None,
svgtype='verticalbarschart2d_01',
tabletype='responsivetable_01',
svgx1axislabel='',
svgy1axislabel='',
tablekeymap = [data1_keymap],
svgkeymap = [], #calculated on the fly
formtile2datamap=[0],
tabletile2datamap=[0],
svgtile2datamap=[], #calculated on the fly
svgfilters=None,
svgtileheader='Isotopomer distribution',
tablefilters=None,
tableheaders=None
);
if data_dir_I=='tmp':
filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js'
elif data_dir_I=='data_json':
data_json_O = nsvgtable.get_allObjects_js();
return data_json_O;
with open(filename_str,'w') as file:
file.write(nsvgtable.get_allObjects());
def export_dataStage01MQResultsTable_js(
self,analysis_id_I,
features_I=[
'calculated_concentration',
'height_ratio',
'retention_time'],
include_IS_I=False,
data_dir_I="tmp"):
''' export a metric plot
INPUT:
experiment_id_I = experiment_id
sample_names_I = sample_names
component_names_I = component names
features_I = features to plot,
supports calculated_concentration, height_ratio,
area_ratio, height, area, retention_time
'''
#quantification_analysis_query = stage01_quantification_analysis_query(self.session,self.engine,self.settings)
# get the data:
data_O = [];
data_dict_O = {};
data_tmp = [];
data_tmp = self.getRowsJoin_analysisID_dataStage01QuantificationMQResultsTable(analysis_id_I);
# make a unique index for each component
component_names = list(set([d['component_name'] for d in data_tmp]));
component_names_dict = {d:0 for d in component_names};
for d in data_tmp:
tmp = {};
tmp['acquisition_date_and_time'] = self.convert_datetime2string(d['acquisition_date_and_time'])
tmp['sample_name'] = d['sample_name'];
tmp['sample_type'] = d['sample_type'];
#tmp['experiment_id'] = d['experiment_id'];
#tmp['analysis_id'] = d['analysis_id'];
tmp['analysis_id']=analysis_id_I;
tmp['component_name'] = d['component_name'];
tmp['component_group_name'] = d['component_group_name'];
tmp['component_index'] = component_names_dict[d['component_name']];
for m in features_I:
if not m in data_dict_O: data_dict_O[m]=[];
if m in d.keys() and not d[m] is None:
tmp1 = copy.copy(tmp);
tmp1['feature_name'] = m;
tmp1['feature_value'] = d[m];
data_dict_O[m].append(tmp1);
data_O.append(tmp1);
component_names_dict[d['component_name']]+=1;
# dump chart parameters to a js files
data1_keys = [
'analysis_id',
'sample_name',
'component_name',
'feature_name',
'acquisition_date_and_time',
'sample_type',
];
data1_nestkeys = ['component_name'];
data1_keymap = {'xdata':'component_index',
'ydata':'feature_value',
'serieslabel':'component_name',
'featureslabel':'sample_name'};
svgkeymap1 = [[data1_keymap,data1_keymap] for k in data_dict_O.keys()]
parameters = {"chart1margin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 },
"chart1width":500,"chart1height":350,
"chart1title":"Metric Plot", "chart1x1axislabel":"sample_name","chart1y1axislabel":"measurement"}
## make the data object
#dataobject_O = [{"data":data_O,"datakeys":data1_keys,"datanestkeys":data1_nestkeys},{"data":data_O,"datakeys":data1_keys,"datanestkeys":data1_nestkeys}];
## make the tile parameter objects
#formtileparameters_O = {'tileheader':'Filter menu','tiletype':'html','tileid':"filtermenu1",'rowid':"row1",'colid':"col1",
# 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-4"};
#formparameters_O = {'htmlid':'filtermenuform1','htmltype':'form_01',"formsubmitbuttonidtext":{'id':'submit1','text':'submit'},"formresetbuttonidtext":{'id':'reset1','text':'reset'},"formupdatebuttonidtext":{'id':'update1','text':'update'}};
#formtileparameters_O.update(formparameters_O);
#svgparameters_O = {"svgtype":'scatterlineplot2d_01',"svgkeymap":[data1_keymap,data1_keymap],
# 'svgid':'svg1',
# "svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 },
# "svgwidth":500,"svgheight":350,
# "svgx1axislabel":"sample_name","svgy1axislabel":"measurement_value",
# 'svgformtileid':'filtermenu1','svgresetbuttonid':'reset1','svgsubmitbuttonid':'submit1'};
#svgtileparameters_O = {'tileheader':'Metric Plot','tiletype':'svg','tileid':"tile2",'rowid':"row1",'colid':"col2",
# 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-8"};
#svgtileparameters_O.update(svgparameters_O);
#tableparameters_O = {"tabletype":'responsivetable_01',
# 'tableid':'table1',
# "tablefilters":None,
# "tableclass":"table table-condensed table-hover",
# 'tableformtileid':'filtermenu1','tableresetbuttonid':'reset1','tablesubmitbuttonid':'submit1'};
#tabletileparameters_O = {'tileheader':'Metric plot','tiletype':'table','tileid':"tile3",'rowid':"row2",'colid':"col1",
# 'tileclass':"panel panel-default",'rowclass':"row",'colclass':"col-sm-12"};
#tabletileparameters_O.update(tableparameters_O);
#parametersobject_O = [formtileparameters_O,svgtileparameters_O,tabletileparameters_O];
#tile2datamap_O = {"filtermenu1":[0],"tile2":[0,1],"tile3":[0]};
## dump the data to a json file
#ddtutilities = ddt_container(parameters_I = parametersobject_O,data_I = dataobject_O,tile2datamap_I = tile2datamap_O,filtermenu_I = None);
#if data_dir_I=='tmp':
# filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js'
#elif data_dir_I=='data_json':
# data_json_O = ddtutilities.get_allObjects_js();
# return data_json_O;
#with open(filename_str,'w') as file:
# file.write(ddtutilities.get_allObjects());
nsvgtable = ddt_container_filterMenuAndChart2dAndTable();
nsvgtable.make_filterMenuAndChart2dAndTable(
data_filtermenu=data_O,
data_filtermenu_keys=data1_keys,
data_filtermenu_nestkeys=data1_nestkeys,
data_filtermenu_keymap=data1_keymap,
data_svg_keys=None,
data_svg_nestkeys=None,
data_svg_keymap=None,
data_table_keys=None,
data_table_nestkeys=None,
data_table_keymap=None,
data_svg=data_dict_O,
data_table=None,
#svgtype='scatterlineplot2d_01',
svgtype='scatterplot2d_01',
tabletype='responsivetable_01',
svgx1axislabel='',
svgy1axislabel='',
tablekeymap = [data1_keymap],
svgkeymap = [], #calculated on the fly
formtile2datamap=[0],
tabletile2datamap=[0],
svgtile2datamap=[], #calculated on the fly
svgfilters=None,
svgtileheader='Metric Plot',
tablefilters=None,
tableheaders=None,
svgparameters_I = parameters
);
if data_dir_I=='tmp':
filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js'
elif data_dir_I=='data_json':
data_json_O = nsvgtable.get_allObjects_js();
return data_json_O;
with open(filename_str,'w') as file:
file.write(nsvgtable.get_allObjects());
def export_dataStage01PhysiologyData_js(self,
analysis_id_I,
data_units_I=['mM','OD600'],
data_dir_I="tmp"
):
"""
Export data_stage01_physiology_data to js file
"""
physiology_analysis_query = stage01_physiology_analysis_query(self.session,self.engine,self.settings);
# get the analysis information
sample_name_shorts = [];
sample_name_shorts = physiology_analysis_query.get_rows_analysisID_dataStage01PhysiologyAnalysis(analysis_id_I);
data_O = [];
#TODO optimize to a single query
for sns_cnt,sns in enumerate(sample_name_shorts):
data_tmp = [];
data_tmp = self.get_sampleDateAndDataCorrected_experimentIDAndSampleNameShort(sns['experiment_id'],sns['sample_name_short'],data_units_I=data_units_I);
for d in data_tmp:
d['sample_date'] = d['sample_date'].year*8765.81277 + \
d['sample_date'].month*730.484 + d['sample_date'].day*365.242 + \
d['sample_date'].hour + d['sample_date'].minute / 60. + \
d['sample_date'].second / 3600.; #convert using datetime object
data_O.append(d);
# visualization parameters
data1_keys = ['experiment_id',
'sample_id',
'sample_name_short',
'sample_name_abbreviation',
'sample_date',
'met_id',
'data_units'
];
#TODO xaxis = time
#TODO seperate plot for each metabolite
data1_nestkeys = ['met_id'];
data1_keymap = {
'xdata':'sample_date',
'ydata':'data_corrected',
'serieslabel':'sample_name_short',
'featureslabel':'sample_id',
};
nsvgtable = ddt_container_filterMenuAndChart2dAndTable();
nsvgtable.make_filterMenuAndChart2dAndTable(
data_filtermenu=data_O,
data_filtermenu_keys=data1_keys,
data_filtermenu_nestkeys=data1_nestkeys,
data_filtermenu_keymap=data1_keymap,
data_svg_keys=None,
data_svg_nestkeys=None,
data_svg_keymap=None,
data_table_keys=None,
data_table_nestkeys=None,
data_table_keymap=None,
data_svg=None,
data_table=None,
svgtype='scatterplot2d_01',
tabletype='responsivetable_01',
svgx1axislabel='time (hrs)',
svgy1axislabel='data_corrected',
tablekeymap = [data1_keymap],
svgkeymap = [data1_keymap], #calculated on the fly
formtile2datamap=[0],
tabletile2datamap=[0],
svgtile2datamap=[0], #calculated on the fly
svgfilters=None,
svgtileheader='Physiological data',
tablefilters=None,
tableheaders=None,
svgparameters_I= {
"svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 },
"svgwidth":500,"svgheight":350,
'colclass':"col-sm-8"
}
);
if data_dir_I=='tmp':
filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js'
elif data_dir_I=='data_json':
data_json_O = nsvgtable.get_allObjects_js();
return data_json_O;
with open(filename_str,'w') as file:
file.write(nsvgtable.get_allObjects());
def export_dataStage01RNASequencingGenesFpkmTracking_pairWisePlot_js(self,analysis_id_I,log2normalization_I=True,data_dir_I='tmp'):
'''Export data for a pairwise scatter plot
INPUT:
analysis_id = String, analysis_id
log2normalization_I = Boolean, apply a log2 normalization the FPKM values (default: True)
data_dir_I = string, data directory
OUTPUT:
'''
# get the analysis information
experiment_ids,sample_names = [],[];
experiment_ids,sample_names = self.get_experimentIDAndSampleName_analysisID_dataStage01RNASequencingAnalysis(analysis_id_I);
data_O = [];
for sample_name_cnt,sample_name in enumerate(sample_names):
# query fpkm data:
fpkms = [];
fpkms = self.get_rows_experimentIDAndSampleName_dataStage01RNASequencingGenesFpkmTracking(experiment_ids[sample_name_cnt],sample_name);
if log2normalization_I:
for f in fpkms:
if f['FPKM'] == 0.0: f['FPKM'] = 0.0;
else: f['FPKM'] = log2(f['FPKM']);
data_O.extend(fpkms);
# reorganize the data
listdict = listDict(data_O);
data_O,columnValueHeader_O = listdict.convert_listDict2ColumnGroupListDict(
#value_labels_I = ['FPKM','FPKM_conf_lo','FPKM_conf_hi'],
value_labels_I = ['FPKM',],
column_labels_I = ['experiment_id','sample_name'],
feature_labels_I = ['gene_id','gene_short_name'],
na_str_I=0.0,
columnValueConnector_str_I='_',
);
# make the tile object
#data1 = filtermenu/table
data1_keymap_table = {
'xdata':'svd_method',
'ydata':'singular_value_index',
'zdata':'d_vector',
'rowslabel':'svd_method',
'columnslabel':'singular_value_index',
};
#data2 = svg
#if single plot, data2 = filter menu, data2, and table
data1_keys = ['gene_id','gene_short_name'
];
data1_nestkeys = ['gene_short_name'];
data1_keymap_svg = [];
svgtype = [];
svgtile2datamap = [];
data_svg_keymap = [];
for cnt1,column1 in enumerate(columnValueHeader_O):
for cnt2,column2 in enumerate(columnValueHeader_O[cnt1+1:]):
keymap = {
'xdata':column1,
'ydata':column2,
'serieslabel':'',
'featureslabel':'gene_short_name',
'tooltipdata':'gene_short_name',
};
data1_keymap_svg.append([keymap]);
data_svg_keymap.append(keymap);
svgtype.append('pcaplot2d_scores_01');
svgtile2datamap.append([0]);
nsvgtable = ddt_container_filterMenuAndChart2dAndTable();
nsvgtable.make_filterMenuAndChart2dAndTable(
data_filtermenu=data_O,
data_filtermenu_keys=data1_keys,
data_filtermenu_nestkeys=data1_nestkeys,
data_filtermenu_keymap=data1_keymap_table,
data_svg_keys=data1_keys,
data_svg_nestkeys=data1_nestkeys,
data_svg_keymap=data_svg_keymap,
data_table_keys=data1_keys,
data_table_nestkeys=data1_nestkeys,
data_table_keymap=data1_keymap_table,
data_svg=None,
data_table=None,
svgtype=svgtype,
tabletype='responsivetable_01',
svgx1axislabel='',
svgy1axislabel='',
tablekeymap = [data1_keymap_table],
svgkeymap = data1_keymap_svg,
formtile2datamap=[0],
tabletile2datamap=[0],
svgtile2datamap=svgtile2datamap,
svgfilters=None,
svgtileheader='Pair-wise scatter plot',
tablefilters=None,
tableheaders=None
);
if data_dir_I=='tmp':
filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js'
elif data_dir_I=='data_json':
data_json_O = nsvgtable.get_allObjects_js();
return data_json_O;
with open(filename_str,'w') as file:
file.write(nsvgtable.get_allObjects());
def export_dataStage02PhysiologySampledPoints_js(self,
analysis_id_I,
simulation_ids_I = [],
rxn_ids_I = [],
query_I={},
data_dir_I='tmp'
):
'''Visualize the sampling distribution
DESCRIPTION:
tile1=filtermenu
tile2=sampling distribution
tile3=table
'''
calculatehistogram = calculate_histogram();
physiology_analysis_query = stage02_physiology_analysis_query(self.session,self.engine,self.settings);
data_sampledPoints_O = [];
data_O = [];
#get the analysis info
simulation_ids = physiology_analysis_query.get_simulationID_analysisID_dataStage02PhysiologyAnalysis(analysis_id_I);
if simulation_ids_I:
simulation_ids = [s for s in simulation_ids if s in simulation_ids_I];
for simulation_id in simulation_ids:
#get the data_dirs for the simulations and read in the points
sampledPoints = self.get_rows_simulationID_dataStage02PhysiologySampledPoints(simulation_id);
sampledPoints = sampledPoints[0];
data_sampledPoints_O.append(sampledPoints);
# get simulation information
simulation_info_all = [];
simulation_info_all = self.get_rows_simulationIDAndSimulationType_dataStage02PhysiologySimulation(simulation_id,'sampling');
if not simulation_info_all:
print('simulation not found!')
return;
simulation_info = simulation_info_all[0]; # unique constraint guarantees only 1 row will be returned
# get simulation parameters
simulation_parameters_all = [];
simulation_parameters_all = self.get_rows_simulationID_dataStage02PhysiologySamplingParameters(simulation_id);
if not simulation_parameters_all:
print('simulation not found!')
return;
simulation_parameters = simulation_parameters_all[0]; # unique constraint guarantees only 1 row will be returned
#fill the sampledData with the actual points
sampling = cobra_sampling(model_I=None,data_dir_I = sampledPoints['data_dir'],loops_I=sampledPoints['infeasible_loops']);
if simulation_parameters['sampler_id']=='gpSampler':
# load the results of sampling
sampling.get_points_matlab(matlab_data=None,sampler_model_name='sampler_out');
sampling.remove_loopsFromPoints();
elif simulation_parameters['sampler_id']=='optGpSampler':
sampling.get_points_json();
sampling.remove_loopsFromPoints();
else:
print('sampler_id not recognized');
#store the sampledPoints
for k,v in sampling.points.items():
if rxn_ids_I:
if not k in rxn_ids_I:
continue;
n_bins = 100;
calc_bins_I = False;
x_O,dx_O,y_O = calculatehistogram.histogram(data_I=v,n_bins_I=n_bins,calc_bins_I=calc_bins_I);
for i,b in enumerate(x_O):
tmp = {
'simulation_id':simulation_id,
'rxn_id':k,
#'feature_units':feature_units,
'bin':b,
'bin_width':dx_O[i],
'frequency':int(y_O[i]),
'used_':True,
'comment_':None};
data_O.append(tmp);
#data_listDict = listDict();
#data_listDict.set_dictList(sampling.points);
#data_listDict.convert_dictList2DataFrame();
#points, rxn_ids = data_listDict.get_flattenedDataAndColumnLabels();
#data_listDict.clear_allData();
#data_listDict.add_column2DataFrame('rxn_id',rxn_ids);
#data_listDict.add_column2DataFrame('points',points);
#data_listDict.add_column2DataFrame('simulation_id',simulation_id);
#data_listDict.convert_dataFrame2ListDict();
#data_O.extend(data_listDict.get_listDict());
#make the DDT histograms
# visualization parameters
data1_keys = ['simulation_id',
'rxn_id',
'bin',
];
data1_nestkeys = [
'bin'
];
data1_keymap = {
'xdata':'bin',
'ydata':'frequency',
'serieslabel':'simulation_id',
'featureslabel':'bin',
'tooltiplabel':'rxn_id',
'ydatalb':None,
'ydataub':None};
nsvgtable = ddt_container_filterMenuAndChart2dAndTable();
nsvgtable.make_filterMenuAndChart2dAndTable(
data_filtermenu=data_O,
data_filtermenu_keys=data1_keys,
data_filtermenu_nestkeys=data1_nestkeys,
data_filtermenu_keymap=data1_keymap,
data_svg_keys=None,
data_svg_nestkeys=None,
data_svg_keymap=None,
data_table_keys=None,
data_table_nestkeys=None,
data_table_keymap=None,
data_svg=None,
data_table=None,
svgtype='verticalbarschart2d_01',
tabletype='responsivetable_01',
svgx1axislabel='',
svgy1axislabel='',
tablekeymap = [data1_keymap],
svgkeymap = [data1_keymap],
formtile2datamap=[0],
tabletile2datamap=[0],
svgtile2datamap=[0], #calculated on the fly
svgfilters=None,
svgtileheader='Sampled points',
tablefilters=None,
tableheaders=None
);
if data_dir_I=='tmp':
filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js'
elif data_dir_I=='data_json':
data_json_O = nsvgtable.get_allObjects_js();
return data_json_O;
with open(filename_str,'w') as file:
file.write(nsvgtable.get_allObjects());
def export_dataStage02PhysiologyGraphDataShortestPaths_js(self,analysis_id_I,data_dir_I='tmp'):
'''export graph of shortest paths'''
data_O = [];
data_reactions_O = [];
data_tmp = self.get_rows_analysisID_dataStage02PhysiologyGraphDataShortestPaths(analysis_id_I);
for d in data_tmp:
d['path_id'] = d['path_start'] + '-->' + d['path_stop']
data_O.append(d);
for i,p in enumerate(d['paths']):
if i==0: continue;
tmp = {};
tmp['left']=d['paths'][i-1];
tmp['right']=p;
link_id = p;
node_id = 'rxn';
if i%2==0:
link_id=d['paths'][i-1];
node_id = 'met';
tmp['link_id']=link_id;
tmp['weight']=1;
tmp['node_id']=node_id;
tmp['path_id'] = d['path_id'];
tmp['path_start'] = d['path_start']
tmp['path_stop'] = d['path_stop']
tmp['algorithm'] = d['algorithm']
tmp['analysis_id'] = d['analysis_id']
tmp['simulation_id'] = d['simulation_id']
data_reactions_O.append(tmp);
# make the data parameters
data1_keys = [
'simulation_id',
'path_start',
'path_stop',
'algorithm',
];
data1_nestkeys = [
'left',
'right',
];
data1_keymap = {'xdata':'node_id',
'ydata':'weight',
'serieslabel':'algorithm',
'featureslabel':''};
data2_keymap = {
'xdata':'node_id',
'ydata':'weight',
'xdatalabel':'',
'ydatalabel':'',
'serieslabel':'algorithm',
'featureslabel':'',
#'tooltiplabel':'component_name',
};
svgparameters = {
"svgmargin":{ 'top': 100, 'right': 100, 'bottom': 100, 'left': 100 },
"svgwidth":500,
"svgheight":500,
"svgduration":750,
'colclass':"col-sm-12",
'svgcolorcategory':'blue2red64',
'svgcolordomain':'min,max',
'svgcolordatalabel':'parent_classes',
'svgcolorscale':'quantile',
};
nsvgtable = ddt_container_filterMenuAndChart2dAndTable();
nsvgtable.make_filterMenuAndChart2dAndTable(
data_filtermenu=data_reactions_O,
data_filtermenu_keys=data1_keys,
data_filtermenu_nestkeys=data1_nestkeys,
data_filtermenu_keymap=data1_keymap,
data_svg_keys=None,
data_svg_nestkeys=None,
data_svg_keymap=data2_keymap,
data_table_keys=None,
data_table_nestkeys=None,
data_table_keymap=None,
data_svg=None,
data_table=data_O,
#svgtype='sankeydiagram2d_01',
svgtype='forcedirectedgraph2d_01',
tabletype='responsivetable_01',
svgx1axislabel='',
svgy1axislabel='',
tablekeymap = [data1_keymap],
svgkeymap = [data2_keymap],
formtile2datamap=[0],
tabletile2datamap=[1],
svgtile2datamap=[0],
svgfilters=None,
svgtileheader='Shortest Path',
tablefilters=None,
tableheaders=None,
svgparameters_I= svgparameters,
);
if data_dir_I=='tmp':
filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js'
elif data_dir_I=='data_json':
data_json_O = nsvgtable.get_allObjects_js();
return data_json_O;
with open(filename_str,'w') as file:
file.write(nsvgtable.get_allObjects());
def export_dataStage01ResequencingEndpointLineages_js(self,analysis_id_I,
query_I={},
data_dir_I='tmp'):
'''
Pie chart or bar chart of mutation counts
INPUT:
analysis_id_I = string,
query_I = additional query deliminators
'''
data_O = self.getGroupAndCount_analysisIDAndMutationTypeAndMutationPositionAndMutationGenes_analysisID_dataStage01ResequencingEndpointLineages(
analysis_id_I,
column_name_I = 'analysis_id',
aggregate_function_I='count',
aggregate_label_I='count_1',
query_I=query_I,
output_O='listDict',
dictColumn_I=None);
#add in the mutationID
genomediff = genome_diff();
for d in data_O:
d['mutation_id'] = genomediff._make_mutationID(
d['mutation_genes'],d['mutation_type'],d['mutation_position']
);
if d['mutation_genes']:
d['mutation_genes']=";".join([x for x in d['mutation_genes'] if x is not None]);
else: d['mutation_genes']=d['mutation_genes'];
# make the tile objects
#data1 = filter menu and table
data1_keys = ['analysis_id',
'mutation_type',
'mutation_position',
'mutation_genes',
'count_1',
'mutation_id'
];
data1_nestkeys = ['mutation_id'];
data1_keymap = {
'xdata':'mutation_id',
'ydata':'count_1',
'serieslabel':'analysis_id',
'featureslabel':'mutation_id',
'tooltiplabel':'mutation_id',
};
nsvgtable = ddt_container_filterMenuAndChart2dAndTable();
nsvgtable.make_filterMenuAndChart2dAndTable(
data_filtermenu=data_O,
data_filtermenu_keys=data1_keys,
data_filtermenu_nestkeys=data1_nestkeys,
data_filtermenu_keymap=data1_keymap,
data_svg_keys=None,
data_svg_nestkeys=None,
data_svg_keymap=None,
data_table_keys=None,
data_table_nestkeys=None,
data_table_keymap=None,
data_svg=None,
data_table=None,
svgtype='verticalbarschart2d_01',
tabletype='responsivetable_01',
svgx1axislabel='',
svgy1axislabel='',
tablekeymap = [data1_keymap],
svgkeymap = [data1_keymap],
formtile2datamap=[0],
tabletile2datamap=[0],
svgtile2datamap=[0], #calculated on the fly
svgfilters=None,
svgtileheader='Mutation counts',
tablefilters=None,
tableheaders=None
);
if data_dir_I=='tmp':
filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js'
elif data_dir_I=='data_json':
data_json_O = nsvgtable.get_allObjects_js();
return data_json_O;
with open(filename_str,'w') as file:
file.write(nsvgtable.get_allObjects());
def export_dataStage01RNASequencingGeneExpDiffFpkmTracking_js(
self,analysis_ids_I,gene_short_names_I=[],query_I={},data_dir_I='tmp'):
'''Export data as a bullet chart
INPUT:
OUTPUT:
'''
data_O = [];
# get fpkm tracking data
data_O = self.get_rows_analysisIDAndGeneShortNames_dataStage01RNASequencingGeneExpDiffFpkmTracking(
analysis_ids_I,
gene_short_names_I,
query_I=query_I,
output_O='listDict',
dictColumn_I=None);
# make the data parameters
data1_keys = [
'analysis_id',
'experiment_id',
'sample_name_abbreviation',
#'gene_id',
'gene_short_name',
];
data1_nestkeys = ['gene_short_name'];
data1_keymap = {
'ydata':'FPKM',
'ydatamean':'FPKM',
'ydatalb':'FPKM_conf_lo',
'ydataub':'FPKM_conf_hi',
'xdata':'gene_short_name',
'serieslabel':'sample_name_abbreviation',
'featureslabel':'sample_name_abbreviation',
'tooltiplabel':'gene_short_name'};
nsvgtable = ddt_container_filterMenuAndChart2dAndTable();
nsvgtable.make_filterMenuAndChart2dAndTable(
data_filtermenu=data_O,
data_filtermenu_keys=data1_keys,
data_filtermenu_nestkeys=data1_nestkeys,
data_filtermenu_keymap=data1_keymap,
data_svg_keys=None,
data_svg_nestkeys=None,
data_svg_keymap=None,
data_table_keys=None,
data_table_nestkeys=None,
data_table_keymap=None,
data_svg=None,
data_table=None,
svgtype='boxandwhiskersplot2d_01',
tabletype='responsivetable_01',
svgx1axislabel='',
svgy1axislabel='FPKM',
tablekeymap = [data1_keymap],
svgkeymap = [data1_keymap],
formtile2datamap=[0],
tabletile2datamap=[0],
svgtile2datamap=[0],
svgfilters=None,
svgtileheader='Cuffdiff FPKM',
tablefilters=None,
tableheaders=None,
svgparameters_I={"svgmargin":{ 'top': 50, 'right': 150, 'bottom': 50, 'left': 50 },
"svgwidth":500,"svgheight":350,}
);
if data_dir_I=='tmp':
filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js'
elif data_dir_I=='data_json':
data_json_O = nsvgtable.get_allObjects_js();
return data_json_O;
with open(filename_str,'w') as file:
file.write(nsvgtable.get_allObjects());
def export_dataStage01RNASequencingGeneExpDiff_count_js(self,
query_I={'where':[
{"table_name":'data_stage01_rnasequencing_geneExpDiff',
'column_name':'experiment_id_1',
'value':'ALEsKOs01',
'operator':'LIKE',
'connector':'AND'
},
],
},
data_dir_I='tmp'):
'''Export data for a count of differentially expressed genes
'''
# get the analysis information
query = {}
tables = ['data_stage01_rnasequencing_geneExpDiff'];
query['select'] = [
{"table_name":tables[0],
"column_name":'experiment_id_1',
},{"table_name":tables[0],
"column_name":'experiment_id_2',
},{"table_name":tables[0],
"column_name":'sample_name_abbreviation_1',
},{"table_name":tables[0],
"column_name":'sample_name_abbreviation_2',
},
];
query['group_by'] = [
{"table_name":tables[0],
"column_name":'experiment_id_1',
},{"table_name":tables[0],
"column_name":'experiment_id_2',
},{"table_name":tables[0],
"column_name":'sample_name_abbreviation_1',
},{"table_name":tables[0],
"column_name":'sample_name_abbreviation_2',
},
];
query['order_by'] = [
{"table_name":tables[0],
"column_name":'experiment_id_1',
"order":'ASC',
},{"table_name":tables[0],
"column_name":'experiment_id_2',
"order":'ASC',
},{"table_name":tables[0],
"column_name":'sample_name_abbreviation_1',
"order":'ASC',
},{"table_name":tables[0],
"column_name":'sample_name_abbreviation_2',
"order":'ASC',
},
];
query['where'] = [
{"table_name":tables[0],
'column_name':'significant',
'value':'yes',
'operator':'LIKE',
'connector':'AND'
},
];
#additional query blocks
for k,v in query_I.items():
if not k in query.keys():
query[k] = [];
for r in v:
query[k].append(r);
data_O = [];
data_O = self.getAggregateFunction_rows_dataStage01RNASequencingGeneExpDiff(
column_name_I = 'gene',
aggregate_function_I='count',
aggregate_label_I='count_1',
query_I=query,
output_O='listDict',
dictColumn_I=None);
# add in new column
for d in data_O:
d['condition'] = ('%s_%s'%(d['sample_name_abbreviation_1'],d['sample_name_abbreviation_2']))
# make the data parameters
data1_keys = list(data_O[0].keys());
data1_nestkeys = ['sample_name_abbreviation_1'];
data1_keymap = {'ydata':'count_1',
'xdata':'condition',
'serieslabel':'sample_name_abbreviation_2',
'featureslabel':'condition',
'tooltiplabel':'condition'};
# make the data object
nsvgtable = ddt_container_filterMenuAndChart2dAndTable();
nsvgtable.make_filterMenuAndChart2dAndTable(
data_filtermenu=data_O,
data_filtermenu_keys=data1_keys,
data_filtermenu_nestkeys=data1_nestkeys,
data_filtermenu_keymap=data1_keymap,
data_svg_keys=None,
data_svg_nestkeys=None,
data_svg_keymap=None,
data_table_keys=None,
data_table_nestkeys=None,
data_table_keymap=None,
data_svg=None,
data_table=None,
svgtype='verticalbarschart2d_01',
tabletype='responsivetable_01',
svgx1axislabel='Condition',
svgy1axislabel='Count',
tablekeymap = [data1_keymap],
svgkeymap = [data1_keymap],
formtile2datamap=[0],
tabletile2datamap=[0],
svgtile2datamap=[0],
svgfilters=None,
svgtileheader='Significant DE count',
tablefilters=None,
tableheaders=None
);
if data_dir_I=='tmp':
filename_str = self.settings['visualization_data'] + '/tmp/ddt_data.js'
elif data_dir_I=='data_json':
data_json_O = nsvgtable.get_allObjects_js();
return data_json_O;
with open(filename_str,'w') as file:
file.write(nsvgtable.get_allObjects());
