def __init__(self, run_id, user, password, run_information=None):
#Create the gasifier data frame, and load the data from the SQL database (this will be automated through the interface later)
self.interface_raw = db.db_interface(host="192.168.13.51",
user=user,
passwd=password)
self.interface_raw.connect()
q = db.use_Query("lab_run_db")
self.interface_raw.query(q)
self.interface_proc = db.db_interface(host="192.168.13.51",
user=user,
passwd=password)
self.interface_proc.connect()
q = db.use_Query("lab_proc_db")
self.interface_proc.query(q)
self.run_id = run_id
self.run_info = RunInformation()
self._load_run_info()
self._load_ts_timeseries_data()
print self.ts['ts']
plot = TimeSeriesPlot(
data=self.ts,
Y_cols=[['mass_flow_brush_feeder', 'CO_MS', 'H2_MS']],
markers='o')
plot.plot()
plot.show()
plot.close()
def __init__(self,
run_id,
user,
password,
server="localhost",
run_information=None):
#Create the pilot data frame, and load the data from the SQL database.
self.interface_raw = db.db_interface(host=server,
user=user,
passwd=password)
self.interface_raw.connect()
q = db.use_Query("pilot_run_db")
self.interface_raw.query(q)
self.interface_proc = db.db_interface(host=server,
user=user,
passwd=password)
self.interface_proc.connect()
q = db.use_Query("pilot_proc_db")
self.interface_proc.query(q)
self.run_id = run_id
self.run_info = RunInformation()
#self.run_info.info = run_information
self._load_run_info()
self._load_timeseries_data()
self._setup_standard_streams()
def SQL_connect(self, host, user, password, database):
"""Creates database interface for connecting to SQL database"""
self.db_interface = db.db_interface(host = host, user = user, passwd = password)
self.db_interface.connect()
q = db.use_Query(database)
self.db_interface.query(q)
def testUpdateLoad(self):
"""The dataframe should successfully update existing records in an existing table"""
# set up the database -- need to make a "safe" user for this, talk to Adrian about this
os.system("mysql -h 192.168.10.20 -u chris -p < dataframe_pd_setup.sql")
# go get the data from the database
data = df.Dataframe()
interface = SQL.db_interface(host = "192.168.10.20", user = "******", passwd = "cmp87ud01")
interface.connect()
q = SQL.use_Query("gas_unit_test")
interface.query(q)
data.SQL_load_data(interface, table = "dataframe_pd_test_table")
#create a new dataframe with one line
data2 = df.Dataframe({'A':[1.2],'B':[1.0],'C':[2.0],'cheetah':[3.0]})
data2.SQL_upload_data(interface, table = "dataframe_upload_test_table", index_col = 'A')
#pull the data back into data
data.SQL_load_data(interface, table = "dataframe_upload_test_table")
#make sure the first row is equal
self.assertEqual(1.2, data.iloc[0][0])
self.assertEqual(1.0, data.iloc[0][1])
self.assertEqual(2.0, data.iloc[0][2])
self.assertEqual(3.0, data.iloc[0][3])
def testInsertLoadNan(self):
"""The dataframe should successfully add new records to an existing table when NaN values are in the table"""
# set up the database -- need to make a "safe" user for this, talk to Adrian about this
os.system("mysql -h 192.168.10.20 -u chris -p < dataframe_pd_setup.sql")
# go get the data from the database
data = df.Dataframe()
interface = SQL.db_interface(host = "192.168.10.20", user = "******", passwd = "cmp87ud01")
interface.connect()
q = SQL.use_Query("gas_unit_test")
interface.query(q)
data.SQL_load_data(interface, table = "dataframe_pd_test_table")
#replace a couple values with nan
data.iloc[0][2] = np.nan
data.iloc[1][1] = np.nan
#upload the data to the other table
data.SQL_upload_data(interface, table = "dataframe_upload_test_table")
#download the data into a new dataframe
data2 = df.Dataframe()
data2.SQL_load_data(interface, table = "dataframe_upload_test_table")
self.assertTrue(np.isnan(data2.iloc[0][2]))
self.assertTrue(np.isnan(data2.iloc[1][1]))
def testCorrectSQLLoad(self):
"""The dataframe should correctly be loaded from an SQL table"""
data = df.Dataframe()
interface = SQL.db_interface(host = "192.168.10.20", user = "******", passwd = "cmp87ud01")
interface.connect()
q = SQL.use_Query("gas_unit_test")
interface.query(q)
data.SQL_load_data(interface, table = "dataframe_pd_test_table")
self.assertTrue(np.all((data==Data.df1).values))
def __init__(self, run_id, user, password, run_information = None):
#Create the gasifier data frame, and load the data from the SQL database (this will be automated through the interface later)
self.interface_raw = db.db_interface(host = "192.168.13.51", user = user, passwd = password)
self.interface_raw.connect()
q = db.use_Query("lab_run_db")
self.interface_raw.query(q)
self.interface_proc = db.db_interface(host = "192.168.13.51", user = user, passwd = password)
self.interface_proc.connect()
q = db.use_Query("lab_proc_db")
self.interface_proc.query(q)
self.run_id = run_id
self.run_info = RunInformation()
#self.run_info.info = run_information
self._load_run_info()
self._load_timeseries_data()
self._setup_standard_streams()
def __init__(self, user, password, host='localhost'):
#set up the database connection
self.interface = db.db_interface(host, user=user, passwd=password)
self.interface.connect()
q = db.use_Query("pilot_proc_db")
self.interface.query(q)
#create any appropriate member objects
pass
def __init__(self, run_id, user, password, run_information = None):
#Create the gasifier data frame, and load the data from the SQL database
# Create connection for obtaining raw time series data from lab_run_db
self.interface_raw = db.db_interface(host = "192.168.13.51", user = user, passwd = password)
self.interface_raw.connect()
q = db.use_Query("lab_run_db")
self.interface_raw.query(q)
# Create connection for obtaining run information and uploading results to lab_proc_db
self.interface_proc = db.db_interface(host = "192.168.13.51", user = user, passwd = password)
self.interface_proc.connect()
q = db.use_Query("lab_proc_db")
self.interface_proc.query(q)
self.run_id = run_id
#Create run information container object to house experiment metadata such as start and start times, carbon in biomass, moisture in biomass, etc...
self.run_info = RunInformation()
#self.run_info.info = run_information
self._load_run_info()
self._load_timeseries_data()
self._setup_standard_streams()
def __init__(self, user, password):
#Create the interface to the data and load the partition data frame
self.interface = db.db_interface(host="192.168.13.51",
user=user,
passwd=password)
self.interface.connect()
q = db.use_Query("lab_proc_db")
self.interface.query(q)
self.runs = partitionDataframe()
self.runs.SQL_load_data(self.interface,
table='integral_summary',
conditions=[
"quality = 'G'",
])
def __init__(self, run_id, run_information=None):
#Create the gasifier data frame, and load the data from the SQL database (this will be automated through the interface later)
self.interface_raw = db.db_interface(host="192.168.10.20",
user="******",
passwd="cmp87ud01")
self.interface_raw.connect()
q = db.use_Query("sunulator2")
self.interface_raw.query(q)
self.interface_proc = db.db_interface(host="192.168.10.20",
user="******",
passwd="cmp87ud01")
self.interface_proc.connect()
q = db.use_Query("sunulator2")
self.interface_proc.query(q)
self.run_id = run_id
self.run_info = RunInformation()
self._load_run_info()
self._load_timeseries_data()
self._setup_standard_streams()
def testSQLError(self):
"""The dataframe should raise an error when the SQL library sends one back"""
data = df.Dataframe()
#error - no database selected
interface = SQL.db_interface(host = "192.168.10.20", user = "******", passwd = "cmp87ud01")
interface.connect()
self.assertRaises(df.dfSQLError, data.SQL_load_data, interface)
q = SQL.use_Query("gas_unit_test")
interface.query(q)
#error - bad table
self.assertRaises(df.dfSQLError, data.SQL_load_data, interface, "moat")
#error - bad condition
self.assertRaises(df.dfSQLError, data.SQL_load_data, interface, "dataframe_pd_test_table", ["denver>2.0"])
def generate_dataFrame(self):
start = self.startentry.get_text()
end=self.endentry.get_text()
dbconn = db.db_interface(host = "192.168.13.51", user = "******", passwd = "f9p2#nH1")
dbconn.connect()
q = db.use_Query("lab_run_db")
dbconn.query(q)
dtstart=datetime.datetime.strptime(start, '%Y-%m-%d %H:%M:%S')
dtend=datetime.datetime.strptime(end, '%Y-%m-%d %H:%M:%S')
df=lfl.ts_data(dtstart, dtend)
df.SQL_load(dbconn, table='gasifier_data_HR_view')
## df.get_ms_from_csv('12101214.csv')
df.interp_ga()
## df.calc_outlet_flowrate()
## df.calc_gas_produced_flowrate()
## df.calc_product_flowrates()
## df.calc_carbon_in(0.522)
## df.calc_carbon_out()
return df
def _setup_db_connection(self, user, password, host):
#set up the database connection
self.interface = db.db_interface(host=host, user=user, passwd=password)
self.interface.connect()
q = db.use_Query("pilot_proc_db")
self.interface.query(q)
def __init__(self, run_id, run_information=None):
#Create the gasifier data frame, and load the data from the SQL database (this will be automated through the interface later)
self.interface_raw = db.db_interface(host="192.168.10.20",
user="******",
passwd="cmp87ud01")
self.interface_raw.connect()
q = db.use_Query("sunulator2")
self.interface_raw.query(q)
self.interface_proc = db.db_interface(host="192.168.10.20",
user="******",
passwd="cmp87ud01")
self.interface_proc.connect()
q = db.use_Query("sunulator2")
self.interface_proc.query(q)
self.run_id = run_id
self.run_info = RunInformation()
self.run_integrals = RunInformation()
#self.run_info.info = run_information
self._load_run_info()
self._create_file_structure()
print "loading integral data..."
self._load_run_integrals()
print "loading timeseries data..."
self._load_ts_timeseries_data()
self._load_ss_timeseries_data()
print "converting units..."
self._add_units_to_run_info()
self._convert_gas_units_run_info()
self._convert_conversions_to_percent()
self._add_std_percent()
#self._convert_steam_flow_to_ml_min()
#Generate pie plot
print "creating pie plot..."
self.calc_gas_comp_pie_plot()
self.gas_pie_plot = PiePlot(
data=self.pie_gasvals,
keys=self.pie_goodgas,
figsize=(7, 7),
save_loc="%s%s" %
(self.directory, str(self.run_info.info['run_id'])))
self.gas_pie_plot.plot()
self.gas_pie_plot.save()
pie_LaTeX = self.gas_pie_plot.save_loc
self.gas_pie_plot.close()
self.run_info.info['piegas'] = pie_LaTeX
print "creating ts plots..."
self.ts['biomass_feedrate'] = self.ts['massflowrate'] * self.run_info[
'feeder_slope'] + self.run_info['feeder_intercept']
ts_plots = {}
#Generate time series plots - This should probably be encapsulated in an object -- you can see WAY too much about how this works; in fact, the chart info should be an object
ts_keys = [
'mass_feed', 'main_comp', 'trace_comp', 'tube_temps', 'zone_temps'
]
ts_Ycols = [['biomass_feedrate'],
['gc_co', 'gc_h2', 'gc_ch4', 'GC_CO2', 'GC_AR'],
['C2H4_GC', 'C2H6_GC', 'C2H2_GC', 'C3H8_GC', 'C3H6_GC'],
[
'tube_temp_1', 'tube_temp_2', 'tube_temp_3',
'tube_temp_4', 'tube_temp_5'
], ['zone1temp', 'zone2temp', 'zone3temp']]
ts_ylabels = [
'Biomass feed rate (lb/hr)', 'Gas Composition (% vol)',
'Gas Composition (%)', 'Tube Skin Temperatures ($^\circ$C)',
'Element Temperatures ($^\circ$C)'
]
ts_captions = [
'Time series plot for biomass flowrate',
'Time series plot for gas composition',
'Time series plot for trace gas composition',
'Time series plot for reactor tube skin temperatures',
'Time series plot for element temperatures'
]
ts_markers = ['-', 'o', 'o', '-', '-']
LaTeX_ts = ""
for key, cols, label, caption, marker in zip(ts_keys, ts_Ycols,
ts_ylabels, ts_captions,
ts_markers):
print cols
ts_plots[key] = TimeSeriesPlot(data=self.ts,
Y_cols=[cols],
y_labels=[label],
caption=caption,
save_loc="%s%s_%s" %
(self.directory, self.run_id, key),
markers=[marker])
ts_plots[key].plot()
ts_plots[key].fill(self.ss['timestamp'])
ts_plots[key].save()
LaTeX_ts += ts_plots[key].LaTeX_insert("ts_%s" % key)
ts_plots[key].close()
print "creating four plots..."
#Generate four plots
fp_plots = {}
fp_keys = [
'mass_feed', 'temp_mid', 'temp_steam', 'pressure', 'CO_GC',
'CO2_GC', 'H2_GC', 'CH4_GC'
]
fp_Y = [
'biomass_feedrate', 'tube_temp_3', 'steamtemp', 'pressure',
'gc_co', 'gc_h2', 'GC_CO2', 'gc_ch4'
]
fp_label = [
'Biomass Flow Rate (lbs/hr)', 'Reactor Skin Middle ($^\circ$C)',
'Steam Temperature ($^\circ$C)', 'System Pressure (psig)',
'Carbon Monoxide (mol%)', 'Hydrogen (mol%)',
'Carbon Dioxide (mol%)', 'Methane (mol%)'
]
fp_caption = [
'Four-plot for biomass flow rate',
'Four-plot for reactor skin temperature',
'Four-plot for temperature of steam at reactor inlet',
'Four-plot for system pressure',
'Four-plot for carbon monoxide readings from the gas chromatograph',
'Four-plot for hydrogen readings from the gas chromatograph',
'Four-plot for carbon dioxide readings from the gas chromatograph',
'Four-plot for methane readings from the gas chromatograph'
]
LaTeX_fp = ""
for key, Y, label, caption in zip(fp_keys, fp_Y, fp_label, fp_caption):
fp_plots[key] = FourPlot(data=self.ss,
x_label='Time',
y_label=label,
x_var='timestamp',
y_var=Y,
caption=caption,
save_loc="%s%s_%s" %
(self.directory, self.run_id, key))
fp_plots[key].plot()
fp_plots[key].save()
LaTeX_fp += fp_plots[key].LaTeX_insert("fp_%s" % key)
fp_plots[key].close()
print "creating control charts..."
#Create ARIMA fits as necessary - will not work for NaN data (i.e. raw MS data -- that should not be autocorrellated anyway)
ARIMA_list = ['biomass_feedrate', 'steamtemp']
for col in ARIMA_list:
#try:
self.fit_ARIMA(col)
#except Exception:
# ARIMA_list.remove(col)
ARIMA_captions = {
'biomass_feedrate': 'biomass flow rate',
'steamtemp': 'reactor inlet steam temperature'
}
cp_plots = {}
cp_keys = [
'mass_feed', 'temp_mid', 'temp_steam', 'pressure', 'CO_GC',
'CO2_GC', 'H2_GC', 'CH4_GC', 'mass_feed_ARIMA', 'temp_steam_ARIMA'
]
cp_Y = [
'biomass_feedrate', 'tube_temp_3', 'steamtemp', 'pressure',
'gc_co', 'gc_h2', 'GC_CO2', 'gc_ch4'
]
cp_caption = []
items = [
'biomass flow rate', 'reactor skin temperature',
'temperature of steam at reactor inlet', 'pressure',
'carbon monoxide (GC)', 'hydrogen (GC)', 'carbon dioxide (GC)',
'methane (GC)'
]
for col in ARIMA_list:
cp_keys.append('%s_ARIMA' % col)
cp_Y.append('%s_ARIMA_resid' % col)
items.append(ARIMA_captions[col])
for item in items:
cp_caption.append(
"Individuals control chart for %s ARIMA(1,1) residuals" % item)
LaTeX_cp = ""
for key, Y, caption in zip(cp_keys, cp_Y, cp_caption):
input_df = ControlChartfromDataframe(data=self.ss,
y_col=Y,
x_col='timestamp',
ignore_nan=True)
cp_plots[key] = IndividualsXBarControlChart(
data=input_df.getDataframe(),
caption=caption,
save_loc="%s%s_%s" % (self.directory, self.run_id, key))
cp_plots[key].plot()
cp_plots[key].annotate(1)
cp_plots[key].annotate(2)
cp_plots[key].save()
LaTeX_cp += cp_plots[key].LaTeX_insert("cp_%s" % key)
cp_plots[key].close()
self.run_info.info['timeseriesplots'] = LaTeX_ts
self.run_info.info['fourplots'] = LaTeX_fp
self.run_info.info['controlplots'] = LaTeX_cp
print "generating standard report..."
self.generate_standard_report()
def __init__(self, run_id, user, password, run_information=None):
#Create the gasifier data frame, and load the data from the SQL database (this will be automated through the interface later)
self.interface_raw = db.db_interface(host="192.168.13.51",
user=user,
passwd=password)
self.interface_raw.connect()
q = db.use_Query("lab_run_db")
self.interface_raw.query(q)
self.interface_proc = db.db_interface(host="192.168.13.51",
user=user,
passwd=password)
self.interface_proc.connect()
q = db.use_Query("lab_proc_db")
self.interface_proc.query(q)
print "databases connected"
self.run_id = run_id
self.run_info = RunInformation()
self.run_integrals = RunInformation()
#self.run_info.info = run_information
self._load_run_info()
self._create_file_structure()
self._load_run_integrals()
self._load_ts_timeseries_data()
self._load_ss_timeseries_data()
self._add_units_to_run_info()
self._convert_gas_units_run_info()
self._convert_conversions_to_percent()
self._add_std_percent()
self._convert_steam_flow_to_ml_min()
#Generate pie plot
self.calc_gas_comp_pie_plot()
self.gas_pie_plot = PiePlot(
data=self.pie_gasvals,
keys=self.pie_goodgas,
figsize=(7, 7),
save_loc="%s%s" %
(self.directory, str(self.run_info.info['run_id'])))
self.gas_pie_plot.plot()
self.gas_pie_plot.save()
pie_LaTeX = self.gas_pie_plot.save_loc
self.gas_pie_plot.close()
self.run_info.info['piegas'] = pie_LaTeX
ts_plots = {}
#Generate time series plots
ts_keys = ['mass_feed', 'main_comp', 'trace_comp', 'tube_temps']
ts_Ycols = [['mass_flow_brush_feeder'],
['CO_MS', 'CO2_MS', 'H2_MS', 'CH4_MS', 'Ar_MS'],
['C2H4_MS', 'C6H6_MS', 'C7H8_MS', 'C10H8_MS'],
[
'temp_skin_tube_top', 'temp_skin_tube_middle',
'temp_skin_tube_bottom'
]]
ts_ylabels = [
'Biomass feed rate (lb/hr)', 'Gas Composition (% vol)',
'Gas Composition (ppm)', 'Tube Skin Temperatures ($^\circ$C)'
]
ts_captions = [
'Time series plot for biomass flowrate',
'Time series plot for gas composition',
'Time series plot for gas composition',
'Time series plot for reactor tube skin temperatures'
]
ts_markers = ['-', 'o', 'o', '-']
LaTeX_ts = ""
for key, cols, label, caption, marker in zip(ts_keys, ts_Ycols,
ts_ylabels, ts_captions,
ts_markers):
ts_plots[key] = TimeSeriesPlot(data=self.ts,
Y_cols=[cols],
y_labels=[label],
caption=caption,
save_loc="%s%s_%s" %
(self.directory, self.run_id, key),
markers=[marker])
ts_plots[key].plot()
ts_plots[key].fill(self.ss['ts'])
ts_plots[key].save()
LaTeX_ts += ts_plots[key].LaTeX_insert("ts_%s" % key)
ts_plots[key].close()
#Generate four plots
fp_plots = {}
fp_keys = [
'mass_feed', 'temp_mid', 'temp_steam', 'pressure_KO', 'CO_MS',
'CO2_MS', 'H2_MS', 'CH4_MS'
]
fp_Y = [
'mass_flow_brush_feeder', 'temp_skin_tube_middle',
'temp_steam_reactor_entry', 'pressure_ash_knockout_vessel',
'CO_MS', 'H2_MS', 'CO2_MS', 'CH4_MS'
]
fp_label = [
'Biomass Flow Rate (lbs/hr)', 'Reactor Skin Middle ($^\circ$C)',
'Steam Temperature ($^\circ$C)', 'Ash Knockout Pressure (psig)',
'Carbon Monoxide (mol%)', 'Hydrogen (mol%)',
'Carbon Dioxide (mol%)', 'Methane (mol%)'
]
fp_caption = [
'Four-plot for biomass flow rate',
'Four-plot for reactor skin temperature',
'Four-plot for temperature of steam at reactor inlet',
'Four-plot for ash knockout pressure',
'Four-plot for carbon monoxide readings from the mass spectrometer',
'Four-plot for hydrogen readings from the mass spectrometer',
'Four-plot for carbon dioxide readings from the mass spectrometer',
'Four-plot for methane readings from the mass spectrometer'
]
LaTeX_fp = ""
for key, Y, label, caption in zip(fp_keys, fp_Y, fp_label, fp_caption):
fp_plots[key] = FourPlot(data=self.ss,
x_label='Time',
y_label=label,
x_var='ts',
y_var=Y,
caption=caption,
save_loc="%s%s_%s" %
(self.directory, self.run_id, key))
fp_plots[key].plot()
fp_plots[key].save()
LaTeX_fp += fp_plots[key].LaTeX_insert("fp_%s" % key)
fp_plots[key].close()
#Create ARIMA fits as necessary - will not work for NaN data (i.e. raw MS data -- that should not be autocorrellated anyway)
ARIMA_list = ['mass_flow_brush_feeder', 'temp_steam_reactor_entry']
for col in ARIMA_list:
try:
self.fit_ARIMA(col)
except Exception:
ARIMA_list.remove(col)
ARIMA_captions = {
'mass_flow_brush_feeder': 'biomass flow rate',
'temp_steam_reactor_entry': 'reactor inlet steam temperature'
}
cp_plots = {}
cp_keys = [
'mass_feed', 'temp_mid', 'temp_steam', 'pressure_KO', 'CO_MS',
'CO2_MS', 'H2_MS', 'CH4_MS', 'X_tot', 'mass_feed_ARIMA',
'temp_steam_ARIMA'
]
cp_Y = [
'mass_flow_brush_feeder', 'temp_skin_tube_middle',
'temp_steam_reactor_entry', 'pressure_ash_knockout_vessel',
'CO_MS', 'H2_MS', 'CO2_MS', 'CH4_MS', 'X_tot'
]
cp_caption = []
items = [
'biomass flow rate', 'reactor skin temperature',
'temperature of steam at reactor inlet', 'ash knockout pressure',
'carbon monoxide (MS)', 'hydrogen (MS)', 'carbon dioxide (MS)',
'methane (MS)', 'total conversion'
]
for col in ARIMA_list:
cp_keys.append('%s_ARIMA' % col)
cp_Y.append('%s_ARIMA_resid' % col)
items.append(ARIMA_captions[col])
for item in items:
cp_caption.append(
"Individuals control chart for %s ARIMA(1,1) residuals" % item)
LaTeX_cp = ""
for key, Y, caption in zip(cp_keys, cp_Y, cp_caption):
input_df = ControlChartfromDataframe(data=self.ss,
y_col=Y,
x_col='ts',
ignore_nan=True)
cp_plots[key] = IndividualsXBarControlChart(
data=input_df.getDataframe(),
caption=caption,
save_loc="%s%s_%s" % (self.directory, self.run_id, key))
cp_plots[key].plot()
cp_plots[key].annotate(1)
cp_plots[key].annotate(2)
cp_plots[key].save()
LaTeX_cp += cp_plots[key].LaTeX_insert("cp_%s" % key)
cp_plots[key].close()
self.run_info.info['timeseriesplots'] = LaTeX_ts
self.run_info.info['fourplots'] = LaTeX_fp
self.run_info.info['controlplots'] = LaTeX_cp
self.generate_standard_report()