def load_sep_cycles(file_name, database_name, datatype):
"""Get data from a specified file, separates out data into
cycles and saves those cycles as .xlsx files in specified
filepath (must be an existing folder)"""
#df_single = pd.read_excel(file_name,1)
(cycle_ind_col, data_point_col, volt_col, curr_col, dis_cap_col,
char_cap_col, charge_or_discharge) = col_variables(datatype)
while '/' in file_name:
file_name = file_name.split('/', maxsplit=1)[1]
name = file_name.split('.')[0] + 'Raw'
#name = file_name.split('.')[0] + 'Raw'
df_single = dbfs.get_file_from_database(name, database_name)
gb = df_single.groupby(by=[cycle_ind_col])
cycle_dict = dict(iter(gb))
battname = file_name.split('.')[0]
for i in range(1, len(cycle_dict) + 1):
cycle_dict[i]['Battery_Label'] = battname
for i in range(1, len(cycle_dict) + 1):
dbfs.update_database_newtable(cycle_dict[i],
battname + '-' + 'Cycle' + str(i),
database_name)
print('All data separated into cycles and saved in database.')
return cycle_dict
def process_data(file_name, database_name, path, datatype, username):
# Takes raw file
# sep_cycles
# cleans cycles
# gets descriptors - peak calcs
# put back together - save
thresh1 = '0.0'
thresh2 = '0.0'
if datatype == 'Arbin':
datatype = 'CALCE'
if not os.path.exists(database_name):
print('That database does not exist-creating it now.')
dbfs.init_master_table(database_name)
con = sql.connect(database_name)
c = con.cursor()
names_list = []
for row in c.execute(
"""SELECT name FROM sqlite_master WHERE type='table'"""):
names_list.append(row[0])
con.close()
file_name3 = file_name
while '/' in file_name3:
file_name3 = file_name3.split('/', maxsplit=1)[1]
name3 = file_name3.split('.')[0]
if name3 + 'Raw' in names_list:
print('That file name has already been uploaded into the database.')
else:
#datatype = input('What datatype do you have (either CALCE or MACCOR)')
print('Processing that data')
parse_update_master(file_name, database_name, datatype, path, username)
# this takes the info from the filename and updates the master table in the database.
# this also adds the raw data fram into the database
cycle_dict = ccf.load_sep_cycles(file_name, database_name, datatype)
clean_cycle_dict = ccf.get_clean_cycles(cycle_dict, file_name,
database_name, datatype,
thresh1, thresh2)
clean_set_df = ccf.get_clean_sets(clean_cycle_dict, file_name,
database_name)
cleanset_name = name3 + 'CleanSet'
df_clean = dbfs.get_file_from_database(cleanset_name, database_name)
v_toappend_c = []
v_toappend_d = []
new_peak_thresh = 0.3
# just as a starter value
feedback = app.generate_model(v_toappend_c, v_toappend_d, df_clean,
file_name, new_peak_thresh,
database_name)
# desc_df = descriptors.get_descriptors(clean_cycle_dict, datatype, windowlength = 3, polyorder = 1)
# dbfs.update_database_newtable(desc_df, name3 + '-descriptors', database_name)
print('Database updated with descriptors.')
return
def param_dicts_to_df(mod_params_name, database):
mod_params_df = dbfs.get_file_from_database(mod_params_name, database)
#charge_df = mod_paramsgraphit[mod_paramsgraphit['C/D'] == 'discharge']
#charge_df = charge_df.reset_index(drop = True)
#charge_df = mod_params_df
charge_descript = pd.DataFrame()
discharge_descript = pd.DataFrame()
for i in range(len(mod_params_df)):
param_dict_charge = ast.literal_eval(
mod_params_df.loc[i, ('Model_Parameters_charge')])
param_dict_discharge = ast.literal_eval(
mod_params_df.loc[i, ('Model_Parameters_discharge')])
charge_peak_heights = ast.literal_eval(
mod_params_df.loc[i, ('charge_peak_heights')])
discharge_peak_heights = ast.literal_eval(
mod_params_df.loc[i, ('discharge_peak_heights')])
charge_keys = []
new_dict_charge = {}
if param_dict_charge is not None:
for key, value in param_dict_charge.items():
if '_amplitude' in key and not 'base_' in key:
#print(int(key.split('_')[0].split('a')[1]))
charge_keys.append(key.split('_')[0])
#print(charge_keys)
new_dict_charge.update({
'c_gauss_sigma':
param_dict_charge[
'base_sigma'], # changed from c0- c4 to base_ .. 10-10-18
'c_gauss_center':
param_dict_charge['base_center'],
'c_gauss_amplitude':
param_dict_charge['base_amplitude'],
'c_gauss_fwhm':
param_dict_charge['base_fwhm'],
'c_gauss_height':
param_dict_charge['base_height'],
#'c_poly_coef5': param_dict_charge['c4'],
})
new_dict_charge.update(
{'c_cycle_number': float(mod_params_df.loc[i, ('Cycle')])})
#new_dict.update({'charge/discharge': mod_params_df.loc[i, ('C/D')],
#'cycle_number': float(mod_params_df.loc[i, ('Cycle')])})
peaknum = 0
for item in charge_keys:
peaknum = peaknum + 1
center = param_dict_charge[item + '_center']
amp = param_dict_charge[item + '_amplitude']
fract = param_dict_charge[item + '_fraction']
sigma = param_dict_charge[item + '_sigma']
height = param_dict_charge[item + '_height']
fwhm = param_dict_charge[item + '_fwhm']
raw_peakheight = charge_peak_heights[peaknum - 1]
#print('center' + str(center))
PeakArea, PeakAreaError = scipy.integrate.quad(my_pseudovoigt,
0.0,
100,
args=(center, amp,
fract, sigma))
#print('peak location is : ' + str(center) + ' Peak area is: ' + str(PeakArea))
new_dict_charge.update({
'c_area_peak_' + str(peaknum):
PeakArea,
'c_center_peak_' + str(peaknum):
center,
'c_amp_peak_' + str(peaknum):
amp,
'c_fract_peak_' + str(peaknum):
fract,
'c_sigma_peak_' + str(peaknum):
sigma,
'c_height_peak_' + str(peaknum):
height,
'c_fwhm_peak_' + str(peaknum):
fwhm,
'c_rawheight_peak_' + str(peaknum):
raw_peakheight
})
#print(new_dict)
new_dict_df = pd.DataFrame(columns=new_dict_charge.keys())
for key1, val1 in new_dict_charge.items():
new_dict_df.at[0, key1] = new_dict_charge[key1]
charge_descript = pd.concat([charge_descript, new_dict_df])
charge_descript = charge_descript.reset_index(drop=True)
charge_descript2 = descriptors.dfsortpeakvals(charge_descript, 'c')
################################ now the discharge
discharge_keys = []
if param_dict_discharge is not None:
for key, value in param_dict_discharge.items():
if '_amplitude' in key and not 'base_' in key:
#print(int(key.split('_')[0].split('a')[1]))
discharge_keys.append(key.split('_')[0])
#print(charge_keys)
new_dict_discharge = {}
new_dict_discharge.update({
'd_gauss_sigma':
param_dict_discharge['base_sigma'], # changed 10-10-18
'd_gauss_center':
param_dict_discharge['base_center'],
'd_gauss_amplitude':
param_dict_discharge['base_amplitude'],
'd_gauss_fwhm':
param_dict_discharge['base_fwhm'],
'd_gauss_height':
param_dict_discharge['base_height'],
#'d_poly_coef5': param_dict_discharge['c4'],
})
new_dict_discharge.update(
{'d_cycle_number': float(mod_params_df.loc[i, ('Cycle')])})
#new_dict.update({'charge/discharge': mod_params_df.loc[i, ('C/D')],
#'cycle_number': float(mod_params_df.loc[i, ('Cycle')])})
peaknum = 0
for item in discharge_keys:
peaknum = peaknum + 1
center = param_dict_discharge[item + '_center']
amp = param_dict_discharge[item + '_amplitude']
fract = param_dict_discharge[item + '_fraction']
sigma = param_dict_discharge[item + '_sigma']
height = param_dict_discharge[item + '_height']
fwhm = param_dict_discharge[item + '_fwhm']
raw_peakheight = discharge_peak_heights[peaknum - 1]
#print('center' + str(center))
PeakArea, PeakAreaError = scipy.integrate.quad(
my_pseudovoigt, 0.0, 100, args=(center, amp, fract, sigma))
#print('peak location is : ' + str(center) + ' Peak area is: ' + str(PeakArea))
new_dict_discharge.update({
'd_area_peak_' + str(peaknum):
PeakArea,
'd_center_peak_' + str(peaknum):
center,
'd_amp_peak_' + str(peaknum):
amp,
'd_fract_peak_' + str(peaknum):
fract,
'd_sigma_peak_' + str(peaknum):
sigma,
'd_height_peak_' + str(peaknum):
height,
'd_fwhm_peak_' + str(peaknum):
fwhm,
'd_rawheight_peak_' + str(peaknum):
raw_peakheight
})
else:
new_dict_discharge = None
#print(new_dict)
if new_dict_discharge is not None:
new_dict_df_d = pd.DataFrame(columns=new_dict_discharge.keys())
for key1, val1 in new_dict_discharge.items():
new_dict_df_d.at[0, key1] = new_dict_discharge[key1]
discharge_descript = pd.concat([discharge_descript, new_dict_df_d])
discharge_descript = discharge_descript.reset_index(drop=True)
discharge_descript2 = descriptors.dfsortpeakvals(
discharge_descript, 'd')
else:
discharge_descript2 = None
# append the two dfs (charge and discharge) before putting them in database
full_df_descript = pd.concat([charge_descript2, discharge_descript2],
axis=1)
dbfs.update_database_newtable(full_df_descript,
mod_params_name + '-descriptors',
database)
return