def merge_longforms(self):
for key in self.merge_dict:
make_dir(self.graph_path + 'final_longform_csv/' + key)
try:
for tag in self.tag_list:
df_list = []
for name in self.merge_dict[key]:
try:
path = self.graph_path + 'basic_longform_csv/' + name + '/' + name + '_' + tag + '.csv'
print('Merge', name, 'to', key, 'from', path)
df = pd.read_csv(path,
delimiter=',',
index_col=self.index_name)
df = df.rename(columns={df.columns[-3]: key})
df.columns = df.columns.str.replace('_', ' ')
df_list.append(df)
print(df)
except Exception as e:
print(e)
concat_df = pd.concat(df_list)
concat_df.to_csv(self.graph_path + 'final_longform_csv/' +
key + '/' + key + '_' + tag + '.csv')
except Exception as e:
print(e)
def __init__(self,
D_PATH='_BIG_D/',
period_string_min='5min',
full_dataset=True):
self.sum_total_power_df = mainDataset(D_PATH, period_string_min,
full_dataset).load_total_power()
self.sum_total_power_array = self.sum_total_power_df.to_numpy()
self.D_PATH = D_PATH + 'statistics/dataset-statistics/'
make_dir(self.D_PATH)
def __init__(self, main_dataset, df, num_past_periods=12):
self.D_PATH = main_dataset.__dict__['D_PATH']
self.period_string_min = main_dataset.__dict__['period_string_min']
self.name = main_dataset.__dict__['lstm_name']
self.alle_inputs_df = df
self.num_past_periods = num_past_periods
self.timer = Timer()
make_dir(self.D_PATH + 'tables/' + self.period_string_min +
'/training-data/')
def logs_to_csv(self):
working_dir = os.getcwd()
for name in self.all_names:
make_dir(self.graph_path + 'seperate_log_csv/' + name)
try:
if self.mode == 'compare':
path = self.log_path + '*' + name + '_t-stamp*'
else:
path = self.log_path + '*' + name + '*'
for folder_path in iglob(path):
print('path:', folder_path)
csv_name = folder_path.split('\\')[-1]
#name = folder_path.split('/')[-1]
ea = event_accumulator.EventAccumulator(folder_path)
ea.Reload()
#self.csv_path_list = []
if self.custom_tags == False:
self.tag_list = []
#for tag in self.tag_list:
for tag in ea.Tags()['scalars']:
try:
tag_str = tag.replace(':', '').split(' ')[-1]
print(tag_str)
#os.chdir(self.graph_path+'seperate_log_csv/'+name+'/')
csv_path = self.graph_path + 'seperate_log_csv/' + name + '/' + csv_name + '-tag-' + tag_str + '.csv'
#print(pd.DataFrame(ea.Scalars(tag)))
pd.DataFrame(ea.Scalars(tag)).to_csv(csv_path)
#print(csv_path)
#test.to_csv(csv_name+'-tag-'+tag_str+'.csv')
#os.chdir(working_dir)
if self.custom_tags == False:
self.tag_list.append(tag_str)
#self.csv_path_list.append(csv_path)
except Exception as e:
print('Exception:', e)
except Exception as e:
print('Exception:', e)
print('Could not open any log with path:', self.log_path,
'that includes', name)
def plot_lstm_output(self):
path = self.graph_path + 'output_graphs/all_runs'
make_dir(path)
index_list = []
max_list = []
mean_list = []
for csv in iglob('lstm-outputs/*.csv'):
print(csv)
csv_name = csv.split('\\')[-1]
df = pd.read_csv(csv, delimiter=',', index_col=[0])
df = df.rename(columns={df.columns[0]: 'prediction'})
df = df.rename(columns={df.columns[1]: 'real value'})
df = df.rename(columns={df.columns[2]: 'absoulte error'})
df = df.rename(columns={df.columns[3]: 'squared error'})
print(df)
index_list.append(csv_name)
max_list.append(np.max(df['absoulte error'].to_numpy()))
mean_list.append(np.mean(df['absoulte error'].to_numpy()))
df.plot()
plt.ylabel('energy demand in kw')
plt.xlabel('step')
plt.savefig(path + '/' + csv_name + '.svg')
plt.close()
error_df = pd.DataFrame({
'run': index_list,
'max_error': max_list,
'mean_error': mean_list,
})
print(error_df)
error_df.to_csv(self.graph_path + 'output_graphs/lstm_error.csv')
def coulmn_to_smoothed_period_df(
self,
dataset_name,
coulmn_name,
c_num=None,
c_total=None
): # returns Datafram: Strombedarf (nur noch eine Spalte, Index = SensorDateTime, neuerstellt)
''' Tries to open the dataset for a specific machine that is already smoothed to the time-period. Creates a new dataset if a dataset for the given time-period can not be opened.
Used by :meth:`schaffer.mainDataset.smoothed_df`.
Args:
dataset_name (string): The name of the downloaded HIPE-dataset for a specific machine.
coulmn_name (string): The name of the new dataset for a specif machine, that will be later used as a column name when all the machine-datasets are merged.
c_num (int): Can be used to show the progress and has to be the count of current the machine-dataset
c_total (int): Can be used to show the progress and has to be the sum of all machine-dataset
Returns:
dataframe: The smoothed dataset for a given machine
'''
# Falls möglich Daten öffnen
try:
dataframe = pd.read_csv(
self.D_PATH + 'tables/' + self.period_string_min +
'/single-column-tables/' + coulmn_name + '.csv',
index_col='SensorDateTime',
parse_dates=True)
# Sonst erstellen
except:
self.timer.start()
print(
'Could not open:',
self.D_PATH + 'tables/' + self.period_string_min +
'/single-column-tables/' + coulmn_name + '.csv')
print('Creating new /single-column-tables/' + coulmn_name +
'.csv...')
make_dir(self.D_PATH + 'tables/' + self.period_string_min +
'/single-column-tables/')
dataframe = pd.read_csv('dataset/' + dataset_name + '.csv',
index_col='SensorDateTime',
parse_dates=True)
dataframe = dataframe["P_kW"].to_frame()
# Converting the index as date
dataframe.index = pd.to_datetime(dataframe.index, utc=True)
dataframe = dataframe.resample(
self.period_string_min).mean().rename(
columns={"P_kW": coulmn_name})
dataframe.to_csv(self.D_PATH + 'tables/' + self.period_string_min +
'/single-column-tables/' + coulmn_name + '.csv')
self.timer.stop()
if c_num == None and c_total == None:
print('Created dataset for', coulmn_name, 'from', dataset_name)
print('Elapsed time:', self.timer.elapsed_time_string())
else:
print('Created dataset for', coulmn_name, 'from', dataset_name,
'({}/{})'.format(c_num, c_total))
print('Elapsed time:', self.timer.elapsed_time_string())
print(
'Estimated time to create the rest of /single-column-tables/:',
self.timer.rest_time_string(c_num, c_total))
print('Table ' + self.D_PATH + 'tables/' + self.period_string_min +
'/single-column-tables/' + coulmn_name +
'.csv loaded successfully')
return dataframe
def __init__(self, D_PATH='_BIG_D/', period_min=5, full_dataset=False):
self.D_PATH = D_PATH
self.period_min = period_min
self.period_string_min = '{}min'.format(period_min)
self.full_dataset = full_dataset
self.timer = Timer()
make_dir(self.D_PATH + 'lstm-models/')
make_dir(self.D_PATH + 'lstm-logs/')
make_dir(self.D_PATH + 'lstm-outputs/')
make_dir(self.D_PATH + 'agent-models/')
make_dir(self.D_PATH + 'agent-logs/')
make_dir(self.D_PATH + 'agent-outputs/')
def simple_plot(self, name, df=None, path=None, save_path=None):
if save_path == None:
make_dir(self.graph_path + 'graphs/' + name)
for tag in self.tag_list:
try:
if path == None:
if not isinstance(df, pd.DataFrame):
path = self.graph_path + 'final_longform_csv/' + name + '/' + name + '_' + tag
if not isinstance(df, pd.DataFrame):
df = pd.read_csv(path + '.csv')
#df = df.drop(columns=[df.columns[0]])
print(df)
'''
if self.graph_name == True:
plt.title(name.replace('_',' '))
sns.lineplot(data=df, dashes=False, x=df.columns[0], y=df.columns[1], hue="type", palette=self.sns_palette)#, label=df.columns[-1])
#plt.xlabel(df.index.names[0])
plt.xlabel('epoch')
plt.ylabel(tag)
#plt.legend(bbox_to_anchor=(1.01, 1),borderaxespad=0)
plt.legend(bbox_to_anchor=(1.02, 1),loc='right')
#plt.savefig(self.graph_path+'graphs/'+name+'/'+name+'_'+tag+'.png')
plt.close()
'''
fig, ax1 = plt.subplots(1, 1)
g = sns.lineplot(data=df,
dashes=False,
x=df.columns[0],
y=df.columns[1],
hue="type",
palette=self.sns_palette,
ax=ax1,
legend=self.legend) #, label=df.columns[-1])
'''
box = g.get_position()
g.set_position([box.x0, box.y0, box.width * 0.85, box.height]) # resize position
# Put a legend to the right side
g.legend(loc='center right', bbox_to_anchor=(1.33, 0.5), ncol=1)
'''
if self.graph_name == True or self.title != None:
if self.title == None:
plt.title(name.replace('_', ' '))
else:
plt.title(self.title)
if self.y_label == None:
plt.ylabel(tag.replace('_', ' '))
else:
plt.ylabel(self.y_label)
plt.xlabel(self.index_name.replace('_', ' '))
#plt.show()
if save_path == None:
save_path = self.graph_path + 'graphs/' + name + '/' + name + '_' + tag + '_' + str(
self.rolling_mean) + '.svg'
plt.savefig(save_path)
plt.close()
print('Saved graph to:', save_path)
except Exception as e:
print(e)
def create_basic_longforms(self):
for name in self.all_names:
make_dir(self.graph_path + 'basic_longform_csv/' + name)
make_dir(self.graph_path + 'learn_time/' + name)
learn_time_dict_list = []
for tag in self.tag_list:
try:
print('Creating longform for', name, 'with', tag, '...')
df_list = []
name_list = []
for filename in iglob(self.graph_path +
'seperate_log_csv/' + name +
'/*-tag-' + tag + '.csv'):
print(filename)
# Name:
df = pd.read_csv(filename, delimiter=',')
df = df.set_index('step')
df = df.rename(columns={df.columns[-1]: name})
df = df.rename(
columns={df.columns[0]: self.index_name})
if self.rolling_mean != 1:
df[name] = df[name].rolling(
self.rolling_mean).mean()
# Index:
df.columns = df.columns.str.replace('_', ' ')
df.index.names = [self.index_name]
# drop unnamed:
df = df.drop(columns=[df.columns[0]])
# Add column for type:
if any(type_string in name
for type_string in self.name_type_list):
type_string = name.split('_')[-1]
df['type'] = type_string
else:
type_string = filename.split(
self.type_first_split)[-1]
type_string = type_string.split(
self.type_second_split)[0]
type_string = type_string.split('_')[-1]
df['type'] = type_string
# Add column for run number:
run = name_list.count(type_string)
df['run'] = run
# Calculate learning time:
learn_time_dict_list.append(
self.calc_learn_time(df[df.columns[-4]], tag, name,
run, type_string))
df = df.drop(columns=[df.columns[-4]])
#df.name = filename
df_list.append(df)
name_list.append(type_string)
# Save longform table:
concat_df = pd.concat(df_list)
concat_df.to_csv(self.graph_path + 'basic_longform_csv/' +
name + '/' + name + '_' + tag + '.csv')
except Exception as e:
print(e)
# Save learning time table:
learn_time_df = pd.DataFrame(learn_time_dict_list)
learn_time_df.to_csv(self.graph_path + 'learn_time/' + name + '/' +
name + '_time_comparison.csv')
'''