def plot_intersection_matrix(elements: List[CMapMatrixElement], index, save):
# fig = graph.plot_matrix_cmap_plain(
# elements, nvalves, nrows, "Valve Sequence", "sample x" + str(rowskip), "valves", xlabels, ylabels)
fig = graph.plot_matrix_cmap_plain(elements, nvalves, nrows, "", "",
"valves", xlabels, ylabels, None)
if save:
graph.save_figure(fig, "./figs/valve_sequence_" + str(index))
def plot_data(x, y, xheader, yheader):
tss = create_timeseries(y, yheader)
tss2 = create_timeseries(x, xheader)
# print(json.dumps(acc, indent=2))
# fig, _ = graph.plot_timeseries_multi(tss, "valve sequence", "samples [x0.1s]", "position [%]", False)
fig, _ = graph.plot_timeseries_multi_sub2(
[tss, tss2], ["valve sequence", "sensor output"], "samples [x0.1s]",
["position [%]", "flow [L/h]"], (9, 16))
graph.save_figure(fig, "./figs/valve_sequence_" + filename)
nrowskip = 0
nrows += 1
for e in elements_buffer:
elements.append(e)
except:
pass
# print(elements)
# quit()
xlabels = [("v" + str(i + 1)) for i in range(nvalves)]
ylabels = [(str(int(i * rowskip / 100))) for i in range(nrows)]
# xlabels = []
# ylabels = []
# intersection_matrix = np.random.randint(0, 10, size=(max_val, max_val))
intersection_matrix = np.zeros((nvalves, nrows))
# print(intersection_matrix)
for e in elements:
intersection_matrix[e.i][e.j] = e.val
print(intersection_matrix)
fig = graph.plot_matrix_cmap_plain(elements, nvalves, nrows, "Valve Sequence",
"sample x" + str(rowskip), "valves",
xlabels, ylabels, None, None)
graph.save_figure(fig, "./figs/valve_sequence")
csvoutput = ""
for e in vect:
# print(e)
csvoutput += e + "," + \
str(vect[e]["avg"]["acc"]) + "," + str(vect[e]["top"]["acc"]) + \
"," + str(vect[e]["avg"]["dt"]) + "," + str(vect[e]["avg"]
["fsize"]) + "," + str(vect[e]["top_file"]) + "\n"
return csvoutput
csvoutput = extract_csv(acc_train_vect)
with open("./data/output/eval_" + output_filename + "_train.csv", "w") as f:
f.write(csvoutput)
csvoutput = extract_csv(acc_test_vect)
with open("./data/output/eval_" + output_filename + "_test.csv", "w") as f:
f.write(csvoutput)
# print(csvoutput)
tss = create_barseries([acc_train_vect, acc_test_vect], ["train", "test"])
fig = graph.plot_barchart_multi(
tss, "model", "accuracy", "Average accuracy (Naive Bayes)",
["1-N-80%", "1-N-1-80%", "1-N-1-50%", "GRAY-80%"], [70, 120])
graph.save_figure(fig, "./figs/mean_accuracy_" + output_filename)
def plot_intersection_matrix(elements, index, nrows, ncols, save, scale):
fig = graph.plot_matrix_cmap_plain(
elements, nrows, ncols, "Sequence evaluation", "sample bins", "", xlabels, ylabels, scale, (16,7))
if save:
graph.save_figure(fig, "./figs/valve_sequence_random_prediction")
def plot_intersection_matrix2(elements, index, nrows, ncols, save, scale):
fig = graph.plot_matrix_cmap_plain(elements, nrows, ncols, "", "", "",
xlabels, ylabels, scale, None)
if save:
graph.save_figure(fig, "./figs/valve_sequence_" + str(index))
def extract_csv(vect):
csvoutput = ""
for e in vect:
# print(e)
csvoutput += e + "," + \
str(vect[e]["avg"]) + "," + str(vect[e]["top"]) + \
"," + str(vect[e]["top_file"]) + "\n"
return csvoutput
csvoutput = extract_csv(acc_train_vect)
with open("./data/output/eval_dtree_1_train.csv", "w") as f:
f.write(csvoutput)
csvoutput = extract_csv(acc_test_vect)
with open("./data/output/eval_dtree_1_test.csv", "w") as f:
f.write(csvoutput)
# print(csvoutput)
tss = create_barseries([acc_train_vect, acc_test_vect], ["train", "test"])
fig = graph.plot_barchart_multi(
tss, "model", "accuracy", "Average accuracy (decision tree)",
["1-N-80%", "1-N-1-80%", "1-N-1-50%", "GRAY-80%"], [70, 120])
graph.save_figure(fig, "./figs/mean_accuracy_dtree_1")
print(np.shape(x))
print(np.shape(y))
# tss = create_timeseries(np.concatenate((x,y), axis=1), np.concatenate((xheader,yheader)))
tss = create_timeseries(y, yheader)
x = remove_outliers(x)
tss2 = create_timeseries(x, xheader)
# print(json.dumps(acc, indent=2))
# fig, _ = graph.plot_timeseries_multi(tss, "valve sequence", "samples [x0.1s]", "position [%]", False)
fig, _ = graph.plot_timeseries_multi_sub2(
[tss, tss2], ["valve sequence", "sensor output"], "samples [x0.1s]",
["position [%]", "flow [L/h]"])
graph.save_figure(fig, "./figs/valve_sequence_" + filename)
# x = remove_outliers(x)
# tss = create_timeseries(x, xheader)
# fig, _ = graph.plot_timeseries_multi(tss, "sensor output", "samples [x0.1s]", "flow [L/h]", False)
# graph.save_figure(fig, "./figs/sensor_output")
# graph.plot_timeseries(ts, "title", "x", "y")
# quit()
res2 = [[e[1], gstdev_pump] for e in res]
tss7 = create_timeseries(res2, header_aux)
print("shape:")
print(np.shape(data2))
print(np.shape(res1))
# head = ["", "", "", ""]
# axhead = ["yk (avg)", "yk (stdev)", "uk (avg)", "uk (stdev)"]
# fig, _ = graph.plot_timeseries_multi_sub2([tss4, tss5, tss6, tss7], head, "samples [x0.1s]", axhead)
# quit()
fig, _ = graph.plot_timeseries_multi_sub2(
[tss1, tss2, tss3], ["valve sequence", "sensor output", "pump output"],
"samples [x0.1s]", ["model", "flow [L/h]", "pump [%]"], (16, 16))
# fig, _ = graph.plot_timeseries_multi_sub2([tss1, tss2, tss3, tss4, tss5], [
# "valve sequence", "sensor output", "pump output", "mean", "stdev"], "samples [x0.1s]", ["model", "flow [L/h]", "pump [%]", "mean [L/h]", "stdev"])
graph.save_figure(fig, "./figs/control" + filename)
# x = remove_outliers(x)
# tss = create_timeseries(x, xheader)
# fig, _ = graph.plot_timeseries_multi(tss, "sensor output", "samples [x0.1s]", "flow [L/h]", False)
# graph.save_figure(fig, "./figs/sensor_output")
# graph.plot_timeseries(ts, "title", "x", "y")
# quit()
def run_clustering(x, times, xheader):
# for each node, split the data into days (2400 samples) => 2D
# for each node get 2 clusters
# then combine all clusters to get 2 final clusters
# print([t.timestamp() for t in times[:10]])
# quit()
# print(xheader)
# seconds, time based
# t_day = 240
t_day = 480
# n_day = 2400
# n_day = 4800
n_day = 9600
# n_skip = 200
# n_skip = 400
n_skip = 1
sx = np.shape(x)
sd = int(math.ceil(sx[0] / n_day))
print(sd)
n3 = int(n_day / n_skip)
xd = np.zeros((sx[1], sd + 1, n3))
xheader1 = ["day " + str(d) for d in range(sd + 1)]
print(np.shape(xd))
print(sx)
xlabels = []
day = 0
for k in range(sx[1]):
# for each sample
t0 = times[0].timestamp()
sample = 0
day = 0
xk = x[:, k]
xk = normalize(xk, False)
for j in range(sx[0]):
try:
xd[k][day][sample] = xk[j]
except:
print("exception at [" + str(k) + "," + str(day) + "," +
str(sample) + "]")
sample += 1
# check days
t1 = times[j].timestamp()
if t1 - t0 >= t_day:
t0 = t1
sample = 0
day += 1
print(np.shape(xd[0]))
print(xd[0])
for row in xd[0][1:]:
print(row)
# quit()
ncons = sx[1]
# ncons = 1
cons = range(ncons)
# cons = [2, 3, 4, 7, 8, 9]
cons = [4, 5, 7]
trim = False
trim = True
# for each node
plot_each = False
# plot_each = True
xc_vect = []
for k in cons:
print(k)
# plot daily demands
if trim:
xt = np.transpose(xd[k][1:-1])
else:
xt = np.transpose(xd[k])
title = "consumer #" + str(k + 1) + " patterns"
if plot_each:
tss = create_timeseries(xt, xheader1)
fig, _ = graph.plot_timeseries_multi_sub2([tss], [title],
"samples [x0.1s]",
["flow [L/h]"], None)
xt = np.transpose(xt)
nc = 2
X, kmeans, _, _ = clustering.clustering_kmeans(xt, nc)
xc = np.transpose(kmeans.cluster_centers_)
# print(xc)
xheader2 = [str(e + 1) for e in range(nc)]
# print(xheader2)
# x = x[:10000]
# xc = remove_outliers(xc)
xc_vect.append(xc)
# x = [list(x[:,0])]
# x = np.transpose([x[:,0]])
# xheader = [xheader[0]]
# # print(x)
# print(xheader)
if plot_each:
tss = create_timeseries(xc, xheader2)
fig, _ = graph.plot_timeseries_multi_sub2([tss], [title],
"samples [x0.1s]",
["flow [L/h]"], None)
xc_vect = np.array(xc_vect)
xcs = np.shape(xc_vect)
xc_vect = xc_vect.reshape(xcs[0] * xcs[2], xcs[1])
print(xc_vect)
print(np.shape(xc_vect))
# xc_vect = np.transpose(xc_vect)
# quit()
# nc = 12
nc = 2
X, kmeans, _, _ = clustering.clustering_kmeans(xc_vect, nc)
xc = np.transpose(kmeans.cluster_centers_)
print(xc)
xheader2 = [str(e + 1) for e in range(nc)]
hours = np.linspace(0, t_day / 2, (np.shape(xc))[0])
xlabels = [[e for e in hours] for i in range(nc)]
xlabels = np.array(xlabels)
xlabels = np.transpose(xlabels)
print(xlabels)
# quit()
title = "consumer patterns"
tss = create_timeseries(xc, xheader2, xlabels)
fig, _ = graph.plot_timeseries_multi_sub2([tss], [title],
"day [240s => 24h]",
["flow [0-1]"], None)
graph.save_figure(fig, "./figs/consumer_patterns_all_2")
tss_valves = create_timeseries(data_valves_b, header_valves)
tss_flow = create_timeseries(data_flow_b, header_flow)
tss_pump = create_timeseries(data_pump_b, header_pump)
if xy:
data_flow_b = data_flow_b[:-50]
data_pump_b = data_pump_b[:-50]
tss_xy = create_timeseries(data_flow_b, header_pump, data_pump_b)
fig, _ = graph.plot_timeseries_multi_sub2([tss_xy], ["sensor output", "pump output"], "samples [x0.1s]", ["flow [L/h]", "pump [%]"], (16,16), not save, i)
else:
if len(bookmarks) == 2:
fig, _ = graph.plot_timeseries_multi_sub2([tss_valves, tss_flow, tss_pump], [
"valve sequence", "sensor output", "pump output"], "samples [x0.1s]", ["model", "flow [L/h]", "pump [%]"], (16,16), not save, i)
else:
fig, _ = graph.plot_timeseries_multi_sub2([tss_flow, tss_pump], ["sensor output", "pump output"], "samples [x0.1s]", ["flow [L/h]", "pump [%]"], (16,16), not save, i)
app1 = ""
if xy:
app1 = "_xy"
if save:
if len(bookmarks) == 2:
graph.save_figure(fig, "./figs/ident_" + title + "_" + filename + app1 + "_full")
loader.save_csv(data_flow_b, "./data/output/ident_flow_" + title + "_" + filename + app1 + "_full.csv")
loader.save_csv(data_pump_b, "./data/output/ident_pump_" + title + "_" + filename + app1 + "_full.csv")
else:
if i%2 == 1 or not skip2:
graph.save_figure(fig, "./figs/ident_" + title + "_" + filename + app1 + "_" + str(i))
loader.save_csv(data_flow_b, "./data/output/ident_flow_" + title + "_" + filename + app1 + "_" + str(i) + ".csv")
loader.save_csv(data_pump_b, "./data/output/ident_pump_" + title + "_" + filename + app1 + "_" + str(i) + ".csv")
# end_index = 9600
# end_index = 4800 * 5
x = x[start_index:, :]
y = y[start_index:, :]
if end_index is not None:
x = x[:end_index, :]
y = y[:end_index, :]
sx = np.shape(x)
sy = np.shape(y)
print(np.shape(x))
print(np.shape(y))
# plot_data(x, y, xheader, yheader)
x_poly, y_poly_pred, rmse, r2 = fit_model(x, y, 0)
tss_xy = create_timeseries(y, ["data"], x)
plot_tss = [tss_xy[0]]
if add_mean:
tss_xy_poly = create_timeseries(y_poly_pred, ["mean: " + str(int(y_poly_pred[0]))], x)
tss_xy_poly[0].color = "orange"
plot_tss.append(tss_xy_poly[0])
fig, _ = graph.plot_timeseries_multi(i, plot_tss, titles[i], xheader[0], yheader[0], scales[i], (10, 8))
graph.save_figure(fig, titles[i])
ncols += 1
fig, ax = graph.get_n_ax(4, (10, 9), [5, 5, 1, 1])
# fig, ax = graph.get_n_ax(3, None)
qs = [0]
lastq = data1_orig[0]
# print(lastq)
for i, d in enumerate(data1_orig):
if d != lastq:
lastq = d
qs.append(i)
graph.plot_timeseries_ax(tss2[0], "", "", "flow [L/h]", fig, ax[0], qs)
graph.plot_timeseries_ax(tss3[0], "", "", "pump [%]", fig, ax[1], qs)
# tab10
plot_intersection_matrix_ax(elements_models, 0, nrows, ncols, None, fig,
ax[2], False, "viridis_r", ["model "],
[(" ") for i in range(n_bins)], "", "")
plot_intersection_matrix_ax(elements_combined, 1, nrows, ncols, (0, 1),
fig, ax[3], False, "Blues", ["acc % "],
[(" ") for i in range(n_bins)],
"samples [x0.1s]", "")
graph.show_fig(fig)
graph.save_figure(fig, "./figs/control_integration" + filename)
# "valve sequence", "sensor output", "pump output", "mean", "stdev"], "samples [x0.1s]", ["model", "flow [L/h]", "pump [%]", "mean [L/h]", "stdev"], (16,16))
trim_ts(tss1, start_index, stop_index)
trim_ts(tss2, start_index, stop_index)
trim_ts(tss3, start_index, stop_index)
trim_ts(tss4, start_index_bm, stop_index_bm)
trim_ts(tss5, start_index_bm, stop_index_bm)
trim_ts(tss6, start_index_bm, stop_index_bm)
trim_ts(tss7, start_index_bm, stop_index_bm)
head = ["control metrics (" + title + ")", "", "", "", "", "", ""]
axhead = ["valve sequence", "flow [L/h]", "pump [%]"]
fig, _ = graph.plot_timeseries_multi_sub2([tss1, tss2, tss3], head,
"sample [x0.1 s]", axhead,
(16, 16))
graph.save_figure(fig, "./figs/control_" + filename)
if show_extra:
head = [
"control metrics (" + title + " - local averages)", "", "", "", "",
"", ""
]
axhead = [
"valve sequence", "sensor output", "pump output", "flow [L/h]",
"stdev [L/h]", "pump [%]", "stdev [%]"
]
fig, _ = graph.plot_timeseries_multi_sub2(
[tss1, tss2, tss3, tss4, tss5, tss6, tss7], head, "sequence step",
axhead, (16, 16))
else:
head = ["control metrics (" + title + " - local averages)", "", "", ""]
ts: Timeseries = Timeseries()
ts.label = key
ts.color = colors[ck]
ck += 1
if ck >= len(colors):
ck = 0
for (i, val) in enumerate(acc[key]):
ts.x.append(i + 1)
ts.y.append(val)
# print(len(ts.y))
tss.append(ts)
ts = None
return tss
tss = create_timeseries(acc)
print(json.dumps(acc, indent=2))
labels = [key for key in acc]
labels = ["1-N-80%", "1-N-1-80%", "1-N-1-50%", "GRAY-80%"]
fig = graph.plot_barchart(labels, [acc[key][0] * 100 for key in acc], "model",
"accuracy [%]", "Max accuracy (deep learning)",
"blue")
graph.save_figure(fig, "./figs/accuracy_models_comp")
# graph.plot_timeseries_multi(tss, "deep learning", "run", "accuracy", False)
# graph.stem_timeseries_multi(tss, "deep learning", "run", "accuracy", True)
