def plot_intersection_matrix(elements: List[CMapMatrixElement], index, save): # fig = graph.plot_matrix_cmap_plain( # elements, nvalves, nrows, "Valve Sequence", "sample x" + str(rowskip), "valves", xlabels, ylabels) fig = graph.plot_matrix_cmap_plain(elements, nvalves, nrows, "", "", "valves", xlabels, ylabels, None) if save: graph.save_figure(fig, "./figs/valve_sequence_" + str(index))
def plot_data(x, y, xheader, yheader): tss = create_timeseries(y, yheader) tss2 = create_timeseries(x, xheader) # print(json.dumps(acc, indent=2)) # fig, _ = graph.plot_timeseries_multi(tss, "valve sequence", "samples [x0.1s]", "position [%]", False) fig, _ = graph.plot_timeseries_multi_sub2( [tss, tss2], ["valve sequence", "sensor output"], "samples [x0.1s]", ["position [%]", "flow [L/h]"], (9, 16)) graph.save_figure(fig, "./figs/valve_sequence_" + filename)
nrowskip = 0 nrows += 1 for e in elements_buffer: elements.append(e) except: pass # print(elements) # quit() xlabels = [("v" + str(i + 1)) for i in range(nvalves)] ylabels = [(str(int(i * rowskip / 100))) for i in range(nrows)] # xlabels = [] # ylabels = [] # intersection_matrix = np.random.randint(0, 10, size=(max_val, max_val)) intersection_matrix = np.zeros((nvalves, nrows)) # print(intersection_matrix) for e in elements: intersection_matrix[e.i][e.j] = e.val print(intersection_matrix) fig = graph.plot_matrix_cmap_plain(elements, nvalves, nrows, "Valve Sequence", "sample x" + str(rowskip), "valves", xlabels, ylabels, None, None) graph.save_figure(fig, "./figs/valve_sequence")
csvoutput = "" for e in vect: # print(e) csvoutput += e + "," + \ str(vect[e]["avg"]["acc"]) + "," + str(vect[e]["top"]["acc"]) + \ "," + str(vect[e]["avg"]["dt"]) + "," + str(vect[e]["avg"] ["fsize"]) + "," + str(vect[e]["top_file"]) + "\n" return csvoutput csvoutput = extract_csv(acc_train_vect) with open("./data/output/eval_" + output_filename + "_train.csv", "w") as f: f.write(csvoutput) csvoutput = extract_csv(acc_test_vect) with open("./data/output/eval_" + output_filename + "_test.csv", "w") as f: f.write(csvoutput) # print(csvoutput) tss = create_barseries([acc_train_vect, acc_test_vect], ["train", "test"]) fig = graph.plot_barchart_multi( tss, "model", "accuracy", "Average accuracy (Naive Bayes)", ["1-N-80%", "1-N-1-80%", "1-N-1-50%", "GRAY-80%"], [70, 120]) graph.save_figure(fig, "./figs/mean_accuracy_" + output_filename)
def plot_intersection_matrix(elements, index, nrows, ncols, save, scale): fig = graph.plot_matrix_cmap_plain( elements, nrows, ncols, "Sequence evaluation", "sample bins", "", xlabels, ylabels, scale, (16,7)) if save: graph.save_figure(fig, "./figs/valve_sequence_random_prediction")
def plot_intersection_matrix2(elements, index, nrows, ncols, save, scale): fig = graph.plot_matrix_cmap_plain(elements, nrows, ncols, "", "", "", xlabels, ylabels, scale, None) if save: graph.save_figure(fig, "./figs/valve_sequence_" + str(index))
def extract_csv(vect): csvoutput = "" for e in vect: # print(e) csvoutput += e + "," + \ str(vect[e]["avg"]) + "," + str(vect[e]["top"]) + \ "," + str(vect[e]["top_file"]) + "\n" return csvoutput csvoutput = extract_csv(acc_train_vect) with open("./data/output/eval_dtree_1_train.csv", "w") as f: f.write(csvoutput) csvoutput = extract_csv(acc_test_vect) with open("./data/output/eval_dtree_1_test.csv", "w") as f: f.write(csvoutput) # print(csvoutput) tss = create_barseries([acc_train_vect, acc_test_vect], ["train", "test"]) fig = graph.plot_barchart_multi( tss, "model", "accuracy", "Average accuracy (decision tree)", ["1-N-80%", "1-N-1-80%", "1-N-1-50%", "GRAY-80%"], [70, 120]) graph.save_figure(fig, "./figs/mean_accuracy_dtree_1")
print(np.shape(x)) print(np.shape(y)) # tss = create_timeseries(np.concatenate((x,y), axis=1), np.concatenate((xheader,yheader))) tss = create_timeseries(y, yheader) x = remove_outliers(x) tss2 = create_timeseries(x, xheader) # print(json.dumps(acc, indent=2)) # fig, _ = graph.plot_timeseries_multi(tss, "valve sequence", "samples [x0.1s]", "position [%]", False) fig, _ = graph.plot_timeseries_multi_sub2( [tss, tss2], ["valve sequence", "sensor output"], "samples [x0.1s]", ["position [%]", "flow [L/h]"]) graph.save_figure(fig, "./figs/valve_sequence_" + filename) # x = remove_outliers(x) # tss = create_timeseries(x, xheader) # fig, _ = graph.plot_timeseries_multi(tss, "sensor output", "samples [x0.1s]", "flow [L/h]", False) # graph.save_figure(fig, "./figs/sensor_output") # graph.plot_timeseries(ts, "title", "x", "y") # quit()
res2 = [[e[1], gstdev_pump] for e in res] tss7 = create_timeseries(res2, header_aux) print("shape:") print(np.shape(data2)) print(np.shape(res1)) # head = ["", "", "", ""] # axhead = ["yk (avg)", "yk (stdev)", "uk (avg)", "uk (stdev)"] # fig, _ = graph.plot_timeseries_multi_sub2([tss4, tss5, tss6, tss7], head, "samples [x0.1s]", axhead) # quit() fig, _ = graph.plot_timeseries_multi_sub2( [tss1, tss2, tss3], ["valve sequence", "sensor output", "pump output"], "samples [x0.1s]", ["model", "flow [L/h]", "pump [%]"], (16, 16)) # fig, _ = graph.plot_timeseries_multi_sub2([tss1, tss2, tss3, tss4, tss5], [ # "valve sequence", "sensor output", "pump output", "mean", "stdev"], "samples [x0.1s]", ["model", "flow [L/h]", "pump [%]", "mean [L/h]", "stdev"]) graph.save_figure(fig, "./figs/control" + filename) # x = remove_outliers(x) # tss = create_timeseries(x, xheader) # fig, _ = graph.plot_timeseries_multi(tss, "sensor output", "samples [x0.1s]", "flow [L/h]", False) # graph.save_figure(fig, "./figs/sensor_output") # graph.plot_timeseries(ts, "title", "x", "y") # quit()
def run_clustering(x, times, xheader): # for each node, split the data into days (2400 samples) => 2D # for each node get 2 clusters # then combine all clusters to get 2 final clusters # print([t.timestamp() for t in times[:10]]) # quit() # print(xheader) # seconds, time based # t_day = 240 t_day = 480 # n_day = 2400 # n_day = 4800 n_day = 9600 # n_skip = 200 # n_skip = 400 n_skip = 1 sx = np.shape(x) sd = int(math.ceil(sx[0] / n_day)) print(sd) n3 = int(n_day / n_skip) xd = np.zeros((sx[1], sd + 1, n3)) xheader1 = ["day " + str(d) for d in range(sd + 1)] print(np.shape(xd)) print(sx) xlabels = [] day = 0 for k in range(sx[1]): # for each sample t0 = times[0].timestamp() sample = 0 day = 0 xk = x[:, k] xk = normalize(xk, False) for j in range(sx[0]): try: xd[k][day][sample] = xk[j] except: print("exception at [" + str(k) + "," + str(day) + "," + str(sample) + "]") sample += 1 # check days t1 = times[j].timestamp() if t1 - t0 >= t_day: t0 = t1 sample = 0 day += 1 print(np.shape(xd[0])) print(xd[0]) for row in xd[0][1:]: print(row) # quit() ncons = sx[1] # ncons = 1 cons = range(ncons) # cons = [2, 3, 4, 7, 8, 9] cons = [4, 5, 7] trim = False trim = True # for each node plot_each = False # plot_each = True xc_vect = [] for k in cons: print(k) # plot daily demands if trim: xt = np.transpose(xd[k][1:-1]) else: xt = np.transpose(xd[k]) title = "consumer #" + str(k + 1) + " patterns" if plot_each: tss = create_timeseries(xt, xheader1) fig, _ = graph.plot_timeseries_multi_sub2([tss], [title], "samples [x0.1s]", ["flow [L/h]"], None) xt = np.transpose(xt) nc = 2 X, kmeans, _, _ = clustering.clustering_kmeans(xt, nc) xc = np.transpose(kmeans.cluster_centers_) # print(xc) xheader2 = [str(e + 1) for e in range(nc)] # print(xheader2) # x = x[:10000] # xc = remove_outliers(xc) xc_vect.append(xc) # x = [list(x[:,0])] # x = np.transpose([x[:,0]]) # xheader = [xheader[0]] # # print(x) # print(xheader) if plot_each: tss = create_timeseries(xc, xheader2) fig, _ = graph.plot_timeseries_multi_sub2([tss], [title], "samples [x0.1s]", ["flow [L/h]"], None) xc_vect = np.array(xc_vect) xcs = np.shape(xc_vect) xc_vect = xc_vect.reshape(xcs[0] * xcs[2], xcs[1]) print(xc_vect) print(np.shape(xc_vect)) # xc_vect = np.transpose(xc_vect) # quit() # nc = 12 nc = 2 X, kmeans, _, _ = clustering.clustering_kmeans(xc_vect, nc) xc = np.transpose(kmeans.cluster_centers_) print(xc) xheader2 = [str(e + 1) for e in range(nc)] hours = np.linspace(0, t_day / 2, (np.shape(xc))[0]) xlabels = [[e for e in hours] for i in range(nc)] xlabels = np.array(xlabels) xlabels = np.transpose(xlabels) print(xlabels) # quit() title = "consumer patterns" tss = create_timeseries(xc, xheader2, xlabels) fig, _ = graph.plot_timeseries_multi_sub2([tss], [title], "day [240s => 24h]", ["flow [0-1]"], None) graph.save_figure(fig, "./figs/consumer_patterns_all_2")
tss_valves = create_timeseries(data_valves_b, header_valves) tss_flow = create_timeseries(data_flow_b, header_flow) tss_pump = create_timeseries(data_pump_b, header_pump) if xy: data_flow_b = data_flow_b[:-50] data_pump_b = data_pump_b[:-50] tss_xy = create_timeseries(data_flow_b, header_pump, data_pump_b) fig, _ = graph.plot_timeseries_multi_sub2([tss_xy], ["sensor output", "pump output"], "samples [x0.1s]", ["flow [L/h]", "pump [%]"], (16,16), not save, i) else: if len(bookmarks) == 2: fig, _ = graph.plot_timeseries_multi_sub2([tss_valves, tss_flow, tss_pump], [ "valve sequence", "sensor output", "pump output"], "samples [x0.1s]", ["model", "flow [L/h]", "pump [%]"], (16,16), not save, i) else: fig, _ = graph.plot_timeseries_multi_sub2([tss_flow, tss_pump], ["sensor output", "pump output"], "samples [x0.1s]", ["flow [L/h]", "pump [%]"], (16,16), not save, i) app1 = "" if xy: app1 = "_xy" if save: if len(bookmarks) == 2: graph.save_figure(fig, "./figs/ident_" + title + "_" + filename + app1 + "_full") loader.save_csv(data_flow_b, "./data/output/ident_flow_" + title + "_" + filename + app1 + "_full.csv") loader.save_csv(data_pump_b, "./data/output/ident_pump_" + title + "_" + filename + app1 + "_full.csv") else: if i%2 == 1 or not skip2: graph.save_figure(fig, "./figs/ident_" + title + "_" + filename + app1 + "_" + str(i)) loader.save_csv(data_flow_b, "./data/output/ident_flow_" + title + "_" + filename + app1 + "_" + str(i) + ".csv") loader.save_csv(data_pump_b, "./data/output/ident_pump_" + title + "_" + filename + app1 + "_" + str(i) + ".csv")
# end_index = 9600 # end_index = 4800 * 5 x = x[start_index:, :] y = y[start_index:, :] if end_index is not None: x = x[:end_index, :] y = y[:end_index, :] sx = np.shape(x) sy = np.shape(y) print(np.shape(x)) print(np.shape(y)) # plot_data(x, y, xheader, yheader) x_poly, y_poly_pred, rmse, r2 = fit_model(x, y, 0) tss_xy = create_timeseries(y, ["data"], x) plot_tss = [tss_xy[0]] if add_mean: tss_xy_poly = create_timeseries(y_poly_pred, ["mean: " + str(int(y_poly_pred[0]))], x) tss_xy_poly[0].color = "orange" plot_tss.append(tss_xy_poly[0]) fig, _ = graph.plot_timeseries_multi(i, plot_tss, titles[i], xheader[0], yheader[0], scales[i], (10, 8)) graph.save_figure(fig, titles[i])
ncols += 1 fig, ax = graph.get_n_ax(4, (10, 9), [5, 5, 1, 1]) # fig, ax = graph.get_n_ax(3, None) qs = [0] lastq = data1_orig[0] # print(lastq) for i, d in enumerate(data1_orig): if d != lastq: lastq = d qs.append(i) graph.plot_timeseries_ax(tss2[0], "", "", "flow [L/h]", fig, ax[0], qs) graph.plot_timeseries_ax(tss3[0], "", "", "pump [%]", fig, ax[1], qs) # tab10 plot_intersection_matrix_ax(elements_models, 0, nrows, ncols, None, fig, ax[2], False, "viridis_r", ["model "], [(" ") for i in range(n_bins)], "", "") plot_intersection_matrix_ax(elements_combined, 1, nrows, ncols, (0, 1), fig, ax[3], False, "Blues", ["acc % "], [(" ") for i in range(n_bins)], "samples [x0.1s]", "") graph.show_fig(fig) graph.save_figure(fig, "./figs/control_integration" + filename)
# "valve sequence", "sensor output", "pump output", "mean", "stdev"], "samples [x0.1s]", ["model", "flow [L/h]", "pump [%]", "mean [L/h]", "stdev"], (16,16)) trim_ts(tss1, start_index, stop_index) trim_ts(tss2, start_index, stop_index) trim_ts(tss3, start_index, stop_index) trim_ts(tss4, start_index_bm, stop_index_bm) trim_ts(tss5, start_index_bm, stop_index_bm) trim_ts(tss6, start_index_bm, stop_index_bm) trim_ts(tss7, start_index_bm, stop_index_bm) head = ["control metrics (" + title + ")", "", "", "", "", "", ""] axhead = ["valve sequence", "flow [L/h]", "pump [%]"] fig, _ = graph.plot_timeseries_multi_sub2([tss1, tss2, tss3], head, "sample [x0.1 s]", axhead, (16, 16)) graph.save_figure(fig, "./figs/control_" + filename) if show_extra: head = [ "control metrics (" + title + " - local averages)", "", "", "", "", "", "" ] axhead = [ "valve sequence", "sensor output", "pump output", "flow [L/h]", "stdev [L/h]", "pump [%]", "stdev [%]" ] fig, _ = graph.plot_timeseries_multi_sub2( [tss1, tss2, tss3, tss4, tss5, tss6, tss7], head, "sequence step", axhead, (16, 16)) else: head = ["control metrics (" + title + " - local averages)", "", "", ""]
ts: Timeseries = Timeseries() ts.label = key ts.color = colors[ck] ck += 1 if ck >= len(colors): ck = 0 for (i, val) in enumerate(acc[key]): ts.x.append(i + 1) ts.y.append(val) # print(len(ts.y)) tss.append(ts) ts = None return tss tss = create_timeseries(acc) print(json.dumps(acc, indent=2)) labels = [key for key in acc] labels = ["1-N-80%", "1-N-1-80%", "1-N-1-50%", "GRAY-80%"] fig = graph.plot_barchart(labels, [acc[key][0] * 100 for key in acc], "model", "accuracy [%]", "Max accuracy (deep learning)", "blue") graph.save_figure(fig, "./figs/accuracy_models_comp") # graph.plot_timeseries_multi(tss, "deep learning", "run", "accuracy", False) # graph.stem_timeseries_multi(tss, "deep learning", "run", "accuracy", True)