def test(): name = "test" network.clear(name) params = get_params(name=name, FX_sel='basic', location='hips', dset_L='train', dset_U='validation', dset_V='validation', ratio_L=1, ratio_U=1, ratio_V=1, prediction=True, pretrain='final', runs=1, epochs=10, save_step=2, oversampling=True, G_label_sample=True, G_label_factor=1, C_basic_train=True, R_active=True, G_no=1, D_no=1, C_no=1, log_name='log') train(params=params) GAN.get_prediction_accuracy(params)
def run(protocol, csvfile): for path in sorted(glob.glob(f'../../data/grid4/*.json')): state = tools.load_json(path) (node_count, link_count) = tools.json_count(state) print(f'run {protocol} on {path}') network.apply(state=state, link_command=get_tc_command, remotes=remotes) tools.sleep(10) software_start_ms = tools.millis() software.start(protocol, remotes) software_startup_ms = tools.millis() - software_start_ms tools.sleep(30) paths = tools.get_random_paths(state, 2 * link_count) paths = tools.filter_paths(state, paths, min_hops=2, path_count=link_count) ping_result = tools.ping_paths(remotes=remotes, paths=paths, duration_ms=30000, verbosity='verbose') sysload_result = tools.sysload(remotes) software.clear(remotes) # add data to csv file extra = (['node_count', 'software_startup_ms'], [node_count, software_startup_ms]) tools.csv_update(csvfile, '\t', extra, ping_result.getData(), sysload_result) network.clear(remotes) # abort benchmark when less then 40% of the pings arrive if (ping_result.received / ping_result.transmitted) < 0.4: break
def run(protocol, csvfile, step_duration, step_distance): shared.seed_random(42) node_count = 50 state = topology.create_nodes(node_count) mobility.randomize_positions(state, xy_range=1000) mobility.connect_range(state, max_links=150) # create network and start routing software network.apply(state, link_command=get_tc_command, remotes=remotes) software.start(protocol) test_beg_ms = shared.millis() for n in range(0, 30): print(f'{protocol}: iteration {n}') #with open(f'graph-{step_duration}-{step_distance}-{n:03d}.json', 'w+') as file: # json.dump(state, file, indent=' ') # connect nodes range wait_beg_ms = shared.millis() # update network representation mobility.move_random(state, distance=step_distance) mobility.connect_range(state, max_links=150) # update network tmp_ms = shared.millis() network.apply(state=state, link_command=get_tc_command, remotes=remotes) #software.apply(protocol=protocol, state=state) # we do not change the node count network_ms = shared.millis() - tmp_ms # Wait until wait seconds are over, else error shared.wait(wait_beg_ms, step_duration) paths = ping.get_random_paths(state, 2 * 400) paths = ping.filter_paths(state, paths, min_hops=2, path_count=200) ping_result = ping.ping(paths=paths, duration_ms=2000, verbosity='verbose', remotes=remotes) # add data to csv file extra = (['node_count', 'time_ms'], [node_count, shared.millis() - test_beg_ms]) shared.csv_update(csvfile, '\t', extra, ping_result.getData()) software.clear(remotes) network.clear(remotes)
def run(protocol, csvfile): tools.seed_random(23) node_count = 50 state = topology.create_nodes(node_count) mobility.randomize_positions(state, xy_range=1000) mobility.connect_range(state, max_links=150) # create network and start routing software network.apply(state=state, link_command=get_tc_command) software.start(protocol) tools.sleep(30) for step_distance in [50, 100, 150, 200, 250, 300, 350, 400]: print(f'{protocol}: step_distance {step_distance}') traffic_beg = tools.traffic() for n in range(0, 6): #with open(f'graph-{step_distance}-{n}.json', 'w+') as file: # json.dump(state, file, indent=' ') # connect nodes range wait_beg_ms = tools.millis() # update network representation mobility.move_random(state, distance=step_distance) mobility.connect_range(state, max_links=150) # update network network.apply(state=state, link_command=get_tc_command) # Wait until wait seconds are over, else error tools.wait(wait_beg_ms, 10) paths = tools.get_random_paths(state, 2 * 200) paths = tools.filter_paths(state, paths, min_hops=2, path_count=200) ping_result = tools.ping_paths(paths=paths, duration_ms=2000, verbosity='verbose') packets_arrived_pc = 100 * (ping_result.received / ping_result.send) traffic_end = tools.traffic() # add data to csv file extra = (['node_count', 'time_ms', 'step_distance_m', 'n', 'packets_arrived_pc'], [node_count, tools.millis() - wait_beg_ms, step_distance, n, packets_arrived_pc]) tools.csv_update(csvfile, '\t', extra, (traffic_end - traffic_beg).getData(), ping_result.getData()) traffic_beg = traffic_end software.clear() network.clear()
def run(protocol, files, csvfile): tools.seed_random(1234) for path in sorted(glob.glob(files)): state = tools.load_json(path) (node_count, link_count) = tools.json_count(state) print(f'run {protocol} on {path}') network.apply(state=state, link_command=get_tc_command) tools.sleep(10) for offset in range(0, 60, 2): tmp_ms = tools.millis() traffic_beg = tools.traffic() traffic_ms = tools.millis() - tmp_ms tmp_ms = tools.millis() software.start(protocol) software_ms = tools.millis() - tmp_ms # Wait until wait seconds are over, else error tools.sleep(offset) paths = tools.get_random_paths(state, 2 * 200) paths = tools.filter_paths(state, paths, min_hops=2, path_count=200) ping_result = tools.ping_paths(paths=paths, duration_ms=2000, verbosity='verbose') traffic_end = tools.traffic() sysload_result = tools.sysload() software.clear() # add data to csv file extra = (['node_count', 'traffic_ms', 'software_ms', 'offset_ms'], [node_count, traffic_ms, software_ms, offset * 1000]) tools.csv_update(csvfile, '\t', extra, (traffic_end - traffic_beg).getData(), ping_result.getData(), sysload_result) network.clear()
def run(protocol, files, csvfile): for path in sorted(glob.glob(files)): state = shared.load_json(path) (node_count, link_count) = shared.json_count(state) # Limit node count to 300 if node_count > 300: continue print(f'run {protocol} on {path}') network.apply(state=state, link_command=get_tc_command, remotes=remotes) shared.sleep(10) software_start_ms = shared.millis() software.start(protocol, remotes) software_startup_ms = shared.millis() - software_start_ms shared.sleep(300) start_ms = shared.millis() traffic_beg = traffic.traffic(remotes) paths = ping.get_random_paths(state, 2 * 200) paths = ping.filter_paths(state, paths, min_hops=2, path_count=200) ping_result = ping.ping(remotes=remotes, paths=paths, duration_ms=300000, verbosity='verbose') traffic_ms = shared.millis() - start_ms traffic_end = traffic.traffic(remotes) sysload_result = shared.sysload(remotes) software.clear(remotes) network.clear(remotes) # add data to csv file extra = (['node_count', 'traffic_ms', 'software_startup_ms'], [node_count, traffic_ms, software_startup_ms]) shared.csv_update(csvfile, '\t', extra, (traffic_end - traffic_beg).getData(), ping_result.getData(), sysload_result)
def run(protocol, csvfile): shared.seed_random(1377) for path in sorted(glob.glob(f'../../data/freifunk/*.json')): state = shared.load_json(path) (node_count, link_count) = shared.json_count(state) dataset_name = '{}-{:04d}'.format(os.path.basename(path)[9:-5], node_count) # limit to what the host can handle if node_count > 310: continue print(f'run {protocol} on {path}') state = network.apply(state=state, link_command=get_tc_command, remotes=remotes) shared.sleep(10) software.start(protocol, remotes) shared.sleep(300) start_ms = shared.millis() traffic_beg = traffic.traffic(remotes) paths = ping.get_random_paths(state, 2 * node_count) paths = shared.filter_paths(state, paths, min_hops=2, path_count=node_count) ping_result = shared.ping(remotes=remotes, paths=paths, duration_ms=300000, verbosity='verbose') sysload_result = shared.sysload(remotes) traffic_ms = shared.millis() - start_ms traffic_end = traffic.traffic(remotes) software.clear(remotes) # add data to csv file extra = (['dataset_name', 'node_count', 'traffic_ms'], [dataset_name, node_count, traffic_ms]) shared.csv_update(csvfile, '\t', extra, (traffic_end - traffic_beg).getData(), ping_result.getData(), sysload_result) network.clear(remotes)
def run(protocol, tasks, csvfile): for path, gateways in tasks: state = shared.load_json(path) (node_count, link_count) = shared.json_count(state) # Limit node count to 300 if node_count > 300: continue print(f'run {protocol} on {path}') network.apply(state=state, remotes=remotes) shared.sleep(10) software_start_ms = shared.millis() software.start(protocol, remotes) software_startup_ms = shared.millis() - software_start_ms shared.sleep(30) start_ms = shared.millis() traffic_beg = traffic.traffic(remotes) paths = ping.get_paths_to_gateways(state, gateways) ping_result = ping.ping(remotes=remotes, paths=paths, duration_ms=300000, verbosity='verbose') traffic_ms = shared.millis() - start_ms traffic_end = traffic.traffic(remotes) sysload_result = shared.sysload(remotes) software.clear(remotes) network.clear(remotes) # add data to csv file extra = (['node_count', 'traffic_ms', 'software_startup_ms'], [node_count, traffic_ms, software_startup_ms]) shared.csv_update(csvfile, '\t', extra, (traffic_end - traffic_beg).getData(), ping_result.getData(), sysload_result)
#!/usr/bin/env python3 import os import sys import glob sys.path.append('../../') import software import network import tools software.clear() network.clear() prefix = os.environ.get('PREFIX', '') # 100MBit LAN cable def get_tc_command(link, ifname): return f'tc qdisc replace dev "{ifname}" root tbf rate 100mbit burst 8192 latency 1ms' def run(protocol, files, csvfile): tools.seed_random(1234) for path in sorted(glob.glob(files)): state = tools.load_json(path) (node_count, link_count) = tools.json_count(state) print(f'run {protocol} on {path}')
import os import sys import glob sys.path.append('../../') from shared import Remote import shared import ping import software import network remotes = [Remote()] #[Remote('192.168.44.133'), Remote('192.168.44.137')] shared.check_access(remotes) software.clear(remotes) network.clear(remotes) prefix = os.environ.get('PREFIX', '') # 100MBit LAN cable def get_tc_command(link, ifname): return f'tc qdisc replace dev "{ifname}" root tbf rate 100mbit burst 8192 latency 1ms' def run(protocol, csvfile): for path in sorted(glob.glob(f'../../data/grid4/*.json')): state = shared.load_json(path) (node_count, link_count) = shared.json_count(state) print(f'run {protocol} on {path}')
def train_GAN(params): # ------------------- # Parameters # ------------------- log(str(params), name=params['log_name']) # Clear remaining model if params['ratio_L'] < 1.0 or params['ratio_U'] < 1.0: network.clear(params['name'] + '_R' + str(params['start_run'])) plt.close('all') # ------------------- # CUDA # ------------------- cuda = True if torch.cuda.is_available() else False G_Loss = torch.nn.BCELoss() D_Loss = torch.nn.BCELoss() C_Loss = torch.nn.BCELoss() if cuda: G_Loss.cuda() D_Loss.cuda() C_Loss.cuda() floatTensor = torch.cuda.FloatTensor log("CUDA Training.", name=params['log_name']) network.clear_cache() else: floatTensor = torch.FloatTensor log("CPU Training.", name=params['log_name']) # ------------------- # Data scaling # ------------------- ''' XTL ... Original labelled data XTU ... Original unlabelled data XTV ... Original validation data XL ... Labelled data XU ... Unlabelled data XV ... Validation data ''' dset_L = params['dset_L'] dset_U = params['dset_U'] dset_V = params['dset_V'] if dset_L == dset_U: X, Y = pp.get_data(params, dset_L) XTL, XTU, YTL, YTU = pp.split_data(X, Y) else: XTL, YTL = pp.get_data(params, dset_L) XTU, YTU = pp.get_data(params, dset_U) if dset_V is None: XTV, YTV = XTU, YTU else: XTV, YTV = pp.get_data(params, dset_V) XTL = pp.scale_minmax(XTL) XTU = pp.scale_minmax(XTU) XTV = pp.scale_minmax(XTV) if params['ratio_V'] < 1.0: XTV, YTV = pp.select_random(XTV, YTV, params['ratio_L']) log("Selected %s of validation samples." % (format(params['ratio_V'], '0.2f')), name=params['log_name']) DL_V = pp.get_dataloader(params, XTV, YTV, batch_size=1024) # ------------------- # Load accuracy # ------------------- mat_accuracy_G, mat_accuracy_D, mat_accuracy_C = network.load_Acc(params) if (params['R_active']): mat_accuracy_R = network.load_R_Acc(params) # ------------------- # Final prediction # ------------------- if (params['prediction']): Y_pred = torch.zeros(XTU.shape[0], 8) # ------------------- # Start Training # ------------------- YF = None PF = None RF = None for run in range(params['runs']): # ------------------- # Labelled Data # ------------------- XL, YL = XTL, YTL if params['ratio_L'] < 1.0: XL, YL = pp.select_random(XL, YL, params['ratio_L']) log("Selected %s of labelled samples." % (format(params['ratio_L'], '0.2f')), name=params['log_name']) count_L = YL.shape[0] log("Number of labelled samples = %d." % (count_L), name=params['log_name']) DL_L = pp.get_dataloader(params, XL, YL) # ------------------- # Unlabelled Data # ------------------- XU, YU = XTU, YTU if params['ratio_U'] < 1.0: XU, YU = pp.select_random(XU, YU, params['ratio_U']) log("Selected %s of unlabelled samples." % (format(params['ratio_U'], '0.2f')), name=params['log_name']) log("Number of unlabelled samples = %d." % (XU.shape[0]), name=params['log_name']) DL_U_iter = pp.get_perm_dataloader(params, XU, YU) # ------------------- # Networks # ------------------- G, D, C = network.load_GAN(run, params) if (params['R_active']): R = network.load_Ref(run, params) # ------------------- # Optimizers # ------------------- optimizer_G = torch.optim.Adam(G.parameters(), lr=params['GLR'], betas=(params['GB1'], params['GB2'])) optimizer_D = torch.optim.Adam(D.parameters(), lr=params['DLR'], betas=(params['DB1'], params['DB2'])) optimizer_C = torch.optim.Adam(C.parameters(), lr=params['CLR'], betas=(params['CB1'], params['CB2'])) if (params['R_active']): optimizer_R = torch.optim.Adam(R.parameters(), lr=params['CLR'], betas=(params['CB1'], params['CB2'])) # ------------------- # Training # ------------------- if run >= params['start_run']: if params['oversampling']: XL, YL = pp.over_sampling(params, XL, YL) log("Oversampling: created %d new labelled samples." % (XL.shape[0] - count_L), name=params['log_name']) for epoch in range(params['epochs']): # Jump to start epoch if run == params['start_run']: if epoch < params['start_epoch']: continue running_loss_G = 0.0 running_loss_D = 0.0 running_loss_C = 0.0 """ X1, P1 - Labelled Data, predicted Labels (C) | Regular training of classifier W1 = (X1, Y1), A1 - Labelled Data, actual Labels, predicted Authenticity (D) | Real samples W2 = (X2, Y2), A2 - Unlabelled Data, predicted Labels (C), predicted Authenticity (D) | Real data with fake labels W3 = (X3, Y3), A3 - Synthetic Data (G), actual Labels, predicted Authenticity (D) | Fake data with real labels W4 = (X4, Y4), A4 - Unlabbeled Data, predicted Labels (C), predicted Authenticity (D) | Fake positive to prevent overfitting XV, YV, PV - Validation Data, actual Labels, predicted Labels (C) | Validation samples R1, F2, F3, R4 - Real/Fake Labels """ for i, data in enumerate(DL_L, 1): loss_G = [] loss_D = [] loss_C = [] # ------------------- # Train the classifier on real samples # ------------------- X1, Y1 = data W1 = torch.cat((X1, Y1), dim=1) R1 = floatTensor(W1.shape[0], 1).fill_(1.0) if params['C_basic_train']: optimizer_C.zero_grad() P1 = C(X1) loss = C_Loss(P1, Y1) loss_C.append(loss) loss.backward() optimizer_C.step() if params['R_active']: optimizer_R.zero_grad() PR = R(X1) loss = C_Loss(PR, Y1) loss.backward() optimizer_R.step() # ------------------- # Train the discriminator to label real samples # ------------------- optimizer_D.zero_grad() A1 = D(W1) loss = D_Loss(A1, R1) loss_D.append(loss) loss.backward() optimizer_D.step() # ------------------- # Classify unlabelled data # ------------------- optimizer_C.zero_grad() X2 = DL_U_iter.get_next()[0] Y2 = C(X2) W2 = torch.cat((X2, Y2), dim=1) # ------------------- # Train the classifier to label unlabelled samples # ------------------- A2 = D(W2) R2 = floatTensor(W2.shape[0], 1).fill_(1.0) loss = C_Loss(A2, R2) loss_C.append(loss) loss.backward() optimizer_C.step() # ------------------- # Train the discriminator to label predicted samples # ------------------- optimizer_D.zero_grad() A2 = D(W2.detach()) F2 = floatTensor(W2.shape[0], 1).fill_(0.0) loss = D_Loss(A2, F2) loss_D.append(loss) loss.backward() optimizer_D.step() # ------------------- # Train the discriminator to label fake positive samples # ------------------- X4 = DL_U_iter.get_next()[0] Y4 = C(X4) W4 = torch.cat((X4, Y4), dim=1) optimizer_D.zero_grad() A4 = D(W4) R4 = floatTensor(W4.shape[0], 1).fill_(1.0) loss = D_Loss(A4, R4) loss_D.append(loss) loss.backward() optimizer_D.step() # ------------------- # Create Synthetic Data # ------------------- optimizer_G.zero_grad() if params['G_label_sample']: # Selected Labels from a uniform distribution of available labels Y3 = floatTensor( pp.get_one_hot_labels(params=params, num=Y1.shape[0] * params['G_label_factor'])) else: # Select labels from current training batch Y3 = torch.cat( ([Y1 for _ in range(params['G_label_factor'])]), dim=0) Z = floatTensor( np.random.normal(0, 1, (Y3.shape[0], params['noise_shape']))) I3 = torch.cat((Z, Y3), dim=1) X3 = G(I3) W3 = torch.cat((X3, Y3), dim=1) # ------------------- # Train the generator to fool the discriminator # ------------------- A3 = D(W3) R3 = floatTensor(W3.shape[0], 1).fill_(1.0) loss = G_Loss(A3, R3) loss_G.append(loss) loss.backward() optimizer_G.step() # ------------------- # Train the discriminator to label synthetic samples # ------------------- optimizer_D.zero_grad() A3 = D(W3.detach()) F3 = floatTensor(W3.shape[0], 1).fill_(0.0) loss = D_Loss(A3, F3) loss_D.append(loss) loss.backward() optimizer_D.step() # ------------------- # Calculate overall loss # ------------------- running_loss_G += np.mean([loss.item() for loss in loss_G]) running_loss_D += np.mean([loss.item() for loss in loss_D]) running_loss_C += np.mean([loss.item() for loss in loss_C]) # ------------------- # Post Epoch # ------------------- logString = "[Run %d/%d] [Epoch %d/%d] [G loss: %f] [D loss: %f] [C loss: %f]" % ( run + 1, params['runs'], epoch + 1, params['epochs'], running_loss_G / (i), running_loss_D / (i), running_loss_C / (i)) log(logString, save=False, name=params['log_name']) if (epoch + 1) % params['save_step'] == 0: idx = run, int(epoch / params['save_step']) + 1 acc_D_real = [] acc_D_vs_C = [] acc_D_vs_G = [] acc_C_real = [] for data in DL_V: XV, YV = data # Predict labels PV = C(XV) if params['R_active']: PR = R(XV) mat_accuracy_R[idx] = get_accuracy(PR, YV) network.save_Ref(params['name'], run, R) network.save_R_Acc(params, mat_accuracy_R) # Generate Synthetic Data Z = floatTensor( np.random.normal( 0, 1, (YV.shape[0], params['noise_shape']))) IV = torch.cat((Z, YV), dim=1) XG = G(IV) # Estimate Discriminator Accuracy WV1 = torch.cat((XV, YV), dim=1) WV2 = torch.cat((XV, PV), dim=1) WV3 = torch.cat((XG, YV), dim=1) RV1 = floatTensor(WV1.shape[0], 1).fill_(1.0) FV2 = floatTensor(WV2.shape[0], 1).fill_(0.0) FV3 = floatTensor(WV3.shape[0], 1).fill_(0.0) AV1 = D(WV1) AV2 = D(WV2) AV3 = D(WV3) acc_D_real.append(get_accuracy_binary(AV1, RV1)) acc_D_vs_C.append(get_accuracy_binary(AV2, FV2)) acc_D_vs_G.append(get_accuracy_binary(AV3, FV3)) acc_C_real.append(get_accuracy(PV, YV)) acc_D_real = np.mean(acc_D_real) acc_D_vs_C = np.mean(acc_D_vs_C) acc_D_vs_G = np.mean(acc_D_vs_G) acc_D = .5 * acc_D_real + .25 * acc_D_vs_G + .25 * acc_D_vs_C mat_accuracy_D[idx] = acc_D acc_C_real = np.mean(acc_C_real) acc_C_vs_D = 1.0 - acc_D_vs_C acc_C = .5 * acc_C_real + .5 * acc_C_vs_D mat_accuracy_C[idx] = acc_C_real acc_G = 1.0 - acc_D_vs_G mat_accuracy_G[idx] = acc_G logString = "[Run %d/%d] [Epoch %d/%d] [G acc: %f] [D acc: %f | vs Real: %f | vs G: %f | vs C: %f] [C acc: %f | vs Real: %f | vs D: %f]" % ( run + 1, params['runs'], epoch + 1, params['epochs'], acc_G, acc_D, acc_D_real, acc_D_vs_G, acc_D_vs_C, acc_C, acc_C_real, acc_C_vs_D) log(logString, save=True, name=params['log_name']) network.save_GAN(params['name'], run, G, D, C) params['start_epoch'] = epoch + 1 network.save_Parameter(params) network.save_Acc(params, mat_accuracy_G, mat_accuracy_D, mat_accuracy_C) # End of Training Run params['start_run'] = run + 1 params['start_epoch'] = 0 network.save_Parameter(params) # ------------------- # Post Run # ------------------- acc_C_real = [] for data in DL_V: XV, YV = data # # Generate Synthetic Data # Z = floatTensor(np.random.normal(0, 1, (YV.shape[0], params['noise_shape']))) # IV = torch.cat((Z,YV),dim=1) # XG = G(IV) # Classify Validation data PC = C(XV) acc_C_real.append(get_accuracy(PC, YV)) if params['R_active']: if RF == None: RF = R(XV) else: RF = torch.cat((RF, R(XV).detach()), 0) if YF == None: YF = YV PF = PC else: YF = torch.cat((YF, YV), 0) PF = torch.cat((PF, PC), 0) mat_accuracy_C[run] = np.mean(acc_C_real) # ------------------- # Final prediction # ------------------- if (params['prediction']): C.hard = False XP = pp.get_tensor(XTU, None)[0] YP = C(XP) Y_pred += YP.cpu().detach() C.hard = True # ------------------- # Post Training # ------------------- timeline = np.arange(0, params['epochs'] + 1, params['save_step']) # ------------------- # Plot Accuracy # ------------------- acc_G = np.mean(mat_accuracy_G, axis=0) std_G = np.std(mat_accuracy_G, axis=0) acc_D = np.mean(mat_accuracy_D, axis=0) std_D = np.std(mat_accuracy_D, axis=0) acc_C = np.mean(mat_accuracy_C, axis=0) std_C = np.std(mat_accuracy_C, axis=0) if params['R_active']: acc_R = np.mean(mat_accuracy_R, axis=0) fig, ax = plt.subplots() legend = [] cmap = plt.get_cmap('gnuplot') indices = np.linspace(0, cmap.N, 7) colors = [cmap(int(i)) for i in indices] ax.plot(timeline, acc_C, c=colors[0], linestyle='solid') ax.fill_between(timeline, acc_C - std_C, acc_C + std_C, alpha=0.3, facecolor=colors[0]) legend.append("Accuracy $A_C$") ax.plot(timeline, acc_D, c=colors[1], linestyle='dashed') ax.fill_between(timeline, acc_D - std_D, acc_D + std_D, alpha=0.3, facecolor=colors[1]) legend.append("Accuracy $A_D$") ax.plot(timeline, acc_G, c=colors[2], linestyle='dotted') ax.fill_between(timeline, acc_G - std_G, acc_G + std_G, alpha=0.3, facecolor=colors[2]) legend.append("Accuracy $A_G$") Y_max = 1.15 if params['R_active']: ax.plot(timeline, acc_R, c=colors[3], linestyle='dashdot') legend.append("Accuracy $A_R$") perf = np.zeros_like(acc_C) perf[0] = 0.0 perf[1:] = (acc_C[1:] - acc_R[1:]) / acc_R[1:] ax.plot(timeline, perf + 1, c=colors[4], linestyle='solid') legend.append("Performance $P_C$") ax.set_xlim(0.0, params['epochs']) ax.set_ylim(0.0, Y_max) ax.legend(legend, fontsize=20) ax.set_xlabel('Epoch', fontsize=20) ax.set_ylabel('Accuracy', fontsize=20) ax.grid() save_fig(params, 'eval', fig) # ------------------- # Compare Classifier to Baseline # ------------------- if params['R_active']: maxC = np.argmax(acc_C, axis=0) bestC = acc_C[maxC] maxR = np.argmax(acc_R, axis=0) bestR = acc_R[maxR] log(' - Peak Accuracy: C: %s after %d epochs | R: %s after %d epochs | Inc: %s' % (format((bestC), '0.4f'), timeline[maxC], format( (bestR), '0.4f'), timeline[maxR], format((bestC - bestR) / bestR, '0.4f')), name='results') Y_max = max(Y_max, max(perf + 1) + 0.025) maxP = np.argmax(perf, axis=0) log(' - Hightest $P_C$: %s after %d epochs.' % (format( (perf[maxP]), '0.4f'), timeline[maxP]), name='results') adva = np.zeros_like(acc_C) for i, v1 in enumerate(acc_C): for j, v2 in enumerate(acc_R): if v2 >= v1: adva[i] = j - i break maxA = np.argmax(adva, axis=0) log(' - Biggest Advantage: %d epochs after %d epochs.' % (adva[maxA] * params['save_step'], timeline[maxA]), name='results') # ------------------- # Log Results # ------------------- if params['evaluate']: log(" - %s ( %s | %s ): [C acc: %f ( ± %f )]" % (params['name'], params['dset_V'], params['location'], acc_C[-1], std_C[-1]), name='results') else: log(" - " + params['name'] + ": [C acc: %f ( ± %f )] [D acc: %f ( ± %f )] [G acc: %f ( ± %f )]" % (acc_C[-1], std_C[-1], acc_D[-1], std_D[-1], acc_G[-1], std_G[-1]), name='results') # ------------------- # Generate Confusion Matrix # ------------------- YF = pp.one_hot_to_labels(params, YF) PF = pp.one_hot_to_labels(params, PF) con_mat = confusion_matrix(YF, PF, labels=None, sample_weight=None, normalize='true') if params['evaluate']: plot_confusion_matrix(con_mat, params, name='%s_%s' % (params['dset_V'], params['location']), title='Confusion matrix') else: plot_confusion_matrix(con_mat, params, name='C', title='Confusion matrix') if params['R_active']: RF = pp.one_hot_to_labels(params, RF) con_mat = confusion_matrix(YF, RF, labels=None, sample_weight=None, normalize='true') plot_confusion_matrix(con_mat, params, name='R', title='Confusion matrix') # ------------------- # Final prediction # ------------------- if (params['prediction']): network.make_dir_pre() pred = torch.argmax(Y_pred, axis=1) f = open(network.S_PATH + params['name'] + '_predictions.txt', "w") for y in pred: f.write(' '.join(['%.6f' % (float(y.item() + 1))] * 500) + '\n') f.close()
def reset(): global tasks_encoded tasks_encoded = {} nn.clear()