def __init__(self): self.arguments = Logger() self.alpha = .9 self.num_schedules = 150 self.home_dir = self.arguments.home_dir self.total_loss_array = [] load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/datasets/' + str( self.num_schedules) + 'dist_early_hili_naive.pkl' self.data = pickle.load(open(load_directory, "rb")) self.X, self.Y, self.schedule_array = create_new_dataset(num_schedules=self.num_schedules, data=self.data) for i, each_element in enumerate(self.X): self.X[i] = each_element + list(range(20)) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") self.model = NNwEmbedding().to(device) print(self.model.state_dict()) self.opt = torch.optim.Adam([{'params': list(self.model.parameters())[:-1]}, {'params': self.model.EmbeddingList.parameters(), 'lr': .01}]) self.embedding_optimizer = torch.optim.Adam(self.model.EmbeddingList.parameters(), lr=.1) self.num_iterations_predict_task = 0 self.total_iterations = 0 self.convergence_epsilon = .01 self.when_to_save = 1000 self.distribution_epsilon = .0001 self.embedding_list = [torch.ones(3) * 1 / 3 for _ in range(self.num_schedules)]
def __init__(self, num_schedules): self.arguments = Logger() self.alpha = .9 self.num_schedules = num_schedules self.home_dir = self.arguments.home_dir self.total_loss_array = [] load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/datasets/' + str( self.num_schedules) + '_inf_hetero_deadline_naive.pkl' self.data = pickle.load(open(load_directory, "rb")) self.X, self.Y, self.schedule_array = create_new_dataset( num_schedules=self.num_schedules, data=self.data) for i, each_element in enumerate(self.X): self.X[i] = each_element + list(range(20)) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") self.model = BLSTMSmall().to(device) print(self.model.state_dict()) blstm_params = list(self.model.parameters()) del blstm_params[12] self.opt = torch.optim.Adam( blstm_params, lr=.0001) # TODO: try together and not together self.embedding_optimizer = torch.optim.SGD( self.model.EmbeddingList.parameters(), lr=.001) self.num_iterations_predict_task = 0 self.total_iterations = 0 self.convergence_epsilon = .01 self.when_to_save = 1000 self.distribution_epsilon = .0001 self.embedding_list = [ torch.ones(1, 8) * 1 / 3 for _ in range(self.num_schedules) ]
def __init__(self, num_schedules): self.arguments = Logger() self.alpha = .9 self.num_schedules = num_schedules self.home_dir = self.arguments.home_dir self.total_loss_array = [] load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/datasets/' + str( self.num_schedules) + '_inf_hetero_deadline_naive.pkl' self.data = pickle.load(open(load_directory, "rb")) self.X, self.Y, self.schedule_array = create_new_dataset( num_schedules=self.num_schedules, data=self.data) for i, each_element in enumerate(self.X): self.X[i] = each_element + list(range(20)) self.model = ProLoNet(input_dim=len(self.X[0]), weights=None, comparators=None, leaves=16, output_dim=20, bayesian_embedding_dim=8, alpha=1.5, use_gpu=True, vectorized=True, is_value=False) use_gpu = True if use_gpu: self.model = self.model.cuda() print(self.model.state_dict()) params = list(self.model.parameters()) del params[0] self.opt = torch.optim.RMSprop([{ 'params': params }, { 'params': self.model.bayesian_embedding, 'lr': .001 }]) self.num_iterations_predict_task = 0 self.total_iterations = 0 self.covergence_epsilon = .01 self.when_to_save = 1000 self.distribution_epsilon = .0001 self.max_depth = 10 # TODO: add back in deepening self.embedding_list = [ torch.ones(8) * 1 / 3 for _ in range(self.num_schedules) ]
def __init__(self, alpha): self.arguments = Logger() self.alpha = alpha self.num_schedules = 150 self.home_dir = self.arguments.home_dir self.total_loss_array = [] load_directory = '/home/ghost/PycharmProjects/scheduling_environment/new_data_pickle/' + str( self.num_schedules) + 'pairwise.pkl' bayesian_embedding_dim = 4 self.data = pickle.load(open(load_directory, "rb")) self.X, self.Y, self.schedule_array = create_new_dataset( self.num_schedules) self.embedding_list = [ torch.ones(bayesian_embedding_dim) * 1 / 3 for _ in range(self.num_schedules) ] # device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") self.model = ProLoNet(input_dim=len(self.X[0]), weights=None, comparators=None, leaves=64, output_dim=2, bayesian_embedding_dim=bayesian_embedding_dim, alpha=1.5, use_gpu=True, vectorized=False, is_value=False).cuda() use_gpu = True if use_gpu: self.model = self.model.cuda() print(self.model.state_dict()) params = list(self.model.parameters()) del params[0] self.opt = torch.optim.RMSprop([{ 'params': params }, { 'params': self.model.bayesian_embedding, 'lr': .001 }]) self.num_iterations_predict_task = 0 self.total_iterations = 0 self.covergence_epsilon = .01 self.when_to_save = 1000 self.distribution_epsilon = .0001
def train(): num_schedules = 150 total_loss_array = [] load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/datasets/' + str( num_schedules) + 'dist_early_hili_naive.pkl' data = pickle.load(open(load_directory, "rb")) X, Y, schedule_array = create_new_dataset(data=data, num_schedules=num_schedules) for i, each_element in enumerate(X): X[i] = each_element + list(range(20)) X_train = copy.deepcopy(X) Y_train = copy.deepcopy(Y) return X_train, Y_train
def __init__(self, num_schedules): self.num_schedules = num_schedules load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/datasets/' + str( num_schedules) + 'dist_early_hili_naive.pkl' self.data = pickle.load(open(load_directory, "rb")) self.X, self.Y, self.schedule_array = create_new_dataset( num_schedules=self.num_schedules, data=self.data) for i, each_element in enumerate(self.X): self.X[i] = each_element + list(range(20)) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") self.model = Autoencoder().to(device) print(self.model.state_dict()) self.opt = torch.optim.SGD(self.model.parameters(), lr=.0001) self.mean_embedding = None self.embedding_np = None self.matrixes = None self.total_binary_embeddings = None self.states = None
def __init__(self): self.arguments = Logger() self.alpha = .9 self.num_schedules = 150 self.home_dir = self.arguments.home_dir self.total_loss_array = [] # TODO: load in new directory load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/datasets/' + str( self.num_schedules) + 'dist_early_hili_naive.pkl' self.data = pickle.load(open(load_directory, "rb")) self.X, self.Y, self.schedule_array = create_new_dataset( num_schedules=self.num_schedules, data=self.data) for i, each_element in enumerate(self.X): self.X[i] = each_element + list(range(20)) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") model1 = NNSmall().to(device) model2 = NNSmall().to(device) model3 = NNSmall().to(device) self.models = [model1, model2, model3] opt1 = torch.optim.RMSprop(model1.parameters(), lr=.0001) opt2 = torch.optim.RMSprop(model2.parameters(), lr=.0001) opt3 = torch.optim.RMSprop(model3.parameters(), lr=.0001) self.optimizers = [opt1, opt2, opt3] self.num_iterations_predict_task = 0 self.total_iterations = 0 self.convergence_epsilon = .01 self.when_to_save = 1000 self.distribution_epsilon = .0001 schedule_matrix_load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/' + str( self.num_schedules) + 'matrixes.pkl' self.matrices = pickle.load(open(schedule_matrix_load_directory, "rb")) self.kmeans_model, self.label = self.cluster_matrices( self.matrices, self.num_schedules)
def __init__(self): self.arguments = Logger() self.alpha = .9 self.num_schedules = 150 self.home_dir = self.arguments.home_dir self.total_loss_array = [] load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/datasets/' + str( self.num_schedules) + '_inf_hetero_deadline_naive.pkl' self.data = pickle.load(open(load_directory, "rb")) self.X, self.Y, self.schedule_array = create_new_dataset( self.data, num_schedules=self.num_schedules) for i, each_element in enumerate(self.X): self.X[i] = each_element + list(range(20)) # device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") self.model = ProLoNet(input_dim=len(self.X[0]), weights=None, comparators=None, leaves=256, output_dim=20, bayesian_embedding_dim=None, alpha=1.5, use_gpu=True, vectorized=False, is_value=False) use_gpu = True if use_gpu: self.model = self.model.cuda() print(self.model.state_dict()) params = list(self.model.parameters()) self.opt = torch.optim.RMSprop([{'params': params}]) self.num_iterations_predict_task = 0 self.total_iterations = 0 self.covergence_epsilon = .01 self.when_to_save = 1000 self.distribution_epsilon = .0001
def test(X_train, Y_train): load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/datasets/' + str( 100) + 'test_dist_early_hili_naive.pkl' data = pickle.load(open(load_directory, "rb")) X, Y, schedule_array = create_new_dataset(data=data, num_schedules=100) for i, each_element in enumerate(X): X[i] = each_element + list(range(20)) X_test = X Y_test = Y clf = DecisionTreeClassifier(max_depth=15) clf.fit(X_train, Y_train) y_pred = clf.predict(X_train) print(accuracy_score(Y_train, y_pred)) y_pred_test = clf.predict(X_test) print(accuracy_score(Y_test, y_pred_test)) counter = 0 acc = 0 accs = [] for j, i in enumerate(y_pred_test): at_20 = False if j % 20 == 0 and j != 0: counter += 1 at_20 = True if at_20: accs.append(acc / 20) acc = 0 else: if i == Y_test[j]: acc += 1 print(np.mean(accs)) print(np.std(accs)) tree.export_graphviz(clf, out_file='tree.dot')
def evaluate_on_test_data(self, load_in_model=False): """ Evaluate performance of a trained network tuned upon the alpha divergence loss. Note this function is called after training convergence :return: """ num_schedules = 75 loss_func = AlphaLoss() # load in new data load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/datasets/test/' + str( num_schedules) + '_inf_hetero_deadline_naive.pkl' data = pickle.load(open(load_directory, "rb")) X, Y, schedule_array = create_new_dataset(num_schedules=num_schedules, data=data) for i, each_element in enumerate(X): X[i] = each_element + list(range(20)) prediction_accuracy = [0, 0] percentage_accuracy_top1 = [] percentage_accuracy_top3 = [] embedding_optimizer = torch.optim.Adam( self.model.EmbeddingList.parameters(), lr=.001) embedding_list = [ torch.ones(1, 8) * 1 / 3 for i in range(num_schedules) ] if load_in_model: # TODO: somehow get the string when the update_model flag is true self.model.load_state_dict( torch.load( '/home/ghost/PycharmProjects/bayesian_prolo/saved_models/pairwise_saved_models/NN_homog.tar' )['nn_state_dict']) for i, schedule in enumerate(schedule_array): self.model.reinitialize_hidden_to_random() load_in_embedding_bnn(self.model, embedding_list, i) for count in range(schedule[0], schedule[1] + 1): previous_hidden_state = tuple( [t.detach().cuda() for t in self.model.hidden]) net_input = X[count] truth = Y[count] if torch.cuda.is_available(): input_nn = Variable( torch.Tensor(np.asarray(net_input).reshape( 1, 242)).cuda()) truth = Variable( torch.Tensor( np.asarray(truth).reshape(1)).cuda().long()) P = Variable(torch.Tensor(np.ones((1, 20)))).cuda() P *= self.distribution_epsilon P[0][truth] = 1 - 19 * self.distribution_epsilon else: input_nn = Variable( torch.Tensor(np.asarray(net_input).reshape(1, 242))) truth = Variable(torch.Tensor( np.asarray(truth).reshape(1))) P = Variable(torch.Tensor(np.ones((1, 20)))) P *= self.distribution_epsilon P[0][truth] = 1 - 19 * self.distribution_epsilon #####forward##### output = self.model.forward(input_nn, previous_hidden_state) loss = loss_func.forward(P, output, self.alpha) if loss.item() < .05 or loss.item() > 5: pass else: loss.backward() torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) embedding_optimizer.step() index = torch.argmax(output).item() # top 3 _, top_three = torch.topk(output, 3) if index == truth.item(): prediction_accuracy[0] += 1 if truth.item() in top_three.detach().cpu().tolist()[0]: prediction_accuracy[1] += 1 store_embedding_back_bnn(self.model, embedding_list, i) # schedule finished print('Prediction Accuracy: top1: ', prediction_accuracy[0] / 20, ' top3: ', prediction_accuracy[1] / 20) print('schedule num:', i) percentage_accuracy_top1.append(prediction_accuracy[0] / 20) percentage_accuracy_top3.append(prediction_accuracy[1] / 20) prediction_accuracy = [0, 0] self.save_performance_results( percentage_accuracy_top1, percentage_accuracy_top3, 'inf_blstm_small_' + str(self.num_schedules))
def evaluate_on_test_data(self, models, schedules_trained_on, load_in_model=False): """ Evaluate performance of a trained network tuned upon the alpha divergence loss. Note this function is called after training convergence :return: """ autoencoder_class = AutoEncoderTrain(150) checkpoint = torch.load( '/home/ghost/PycharmProjects/bayesian_prolo/saved_models/Autoencoder150.tar' ) autoencoder_class.model.load_state_dict(checkpoint['nn_state_dict']) states = self.create_iterables() load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/datasets/' + str( 100) + 'test_dist_early_hili_naive.pkl' data = pickle.load(open(load_directory, "rb")) X_naive, Y_naive, schedule_array = create_new_dataset(data, 100) for i, each_element in enumerate(X_naive): X_naive[i] = each_element + list(range(20)) num_schedules = 100 # load in new data load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/datasets/' + str( num_schedules) + 'test_dist_early_hili_pairwise.pkl' data = pickle.load(open(load_directory, "rb")) X, Y, schedule_array = create_new_data(num_schedules, data) prediction_accuracy = [0, 0] percentage_accuracy_top1 = [] percentage_accuracy_top3 = [] mean_input = [ 1.3277743, 0.32837677, 1.4974482, -1.3519306, -0.64621973, 0.10534518, -2.338118, -2.7345326, 1.7558736, -3.0746384, -3.485554 ] for j in range(0, num_schedules): current_schedule_matrix = np.zeros((2048, 20)) schedule_bounds = schedule_array[j] step = schedule_bounds[0] while step < schedule_bounds[1]: probability_matrix = np.zeros((20, 20)) if current_schedule_matrix.sum() == 0: cluster_num = self.kmeans_model.predict( current_schedule_matrix.reshape(1, -1)) else: matrix = np.divide(current_schedule_matrix, current_schedule_matrix.sum()) cluster_num = self.kmeans_model.predict( matrix.reshape(1, -1)) for m, counter in enumerate(range(step, step + 20)): phi_i = X[counter] phi_i_numpy = np.asarray(phi_i) # for each set of twenty for n, second_counter in enumerate(range(step, step + 20)): # fill entire array with diagnols set to zero if second_counter == counter: # same as m = n continue phi_j = X[second_counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_i_numpy - phi_j_numpy if torch.cuda.is_available(): feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13)).cuda()) else: feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13))) # push through nets to get preferences preference_prob = self.models[int( cluster_num)].forward(feature_input) probability_matrix[m][n] = preference_prob[ 0].data.detach()[0].item( ) # TODO: you can do a check if only this line leads to the same thing as the line below # probability_matrix[n][m] = preference_prob[0].data.detach()[1].item() # Set of twenty is completed column_vec = np.sum(probability_matrix, axis=1) # top 1 highest_val = max(column_vec) all_indexes_that_have_highest_val = [ i for i, e in enumerate(list(column_vec)) if e == highest_val ] if len(all_indexes_that_have_highest_val) > 1: print('length of indexes greater than 1: ', all_indexes_that_have_highest_val) # top 1 choice = np.random.choice(all_indexes_that_have_highest_val) # choice = np.argmax(probability_vector) # top 3 _, top_three = torch.topk(torch.Tensor(column_vec), 3) # Then do training update loop truth = Y[step] # index top 1 if choice == truth: prediction_accuracy[0] += 1 # index top 3 if truth in top_three: prediction_accuracy[1] += 1 embedding_copy = np.zeros((1, 11)) input_element = autoencoder_class.model.forward_only_encoding( Variable( torch.Tensor( np.asarray(X_naive[int(step / 20)]).reshape( 1, 242)).cuda())) for z, each_element in enumerate(mean_input): if each_element > input_element[0][z].item(): embedding_copy[0][z] = 0 else: embedding_copy[0][z] = 1 index = self.pass_in_embedding_out_state_ID( states, embedding_copy[0]) action = Y[step] current_schedule_matrix[index][int(action)] += 1 # add average loss to array step += 20 # schedule finished print('Prediction Accuracy: top1: ', prediction_accuracy[0] / 20, ' top3: ', prediction_accuracy[1] / 20) print('schedule num:', j) percentage_accuracy_top1.append(prediction_accuracy[0] / 20) percentage_accuracy_top3.append(prediction_accuracy[1] / 20) prediction_accuracy = [0, 0] save_performance_results(percentage_accuracy_top1, percentage_accuracy_top3, 'pairwise_NN_kmeans.pkl') return percentage_accuracy_top1
from utils.naive_utils import create_new_dataset from sklearn.metrics import accuracy_score from sklearn import tree sys.path.insert(0, '../') # Naive num_schedules = 150 total_loss_array = [] load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/datasets/' + str( num_schedules) + '_task_num_homog_deadline_naive.pkl' data = pickle.load(open(load_directory, "rb")) X, Y, schedule_array = create_new_dataset(data=data, num_schedules=num_schedules) for i, each_element in enumerate(X): X[i] = each_element + list(range(20)) X_train = X[0:int(len(X) * .8)] Y_train = Y[0:int(len(X) * .8)] X_test = X[int(len(X) * .8):int(len(X))] Y_test = Y[int(len(X) * .8):int(len(X))] clf = DecisionTreeClassifier(max_depth=15) clf.fit(X_train, Y_train) y_pred = clf.predict(X_train) print(accuracy_score(Y_train, y_pred)) y_pred_test = clf.predict(X_test) print(accuracy_score(Y_test, y_pred_test))
def evaluate_on_test_data(self): """ Evaluate performance of a trained network. This is tested on 20% of the data and will be stored in a text file. :return: """ # confusion_matrix = np.zeros((20,20)) num_schedules = 100 autoencoder_class = AutoEncoderTrain(150) checkpoint = torch.load( '/home/ghost/PycharmProjects/bayesian_prolo/saved_models/Autoencoder150.tar' ) autoencoder_class.model.load_state_dict(checkpoint['nn_state_dict']) states = self.create_iterables() # load in new data load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/datasets/' + str( num_schedules) + 'test_dist_early_hili_naive.pkl' data = pickle.load(open(load_directory, "rb")) X, Y, schedule_array = create_new_dataset(data, num_schedules) for i, each_element in enumerate(X): X[i] = each_element + list(range(20)) # only use last 100 as held out dataset # X = X[-2000:] # Y = Y[-2000:] prediction_accuracy = [0, 0] percentage_accuracy_top1 = [] percentage_accuracy_top3 = [] mean_input = [ 1.3277743, 0.32837677, 1.4974482, -1.3519306, -0.64621973, 0.10534518, -2.338118, -2.7345326, 1.7558736, -3.0746384, -3.485554 ] for i, schedule in enumerate(schedule_array): current_schedule_matrix = np.zeros((2048, 20)) for count in range(schedule[0], schedule[1] + 1): if current_schedule_matrix.sum() == 0: cluster_num = self.kmeans_model.predict( current_schedule_matrix.reshape(1, -1)) else: matrix = np.divide(current_schedule_matrix, current_schedule_matrix.sum()) cluster_num = self.kmeans_model.predict( matrix.reshape(1, -1)) net_input = X[count] truth = Y[count] if torch.cuda.is_available(): input_nn = Variable( torch.Tensor(np.asarray(net_input).reshape( 1, 242)).cuda()) truth = Variable( torch.Tensor( np.asarray(truth).reshape(1)).cuda().long()) else: input_nn = Variable( torch.Tensor(np.asarray(net_input).reshape(1, 242))) truth = Variable(torch.Tensor( np.asarray(truth).reshape(1))) #####forward##### output = self.models[int(cluster_num)].forward(input_nn) index = torch.argmax(output).item() # confusion_matrix[truth][index] += 1 # top 3 _, top_three = torch.topk(output, 3) if index == truth.item(): prediction_accuracy[0] += 1 if truth.item() in top_three.detach().cpu().tolist()[0]: prediction_accuracy[1] += 1 # update matrix embedding_copy = np.zeros((1, 11)) input_element = autoencoder_class.model.forward_only_encoding( input_nn) for z, each_element in enumerate(mean_input): if each_element > input_element[0][z].item(): embedding_copy[0][z] = 0 else: embedding_copy[0][z] = 1 index = self.pass_in_embedding_out_state_ID( states, embedding_copy[0]) action = truth.item() current_schedule_matrix[index][int(action)] += 1 print('Prediction Accuracy: top1: ', prediction_accuracy[0] / 20, ' top3: ', prediction_accuracy[1] / 20) print('schedule num:', i) percentage_accuracy_top1.append(prediction_accuracy[0] / 20) percentage_accuracy_top3.append(prediction_accuracy[1] / 20) prediction_accuracy = [0, 0] print(np.mean(percentage_accuracy_top1)) self.save_performance_results(percentage_accuracy_top1, percentage_accuracy_top3, 'kmeans_to_NN_naive') return percentage_accuracy_top1
def evaluate_on_test_data(self): """ Evaluate performance of a trained network. This is tested on 20% of the data and will be stored in a text file. :return: """ loss_func = AlphaLoss() num_schedules = 75 # load in new data load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/datasets/test/' + str( num_schedules) + '_inf_hetero_deadline_naive.pkl' data = pickle.load(open(load_directory, "rb")) X, Y, schedule_array = create_new_dataset(data, num_schedules) for i, each_element in enumerate(X): X[i] = each_element + list(range(20)) prediction_accuracy = [0, 0] percentage_accuracy_top1 = [] percentage_accuracy_top3 = [] embedding_optimizer = torch.optim.SGD( self.model.EmbeddingList.parameters(), lr=.001) embedding_list = [ torch.ones(1, 8) * 1 / 3 for i in range(num_schedules) ] for i, schedule in enumerate(schedule_array): load_in_embedding_bnn(self.model, embedding_list, i) for count in range(schedule[0], schedule[1] + 1): net_input = X[count] truth = Y[count] if torch.cuda.is_available(): input_nn = Variable( torch.Tensor(np.asarray(net_input).reshape( 1, 242)).cuda()) truth = Variable( torch.Tensor( np.asarray(truth).reshape(1)).cuda().long()) P = Variable(torch.Tensor(np.ones((1, 20)))).cuda() P *= self.distribution_epsilon P[0][truth] = 1 - 19 * self.distribution_epsilon else: input_nn = Variable( torch.Tensor(np.asarray(net_input).reshape(1, 242))) truth = Variable(torch.Tensor( np.asarray(truth).reshape(1))) P = Variable(torch.Tensor(np.ones((1, 20)))) P *= self.distribution_epsilon P[0][truth] = 1 - 19 * self.distribution_epsilon #####forward##### output = self.model.forward(input_nn) loss = loss_func.forward(P, output, self.alpha) if loss.item() < .05 or loss.item() > 5: pass else: embedding_optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) embedding_optimizer.step() index = torch.argmax(output).item() # top 3 _, top_three = torch.topk(output, 3) if index == truth.item(): prediction_accuracy[0] += 1 if truth.item() in top_three.detach().cpu().tolist()[0]: prediction_accuracy[1] += 1 print('Prediction Accuracy: top1: ', prediction_accuracy[0] / 20, ' top3: ', prediction_accuracy[1] / 20) print('schedule num:', i) percentage_accuracy_top1.append(prediction_accuracy[0] / 20) percentage_accuracy_top3.append(prediction_accuracy[1] / 20) prediction_accuracy = [0, 0] store_embedding_back_bnn(self.model, embedding_list, i) print(np.mean(percentage_accuracy_top1)) self.save_performance_results( percentage_accuracy_top1, percentage_accuracy_top3, 'inf_bnn_small_' + str(self.num_schedules))
def evaluate_on_test_data(self): """ Evaluate performance of a trained network. This is tested on 20% of the data and will be stored in a text file. :return: """ num_schedules = 100 # load in new data load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/datasets/' + str( num_schedules) + 'test_dist_early_hili_naive.pkl' data = pickle.load(open(load_directory, "rb")) X, Y, schedule_array = create_new_dataset(data, num_schedules) for i, each_element in enumerate(X): X[i] = each_element + list(range(20)) prediction_accuracy = [0, 0] percentage_accuracy_top1 = [] percentage_accuracy_top3 = [] embedding_list = [torch.ones(3) * 1/3 for i in range(num_schedules)] for i, schedule in enumerate(schedule_array): self.model.set_bayesian_embedding(embedding_list[i]) for count in range(schedule[0], schedule[1] + 1): net_input = X[count] truth = Y[count] if torch.cuda.is_available(): input_nn = Variable(torch.Tensor(np.asarray(net_input).reshape(1, 242)).cuda()) truth = Variable(torch.Tensor(np.asarray(truth).reshape(1)).cuda().long()) else: input_nn = Variable(torch.Tensor(np.asarray(net_input).reshape(1, 242))) truth = Variable(torch.Tensor(np.asarray(truth).reshape(1))) #####forward##### output = self.model.forward(input_nn) loss = F.cross_entropy(output, truth) * 10 self.embedding_optimizer.zero_grad() print(self.model.EmbeddingList.state_dict()) loss.backward() self.embedding_optimizer.step() index = torch.argmax(output).item() print(self.model.EmbeddingList.state_dict()) # top 3 _, top_three = torch.topk(output, 3) if index == truth.item(): prediction_accuracy[0] += 1 if truth.item() in top_three.detach().cpu().tolist()[0]: prediction_accuracy[1] += 1 print('Prediction Accuracy: top1: ', prediction_accuracy[0] / 20, ' top3: ', prediction_accuracy[1] / 20) print('schedule num:', i) percentage_accuracy_top1.append(prediction_accuracy[0] / 20) percentage_accuracy_top3.append(prediction_accuracy[1] / 20) prediction_accuracy = [0, 0] print(np.mean(percentage_accuracy_top1)) self.save_performance_results(percentage_accuracy_top1, percentage_accuracy_top3, 'NN_w_embedding')
def evaluate_on_test_data(self, load_in_model=False): """ Evaluate performance of a trained network tuned upon the alpha divergence loss. Note this function is called after training convergence :return: """ # define new optimizer that only optimizes gradient num_schedules = 100 load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/datasets/' + str( num_schedules) + 'test_dist_early_hili_naive.pkl' data = pickle.load(open(load_directory, "rb")) X, Y, schedule_array = create_new_dataset(num_schedules=num_schedules, data=data) for i, each_element in enumerate(X): X[i] = each_element + list(range(20)) embedding_optimizer = torch.optim.SGD([{'params': self.model.bayesian_embedding.parameters()}], lr=.1) embedding_list = [torch.ones(8) * 1 / 3 for _ in range(num_schedules)] prediction_accuracy = [0, 0] percentage_accuracy_top1 = [] percentage_accuracy_top3 = [] if load_in_model: self.model.load_state_dict(torch.load('/home/ghost/PycharmProjects/bayesian_prolo/saved_models/pairwise_saved_models/model_homog.tar')['nn_state_dict']) for i, schedule in enumerate(schedule_array): self.model.set_bayesian_embedding(self.embedding_list[i]) for count in range(schedule[0], schedule[1] + 1): net_input = X[count] truth = Y[count] if torch.cuda.is_available(): input_nn = Variable(torch.Tensor(np.asarray(net_input).reshape(1, 242)).cuda()) # change to 5 to increase batch size truth = Variable(torch.Tensor(np.asarray(truth).reshape(1)).cuda().long()) else: input_nn = Variable(torch.Tensor(np.asarray(net_input).reshape(1, 242))) truth = Variable(torch.Tensor(np.asarray(truth).reshape(1)).long()) #####forward##### output = self.model.forward(input_nn) embedding_optimizer.zero_grad() loss = F.cross_entropy(output, truth) loss.backward() # torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) embedding_optimizer.step() index = torch.argmax(output).item() # top 3 _, top_three = torch.topk(output, 3) if index == truth.item(): prediction_accuracy[0] += 1 if truth.item() in top_three.detach().cpu().tolist()[0]: prediction_accuracy[1] += 1 # add average loss to array embedding_list[i] = torch.Tensor(self.model.get_bayesian_embedding().detach().cpu().numpy()) # very ugly # schedule finished print('Prediction Accuracy: top1: ', prediction_accuracy[0] / 20, ' top3: ', prediction_accuracy[1] / 20) print('schedule num:', i) percentage_accuracy_top1.append(prediction_accuracy[0] / 20) percentage_accuracy_top3.append(prediction_accuracy[1] / 20) prediction_accuracy = [0, 0] self.save_performance_results(percentage_accuracy_top1, percentage_accuracy_top3, 'DDT_w_embedding')
def evaluate_on_test_data(self): """ Evaluate performance of a trained network. This is tested on 20% of the data and will be stored in a text file. :return: """ num_schedules = 75 # load in new data load_directory = '/home/ghost/PycharmProjects/bayesian_prolo/scheduling_env/datasets/test/' + str( num_schedules) + '_inf_hetero_deadline_naive.pkl' data = pickle.load(open(load_directory, "rb")) X, Y, schedule_array = create_new_dataset(data, num_schedules) for i, each_element in enumerate(X): X[i] = each_element + list(range(20)) prediction_accuracy = [0, 0] percentage_accuracy_top1 = [] percentage_accuracy_top3 = [] for i, schedule in enumerate(schedule_array): for count in range(schedule[0], schedule[1] + 1): net_input = X[count] truth = Y[count] if torch.cuda.is_available(): input_nn = Variable( torch.Tensor(np.asarray(net_input).reshape( 1, 242)).cuda()) truth = Variable( torch.Tensor( np.asarray(truth).reshape(1)).cuda().long()) else: input_nn = Variable( torch.Tensor(np.asarray(net_input).reshape(1, 242))) truth = Variable(torch.Tensor( np.asarray(truth).reshape(1))) #####forward##### output = self.model.forward(input_nn) index = torch.argmax(output).item() # top 3 _, top_three = torch.topk(output, 3) if index == truth.item(): prediction_accuracy[0] += 1 if truth.item() in top_three.detach().cpu().tolist()[0]: prediction_accuracy[1] += 1 print('Prediction Accuracy: top1: ', prediction_accuracy[0] / 20, ' top3: ', prediction_accuracy[1] / 20) print('schedule num:', i) percentage_accuracy_top1.append(prediction_accuracy[0] / 20) percentage_accuracy_top3.append(prediction_accuracy[1] / 20) prediction_accuracy = [0, 0] print(np.mean(percentage_accuracy_top1)) self.save_performance_results( percentage_accuracy_top1, percentage_accuracy_top3, 'inf_nn_small_' + str(self.num_schedules))