def generate_data(self): """ Generates a bunch of counterfactual data (poorly done) :return: """ data_matrix = [] output_matrix = [] # variables to keep track of loss and number of tasks trained over while True: # sample a timestep before the cutoff for cross_validation set_of_twenty = np.random.choice(self.start_of_each_set_twenty) truth = self.Y[set_of_twenty] which_schedule = find_which_schedule_this_belongs_to( self.schedule_array, set_of_twenty) # find feature vector of true action taken phi_i_num = truth + set_of_twenty phi_i = self.X[phi_i_num] phi_i_numpy = np.asarray(phi_i) # iterate over pairwise comparisons for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: # if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_i_numpy - phi_j_numpy feature_input = list(feature_input) feature_input.extend( self.embedding_list[which_schedule].data.numpy()) data_matrix.append(list(feature_input)) output_matrix.append(1) for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_j_numpy - phi_i_numpy feature_input = list(feature_input) feature_input.extend( self.embedding_list[which_schedule].data.numpy()) data_matrix.append(list(feature_input)) output_matrix.append(0) if len(data_matrix) > 300000: return data_matrix, output_matrix
def train(self): """ Trains NN. Randomly samples a schedule and timestep within that schedule, produces training data using x_i - x_j and trains upon that. :return: """ sets_of_twenty = None training_done = False loss_func = AlphaLoss() # variables to keep track of loss and number of tasks trained over running_loss_predict_tasks = 0 num_iterations_predict_task = 0 while not training_done: # sample a timestep before the cutoff for cross_validation # Quick Fix found_a_suitable_candidate = False while not found_a_suitable_candidate: rand_timestep_within_sched = np.random.randint( len(self.start_of_each_set_twenty)) set_of_twenty = self.start_of_each_set_twenty[ rand_timestep_within_sched] which_schedule = find_which_schedule_this_belongs_to( self.schedule_array, set_of_twenty) if set_of_twenty + 59 > self.schedule_array[which_schedule][1]: pass else: found_a_suitable_candidate = True sets_of_twenty = [ set_of_twenty, set_of_twenty + 20, set_of_twenty + 40 ] self.model.reinitialize_hidden_to_random() for set_of_twenty in sets_of_twenty: truth = self.Y[set_of_twenty] # find feature vector of true action taken phi_i_num = truth + set_of_twenty phi_i = self.X[phi_i_num] phi_i_numpy = np.asarray(phi_i) previous_hidden_state = tuple( [t.detach().cuda() for t in self.model.hidden]) # iterate over pairwise comparisons for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: # if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_i_numpy - phi_j_numpy if torch.cuda.is_available(): feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13)).cuda()) P = Variable( torch.Tensor([ 1 - self.distribution_epsilon, self.distribution_epsilon ]).cuda()) else: feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13))) P = Variable( torch.Tensor([ 1 - self.distribution_epsilon, self.distribution_epsilon ])) output = self.model.forward(feature_input, previous_hidden_state) if torch.isnan(output[0][0]).item() == 1: print('hi') self.opt.zero_grad() loss = loss_func.forward(P, output, self.alpha) if torch.isnan(loss): print(self.alpha, ' :nan occurred at iteration ', self.total_iterations, ' at', num_iterations_predict_task) if loss.item() < .001 or loss.item() > 50: pass else: loss.backward() torch.nn.utils.clip_grad_norm_( self.model.parameters(), 0.5) self.opt.step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 # second loop for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_j_numpy - phi_i_numpy if torch.cuda.is_available(): feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13)).cuda()) P = Variable( torch.Tensor([ self.distribution_epsilon, 1 - self.distribution_epsilon ]).cuda()) else: feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13))) P = Variable( torch.Tensor([ self.distribution_epsilon, 1 - self.distribution_epsilon ])) output = self.model.forward(feature_input, previous_hidden_state) if torch.isnan(output[0][0]).item() == 1: print('hi') self.opt.zero_grad() loss = loss_func.forward(P, output, self.alpha) if torch.isnan(loss): print(self.alpha, ' :nan occurred at iteration ', self.total_iterations, ' at', num_iterations_predict_task) if loss.item() < .001 or loss.item() > 50: pass else: loss.backward() torch.nn.utils.clip_grad_norm_( self.model.parameters(), 0.5) self.opt.step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 self.total_loss_array.append(running_loss_predict_tasks / num_iterations_predict_task) num_iterations_predict_task = 0 running_loss_predict_tasks = 0 self.total_iterations += 1 if self.total_iterations > 25 and self.total_iterations % 50 == 1: print('total iterations is', self.total_iterations) print('total loss (average for each 40, averaged)', np.mean(self.total_loss_array[-40:])) if self.total_iterations > 0 and self.total_iterations % self.when_to_save == self.when_to_save - 1: self.save_trained_nets('lstm_small' + str(self.num_schedules)) if self.total_iterations > 1500 and np.mean( self.total_loss_array[-100:]) - np.mean( self.total_loss_array[-500:] ) < self.convergence_epsilon: training_done = True
def train(self): """ Trains BDT. Randomly samples a schedule and timestep within that schedule, produces training data using x_i - x_j and trains upon that. :return: """ training_done = False cv_cutoff = .8 * len(self.start_of_each_set_twenty) loss_func = AlphaLoss() # variables to keep track of loss and number of tasks trained over running_loss_predict_tasks = 0 num_iterations_predict_task = 0 while not training_done: # sample a timestep before the cutoff for cross_validation rand_timestep_within_sched = np.random.randint(cv_cutoff) set_of_twenty = self.start_of_each_set_twenty[ rand_timestep_within_sched] # truth is the same as the task chosen # NOTE: if initial truth > 10, it is biggest task first. Else it is smallest task first. Approximate way to keep track of # the current type of schedule truth = self.Y[set_of_twenty] # get the current schedule that the timestep is from which_schedule = find_which_schedule_this_belongs_to( self.schedule_array, set_of_twenty) # load in the embedding from the list into the network load_in_embedding(self.model, self.embedding_list, which_schedule) # find feature vector of true action taken phi_i_num = truth + set_of_twenty phi_i = self.X[phi_i_num] phi_i_numpy = np.asarray(phi_i) # iterate over pairwise comparisons for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: # if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_i_numpy - phi_j_numpy # transform into torch variables if torch.cuda.is_available(): feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13)).cuda()) P = Variable( torch.Tensor([ 1 - self.distribution_epsilon, self.distribution_epsilon ]).cuda()) else: feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13))) P = Variable( torch.Tensor([ 1 - self.distribution_epsilon, self.distribution_epsilon ])) # get model output output = self.model(feature_input) loss = loss_func.forward(P, output, self.alpha) # Updates embeddings and optimizer after 5000 passes through the data if self.total_iterations > 5000: self.opt.zero_grad() self.embedding_optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) self.opt.step() self.embedding_optimizer.step() else: self.opt.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) self.opt.step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_j_numpy - phi_i_numpy if torch.cuda.is_available(): feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13)).cuda()) P = Variable( torch.Tensor([ self.distribution_epsilon, 1 - self.distribution_epsilon ]).cuda()) else: feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13))) P = Variable( torch.Tensor([ self.distribution_epsilon, 1 - self.distribution_epsilon ])) output = self.model(feature_input) loss = loss_func.forward(P, output, self.alpha) # Updates embeddings and optimizer after 5000 passes through the data if self.total_iterations > 5000: self.opt.zero_grad() self.embedding_optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) self.opt.step() self.embedding_optimizer.step() else: self.opt.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) self.opt.step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 # save the embedding back into the list self.embedding_list = store_embedding_back(self.model, self.embedding_list, which_schedule) # add average accuracy across pairwise comparisons to array self.total_loss_array.append(running_loss_predict_tasks / num_iterations_predict_task) num_iterations_predict_task = 0 running_loss_predict_tasks = 0 self.total_iterations += 1 if self.total_iterations > 25 and self.total_iterations % 50 == 1: print('total iterations is', self.total_iterations) print('total loss (average for each 40, averaged)', np.mean(self.total_loss_array[-40:])) if self.total_iterations > 0 and self.total_iterations % self.when_to_save == self.when_to_save - 1: print(self.embedding_list) self.save_trained_nets() if self.total_iterations > 8000 and np.mean( self.total_loss_array[-100:]) - np.mean( self.total_loss_array[-500:]) < self.covergence_epsilon: training_done = True
def train(self): """ Trains NN. Randomly samples a schedule and timestep within that schedule, produces training data using x_i - x_j and trains upon that. :return: """ total_iterations = 0 convergence_epsilon = .01 when_to_save = 1000 distribution_epsilon = .0001 training_done = False total_loss_array = [] criterion = torch.nn.BCELoss() # variables to keep track of loss and number of tasks trained over while not training_done: # choose a random timestep within any schedule set_of_twenty = np.random.choice(self.start_of_each_set_twenty) # get the true action scheduled truth = self.Y[set_of_twenty] which_schedule = find_which_schedule_this_belongs_to(self.schedule_array, set_of_twenty) # set the embedding self.model.set_bayesian_embedding(self.embedding_list[which_schedule]) # find feature vector of true action taken phi_i_num = truth + set_of_twenty phi_i = self.X[phi_i_num] phi_i_numpy = np.asarray(phi_i) running_loss_predict_tasks = 0 num_iterations_predict_task = 0 # iterate over pairwise comparisons for counter in range(set_of_twenty, set_of_twenty + 20): # positive counterfactuals if counter == phi_i_num: # if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_i_numpy - phi_j_numpy if torch.cuda.is_available(): feature_input = Variable(torch.Tensor(feature_input.reshape(1, 13)).cuda()) label = Variable(torch.Tensor(torch.ones((1,1))).cuda()) else: feature_input = Variable(torch.Tensor(feature_input.reshape(1, 13))) label = Variable(torch.Tensor(torch.ones((1, 1)))) output = self.model.forward(feature_input) self.opt.zero_grad() loss = criterion(output, label) loss.backward() # torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) self.opt.step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 # Negative counterfactuals for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_j_numpy - phi_i_numpy if torch.cuda.is_available(): feature_input = Variable(torch.Tensor(feature_input.reshape(1, 13)).cuda()) label = Variable(torch.Tensor(torch.zeros((1,1))).cuda()) else: feature_input = Variable(torch.Tensor(feature_input.reshape(1, 13))) label = Variable(torch.Tensor(torch.zeros((1,1)))) output = self.model.forward(feature_input) loss = criterion(output, label) loss.backward() # torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) self.opt.step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 self.embedding_list[which_schedule] = torch.Tensor(self.model.get_bayesian_embedding().detach().cpu().numpy()[0]) # very ugly print(self.embedding_list[which_schedule]) total_loss_array.append(running_loss_predict_tasks / num_iterations_predict_task) total_iterations += 1 if total_iterations % 50 == 49: print('total loss (average for each 40, averaged) at iteration ', total_iterations, ' is ', np.mean(total_loss_array[-40:])) if total_iterations % when_to_save == when_to_save - 1: self.save_trained_nets('nn_small' + str(self.num_schedules)) if total_iterations > 5000 and np.mean(total_loss_array[-100:]) - np.mean(total_loss_array[-500:]) < convergence_epsilon: training_done = True print(self.model.state_dict())
def train(self): """ Trains NN. Randomly samples a schedule and timestep within that schedule, produces training data using x_i - x_j and trains upon that. :return: """ total_iterations = 0 convergence_epsilon = .01 when_to_save = 1000 distribution_epsilon = .0001 training_done = False total_loss_array = [] criterion = torch.nn.BCELoss() # variables to keep track of loss and number of tasks trained over while not training_done: # sample a timestep before the cutoff for cross_validation set_of_twenty = np.random.choice(self.start_of_each_set_twenty) truth = self.Y[set_of_twenty] which_schedule = find_which_schedule_this_belongs_to( self.schedule_array, set_of_twenty) cluster_num = self.label[which_schedule] # find feature vector of true action taken phi_i_num = truth + set_of_twenty phi_i = self.X[phi_i_num] phi_i_numpy = np.asarray(phi_i) running_loss_predict_tasks = 0 num_iterations_predict_task = 0 # iterate over pairwise comparisons for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: # if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_i_numpy - phi_j_numpy if torch.cuda.is_available(): feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13)).cuda()) label = Variable( torch.Tensor(torch.ones((1, 1))).cuda()) else: feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13))) label = Variable(torch.Tensor(torch.ones((1, 1)))) output = self.models[cluster_num].forward(feature_input) if torch.isnan(output[0][0]).item() == 1: print('hi') self.optimizers[cluster_num].zero_grad() loss = criterion(output, label) loss.backward() # torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) self.optimizers[cluster_num].step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 # second loop for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_j_numpy - phi_i_numpy if torch.cuda.is_available(): feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13)).cuda()) label = Variable( torch.Tensor(torch.zeros((1, 1))).cuda()) else: feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13))) label = Variable(torch.Tensor(torch.zeros((1, 1)))) self.optimizers[cluster_num].zero_grad() output = self.models[cluster_num].forward(feature_input) loss = criterion(output, label) loss.backward() # torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) self.optimizers[cluster_num].step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 total_loss_array.append(running_loss_predict_tasks / num_iterations_predict_task) total_iterations += 1 if total_iterations % 50 == 49: print( 'total loss (average for each 40, averaged) at iteration ', total_iterations, ' is ', np.mean(total_loss_array[-40:])) if total_iterations > 5000 and np.mean( total_loss_array[-100:]) - np.mean( total_loss_array[-500:]) < convergence_epsilon: training_done = True
def train(self): """ Trains BDT. Randomly samples a schedule and timestep within that schedule, produces training data using x_i - x_j and trains upon that. :return: """ threshold = .1 training_done = False loss_func = AlphaLoss() # deepening data deepen_data = {'samples': [], 'labels': [], 'embedding_indices': []} # variables to keep track of loss and number of tasks trained over running_loss_predict_tasks = 0 num_iterations_predict_task = 0 while not training_done: # sample a timestep before the cutoff for cross_validation rand_timestep_within_sched = np.random.randint( len(self.start_of_each_set_twenty)) set_of_twenty = self.start_of_each_set_twenty[ rand_timestep_within_sched] truth = self.Y[set_of_twenty] which_schedule = find_which_schedule_this_belongs_to( self.schedule_array, set_of_twenty) load_in_embedding(self.model, self.embedding_list, which_schedule) # find feature vector of true action taken phi_i_num = truth + set_of_twenty phi_i = self.X[phi_i_num] phi_i_numpy = np.asarray(phi_i) # iterate over pairwise comparisons for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: # if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_i_numpy - phi_j_numpy deepen_data['samples'].append(np.array(feature_input)) # label = add_noise_pairwise(label, self.noise_percentage) if torch.cuda.is_available(): feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13)).cuda()) P = Variable( torch.Tensor([ 1 - self.distribution_epsilon, self.distribution_epsilon ]).cuda()) else: feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13))) P = Variable( torch.Tensor([ 1 - self.distribution_epsilon, self.distribution_epsilon ])) output = self.model(feature_input) loss = loss_func.forward(P, output, self.alpha) # NAN check (fix is in the bnn file) if torch.isnan(loss): print(self.alpha, ' :nan occurred at iteration ', self.total_iterations) # prepare optimizer, compute gradient, update params self.opt.zero_grad() if loss.item() < .001 or loss.item() > 50: pass else: loss.backward() torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) self.opt.step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 deepen_data['labels'].extend([0]) deepen_data['embedding_indices'].extend([which_schedule]) for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_j_numpy - phi_i_numpy deepen_data['samples'].append(np.array(feature_input)) if torch.cuda.is_available(): feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13)).cuda()) P = Variable( torch.Tensor([ self.distribution_epsilon, 1 - self.distribution_epsilon ]).cuda()) else: feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13))) P = Variable( torch.Tensor([ self.distribution_epsilon, 1 - self.distribution_epsilon ])) output = self.model(feature_input) loss = loss_func.forward(P, output, self.alpha) # if num_iterations_predict_task % 5 == 0: # print('loss is :', loss.item()) # clip any very high gradients # prepare optimizer, compute gradient, update params self.opt.zero_grad() if loss.item() < .001 or loss.item() > 50: pass else: loss.backward() torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) self.opt.step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 deepen_data['labels'].extend([1]) deepen_data['embedding_indices'].extend([which_schedule]) # add average loss to array # print(list(self.model.parameters())) self.embedding_list = store_embedding_back(self.model, self.embedding_list, which_schedule) self.total_loss_array.append(running_loss_predict_tasks / num_iterations_predict_task) num_iterations_predict_task = 0 running_loss_predict_tasks = 0 self.total_iterations += 1 if self.total_iterations > 25 and self.total_iterations % 50 == 1: print('total iterations is', self.total_iterations) print('total loss (average for each 40, averaged)', np.mean(self.total_loss_array[-40:])) if self.total_iterations > 0 and self.total_iterations % self.when_to_save == self.when_to_save - 1: self.save_trained_nets('BDDT' + str(self.num_schedules)) threshold -= .025 # if self.total_iterations % 500 == 499: # self.model = deepen_with_embeddings(self.model, deepen_data, self.embedding_list, max_depth=self.max_depth, threshold=threshold) # params = list(self.model.parameters()) # del params[0] # self.opt = torch.optim.RMSprop([{'params': params}, {'params': self.model.bayesian_embedding, 'lr': .001}]) # deepen_data = { # 'samples': [], # 'labels': [], # 'embedding_indices': [] # } if self.total_iterations > 5000 and np.mean( self.total_loss_array[-100:]) - np.mean( self.total_loss_array[-500:]) < self.covergence_epsilon: training_done = True
def train(self): """ Trains NN. Randomly samples a schedule and timestep within that schedule, produces training data using x_i - x_j and trains upon that. :return: """ training_done = False # cv_cutoff = int(.8 * len(self.start_of_each_set_twenty)) # TODO: delete or to not delete loss_func = AlphaLoss() # variables to keep track of loss and number of tasks trained over running_loss_predict_tasks = 0 num_iterations_predict_task = 0 while not training_done: # sample a timestep before the cutoff for cross_validation rand_timestep_within_sched = np.random.randint( len(self.start_of_each_set_twenty) * .8) set_of_twenty = self.start_of_each_set_twenty[ rand_timestep_within_sched] truth = self.Y[set_of_twenty] which_schedule = find_which_schedule_this_belongs_to( self.schedule_array, set_of_twenty) load_in_embedding_bnn(self.model, self.embedding_list, which_schedule) # find feature vector of true action taken phi_i_num = truth + set_of_twenty phi_i = self.X[phi_i_num] phi_i_numpy = np.asarray(phi_i) # iterate over pairwise comparisons for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: # if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_i_numpy - phi_j_numpy if torch.cuda.is_available(): feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13)).cuda()) P = Variable( torch.Tensor([ 1 - self.distribution_epsilon, self.distribution_epsilon ]).cuda()) else: feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13))) P = Variable( torch.Tensor([ 1 - self.distribution_epsilon, self.distribution_epsilon ])) output = self.model.forward(feature_input) self.opt.zero_grad() self.embedding_optimizer.zero_grad() loss = loss_func.forward(P, output, self.alpha) if torch.isnan(loss): print(self.alpha, ' :nan occurred at iteration ', self.total_iterations) if self.total_iterations > 7500: if loss.item() < .001 or loss.item() > 55: pass else: loss.backward() torch.nn.utils.clip_grad_norm_( self.model.parameters(), 0.5) self.embedding_optimizer.step() else: if loss.item() < .001 or loss.item() > 55: pass else: loss.backward() torch.nn.utils.clip_grad_norm_( self.model.parameters(), 0.5) self.embedding_optimizer.step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 # second loop for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_j_numpy - phi_i_numpy if torch.cuda.is_available(): feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13)).cuda()) P = Variable( torch.Tensor([ self.distribution_epsilon, 1 - self.distribution_epsilon ]).cuda()) else: feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13))) P = Variable( torch.Tensor([ self.distribution_epsilon, 1 - self.distribution_epsilon ])) output = self.model.forward(feature_input) self.opt.zero_grad() self.embedding_optimizer.zero_grad() loss = loss_func.forward(P, output, self.alpha) if torch.isnan(loss): print(self.alpha, ' :nan occurred at iteration ', self.total_iterations, ' at', num_iterations_predict_task) if self.total_iterations > 7500: if loss.item() < .001 or loss.item() > 55: pass else: loss.backward() torch.nn.utils.clip_grad_norm_( self.model.parameters(), 0.5) self.embedding_optimizer.step() else: if loss.item() < .001 or loss.item() > 55: pass else: loss.backward() torch.nn.utils.clip_grad_norm_( self.model.parameters(), 0.5) self.embedding_optimizer.step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 self.embedding_list = store_embedding_back_bnn( self.model, self.embedding_list, which_schedule) self.total_loss_array.append(running_loss_predict_tasks / num_iterations_predict_task) num_iterations_predict_task = 0 running_loss_predict_tasks = 0 self.total_iterations += 1 if self.total_iterations > 25 and self.total_iterations % 50 == 1: print('total iterations is', self.total_iterations) print('total loss (average for each 40, averaged)', np.mean(self.total_loss_array[-40:])) if self.total_iterations > 0 and self.total_iterations % self.when_to_save == self.when_to_save - 1: self.save_trained_nets('bnn_small' + str(self.num_schedules)) if self.total_iterations > 5000 and np.mean( self.total_loss_array[-100:]) - np.mean( self.total_loss_array[-500:] ) < self.convergence_epsilon: training_done = True
def train(self): """ Trains BDT. Randomly samples a schedule and timestep within that schedule, produces training data using x_i - x_j and trains upon that. :return: """ threshold = .1 training_done = False loss_func = AlphaLoss() # variables to keep track of loss and number of tasks trained over running_loss_predict_tasks = 0 num_iterations_predict_task = 0 while not training_done: # sample a timestep before the cutoff for cross_validation rand_timestep_within_sched = np.random.randint( len(self.start_of_each_set_twenty)) set_of_twenty = self.start_of_each_set_twenty[ rand_timestep_within_sched] truth = self.Y[set_of_twenty] which_schedule = find_which_schedule_this_belongs_to( self.schedule_array, set_of_twenty) load_in_embedding(self.model, self.embedding_list, which_schedule) # find feature vector of true action taken phi_i_num = truth + set_of_twenty phi_i = self.X[phi_i_num] phi_i_numpy = np.asarray(phi_i) # iterate over pairwise comparisons for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: # if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_i_numpy - phi_j_numpy # label = add_noise_pairwise(label, self.noise_percentage) feature_input, P = transform_into_torch_vars( feature_input, self.distribution_epsilon, True, torch.cuda.is_available()) output = self.model(feature_input) loss = loss_func.forward(P, output, self.alpha) # NAN check (fix is in the bnn file) if torch.isnan(loss): print(self.alpha, ' :nan occurred at iteration ', self.total_iterations) # prepare optimizer, compute gradient, update params self.opt.zero_grad() self.embedding_optimizer.zero_grad() if loss.item() < .001 or loss.item() > 50: pass else: loss.backward() torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) self.opt.step() self.embedding_optimizer.step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_j_numpy - phi_i_numpy feature_input, P = transform_into_torch_vars( feature_input, self.distribution_epsilon, False, torch.cuda.is_available()) output = self.model(feature_input) loss = loss_func.forward(P, output, self.alpha) # if num_iterations_predict_task % 5 == 0: # print('loss is :', loss.item()) # clip any very high gradients # prepare optimizer, compute gradient, update params self.opt.zero_grad() self.embedding_optimizer.zero_grad() if loss.item() < .001 or loss.item() > 50: pass else: loss.backward() torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) self.opt.step() self.embedding_optimizer.step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 self.embedding_list = store_embedding_back(self.model, self.embedding_list, which_schedule) self.total_loss_array.append(running_loss_predict_tasks / num_iterations_predict_task) num_iterations_predict_task = 0 running_loss_predict_tasks = 0 self.total_iterations += 1 if self.total_iterations > 25 and self.total_iterations % 50 == 1: print('total iterations is', self.total_iterations) print('total loss (average for each 40, averaged)', np.mean(self.total_loss_array[-40:])) if self.total_iterations > 0 and self.total_iterations % self.when_to_save == self.when_to_save - 1: self.save_trained_nets('BDDT' + str(self.num_schedules)) self.evaluate_on_test_data() if self.total_iterations > 5000 and np.mean( self.total_loss_array[-100:]) - np.mean( self.total_loss_array[-500:]) < self.covergence_epsilon: training_done = True
def train(self): """ Trains NN. Randomly samples a schedule and timestep within that schedule, produces training data using x_i - x_j and trains upon that. :return: """ total_iterations = 0 convergence_epsilon = .01 when_to_save = 1000 distribution_epsilon = .0001 training_done = False total_loss_array = [] loss_func = AlphaLoss(.9) # variables to keep track of loss and number of tasks trained over while not training_done: # sample a timestep before the cutoff for cross_validation set_of_twenty = np.random.choice(self.start_of_each_set_twenty) which_schedule = find_which_schedule_this_belongs_to(self.schedule_array, set_of_twenty) # get actual task scheduled truth = self.Y[set_of_twenty] # choose cluster based on value produced by gmm augment_proba = self.label[which_schedule] # find feature vector of true action taken phi_i_num = truth + set_of_twenty phi_i = self.X[phi_i_num] phi_i_numpy = np.asarray(phi_i) running_loss_predict_tasks = 0 num_iterations_predict_task = 0 # iterate over pairwise comparisons for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: # if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_i_numpy - phi_j_numpy if torch.cuda.is_available(): feature_input = Variable(torch.Tensor(feature_input.reshape(1, 13)).cuda()) feature_input = torch.cat(feature_input, torch.Tensor(augment_proba)) P = Variable(torch.Tensor([1 - distribution_epsilon, distribution_epsilon]).cuda()) else: feature_input = Variable(torch.Tensor(feature_input.reshape(1, 13))) P = Variable(torch.Tensor([1 - distribution_epsilon, distribution_epsilon])) output = self.model.forward(feature_input) if torch.isnan(output[0][0]).item() == 1: print('hi') self.opt.zero_grad() loss = loss_func.forward(P, output) if torch.isnan(loss): print(self.alpha, ' :nan occurred at iteration ', total_iterations, ' at', num_iterations_predict_task) if loss.item() < .001 or loss.item() > 50: pass else: loss.backward() torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) self.opt.step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 # second loop for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_j_numpy - phi_i_numpy if torch.cuda.is_available(): feature_input = Variable(torch.Tensor(feature_input.reshape(1, 13)).cuda()) P = Variable(torch.Tensor([distribution_epsilon, 1 - distribution_epsilon]).cuda()) else: feature_input = Variable(torch.Tensor(feature_input.reshape(1, 13))) P = Variable(torch.Tensor([distribution_epsilon, 1 - distribution_epsilon])) output = self.model.forward(feature_input) if torch.isnan(output[0][0]).item() == 1: print('hi') self.opt.zero_grad() loss = loss_func.forward(P, output) if loss.item() < .001 or loss.item() > 50: pass else: loss.backward() torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) self.opt.step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 total_loss_array.append(running_loss_predict_tasks / num_iterations_predict_task) total_iterations += 1 if total_iterations % 50 == 49: print('total loss (average for each 40, averaged) at iteration ', total_iterations, ' is ', np.mean(total_loss_array[-40:])) if total_iterations % when_to_save == when_to_save - 1: self.save_trained_nets('gmm_nn_small' + str(self.num_schedules)) if total_iterations > 5000 and np.mean(total_loss_array[-100:]) - np.mean(total_loss_array[-500:]) < convergence_epsilon: training_done = True
def train(self): """ Trains BDT. Randomly samples a schedule and timestep within that schedule, produces training data using x_i - x_j and trains upon that. :return: """ # loss = nn.CrossEntropyLoss() sig = torch.nn.Sigmoid() training_done = False criterion = torch.nn.BCELoss() # variables to keep track of loss and number of tasks trained over while not training_done: # sample a timestep before the cutoff for cross_validation set_of_twenty = np.random.choice(self.start_of_each_set_twenty) truth = self.Y[set_of_twenty] which_schedule = find_which_schedule_this_belongs_to( self.schedule_array, set_of_twenty) self.model.set_bayesian_embedding( self.embedding_list[which_schedule]) # find feature vector of true action taken phi_i_num = truth + set_of_twenty phi_i = self.X[phi_i_num] phi_i_numpy = np.asarray(phi_i) running_loss_predict_tasks = 0 num_iterations_predict_task = 0 # iterate over pairwise comparisons for counter in range(set_of_twenty, set_of_twenty + 20): # positive counterfactuals if counter == phi_i_num: # if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_i_numpy - phi_j_numpy if torch.cuda.is_available(): feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13)).cuda()) label = Variable( torch.Tensor(torch.ones((1, 1))).cuda()) else: feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13))) label = Variable(torch.Tensor(torch.ones((1, 1)))) output = self.model.forward(feature_input) sig = torch.nn.Sigmoid() output = sig(output) self.opt.zero_grad() loss = criterion(output, label) loss.backward() # torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) self.opt.step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 # Negative counterfactuals for counter in range(set_of_twenty, set_of_twenty + 20): if counter == phi_i_num: continue else: phi_j = self.X[counter] phi_j_numpy = np.asarray(phi_j) feature_input = phi_j_numpy - phi_i_numpy if torch.cuda.is_available(): feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13)).cuda()) label = Variable( torch.Tensor(torch.zeros((1, 1))).cuda()) else: feature_input = Variable( torch.Tensor(feature_input.reshape(1, 13))) label = Variable(torch.Tensor(torch.zeros((1, 1)))) output = self.model.forward(feature_input) sig = torch.nn.Sigmoid() output = sig(output) loss = criterion(output, label) loss.backward() # torch.nn.utils.clip_grad_norm_(self.model.parameters(), 0.5) self.opt.step() running_loss_predict_tasks += loss.item() num_iterations_predict_task += 1 # add average loss to array # print(list(self.model.parameters())) self.total_loss_array.append(running_loss_predict_tasks / num_iterations_predict_task) self.embedding_list[which_schedule] = torch.Tensor( self.model.get_bayesian_embedding().detach().cpu().numpy() ) # very ugly self.total_iterations += 1 if self.total_iterations % 500 == 499: print( 'total loss (average for each 40, averaged) at iteration ', self.total_iterations, ' is ', np.mean(self.total_loss_array[-40:])) if self.total_iterations > 10000 and np.mean( self.total_loss_array[-100:]) - np.mean( self.total_loss_array[-500:]) < self.covergence_epsilon: training_done = True