def run_init(self, timestamp, base_program_name, hole_node_ind, graph, trainset, validset, train_config, device, verbose=False): assert isinstance(graph, ProgramGraph) log_and_print("Training root program ...") current = copy.deepcopy(graph.root_node) initial_score, losses, m = execute_and_train_with_full(base_program_name, hole_node_ind, current.program, validset, trainset, train_config, graph.output_type, graph.output_size, neural=True, device=device) log_and_print("Initial training complete. Score from program is {:.4f} \n".format(1 - initial_score)) if device == 'cpu': base_program = CPU_Unpickler(open("%s.p" % base_program_name, "rb")).load() else: base_program = pickle.load(open("%s.p" % base_program_name, "rb")) curr_level = 0 l = [] traverse(base_program.submodules,l) # pprint(l) curr_program = base_program.submodules change_key(base_program.submodules, [], hole_node_ind, current.program.submodules["program"]) new_prog = base_program return 1 - initial_score, new_prog, losses
def crossover_operation(listofprogramdicts): new_prog_dict = {} parents = random.choices(listofprogramdicts, k=2) prog_dict1, prog_dict2 = parents[0], parents[1] xover1 = random.randint(1, prog_dict1['depth']) xover2 = random.randint(2, prog_dict2['depth']) print('DEBUG: trying new crossover') new_program, crossed_over = self.crossover_helper( prog_dict1['program'], prog_dict2['program'], xover1, xover2) if crossed_over: log_and_print("Crossing over the following programs:") log_and_print( print_program(prog_dict1['program'], ignore_constants=True)) log_and_print( print_program(prog_dict2['program'], ignore_constants=True)) new_prog_dict["program"] = new_program new_prog_dict["struct_cost"], new_prog_dict[ "depth"] = graph.compute_program_cost(new_program) log_and_print("Result has structural cost {:.4f}:".format( new_prog_dict["struct_cost"])) log_and_print( print_program(new_prog_dict['program'], ignore_constants=True)) return new_prog_dict else: return None
def run(self, graph, trainset, validset, train_config, device, verbose=False): assert isinstance(graph, ProgramGraph) log_and_print("Training RNN baseline with {} LSTM units ...".format(graph.max_num_units)) current = copy.deepcopy(graph.root_node) score = execute_and_train(current.program, validset, trainset, train_config, graph.output_type, graph.output_size, neural=True, device=device) log_and_print("Score of RNN is {:.4f}".format(1-score)) return current.program
def add(self, item): assert len(item) == 3 assert isinstance(item[2], ProgramNode) self.prioqueue.append(item) if len(self.prioqueue) > self.capacity: # self.sort(tup_idx=0) popped_f_score, _, popped = self.pop(-1) log_and_print("POP {} with fscore {:.4f}".format( print_program(popped.program, ignore_constants=True), popped_f_score))
def run_train_longer(self, timestamp, base_program_name, hole_node_ind, graph, trainset, validset, train_config, device, verbose=False): assert isinstance(graph, ProgramGraph) log_and_print("Training root program ...") current = copy.deepcopy(graph.root_node) initial_score, program = execute_and_train_og(base_program_name, validset, trainset, train_config, graph.output_type, graph.output_size, neural=True, device=device) log_and_print("Re-training complete. Score from program is {:.4f} \n".format(1 - initial_score)) return [{ "program" : program, "score" : 1- initial_score, }]
def load_base_program(self): print("Loading %s" % self.base_program_name) if self.device == 'cpu': self.base_program = CPU_Unpickler(open("%s.p" % self.base_program_name, "rb")).load() else: self.base_program = pickle.load(open("%s.p" % self.base_program_name, "rb")) base_folder = os.path.dirname(self.base_program_name) data = self.base_program.submodules l = [] traverse(data,l) log_and_print(l) return l
def evaluate_final(self): if self.device == 'cpu': program = CPU_Unpickler(open(self.full_path, "rb").load()) else: program = pickle.load(open(self.full_path, "rb")) log_and_print(print_program(program, ignore_constants=True)) l = [] traverse(program.submodules,l) with torch.no_grad(): test_input, test_output = map(list, zip(*self.testset)) true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device) predicted_vals = self.process_batch(program, test_input, self.output_type, self.output_size, self.device) metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels) log_and_print("F1 score achieved is {:.4f}".format(1 - metric))
def evaluate(self): assert os.path.isfile(self.program_path) program = pickle.load(open(self.program_path, "rb")) with torch.no_grad(): test_input, test_output = map(list, zip(*self.testset)) true_vals = torch.flatten(torch.stack(test_output)).float().to( self.device) predicted_vals = self.process_batch(program, test_input, self.output_type, self.output_size, self.device) metric, additional_params = label_correctness( predicted_vals, true_vals, num_labels=self.num_labels) log_and_print("F1 score achieved is {:.4f}".format(1 - metric))
def run_near(self, model, num_iter): train_config = { 'lr': self.learning_rate, 'neural_epochs': self.neural_epochs, 'symbolic_epochs': self.symbolic_epochs, 'optimizer': optim.Adam, 'lossfxn': nn.CrossEntropyLoss(), #todo 'evalfxn': label_correctness, 'num_labels': self.num_labels } # Initialize program graph starting from trained NN program_graph = ProgramGraph(model, DSL_DICT, CUSTOM_EDGE_COSTS, self.input_type, self.output_type, self.input_size, self.output_size, self.max_num_units, self.min_num_units, self.max_num_children, self.max_depth, self.penalty, ite_beta=self.ite_beta) # Initialize algorithm algorithm = ASTAR_NEAR(frontier_capacity=self.frontier_capacity) best_programs = algorithm.run(program_graph, self.batched_trainset, self.validset, train_config, self.device) if self.algorithm == "rnn": # special case for RNN baseline best_program = best_programs else: # Print all best programs found log_and_print("\n") log_and_print("BEST programs found:") for item in best_programs: print_program_dict(item) best_program = best_programs[-1]["program"] # Save best program self.program_path = os.path.join(self.save_path, "program_%d.p" % num_iter) pickle.dump(best_program, open(self.program_path, "wb"))
def test_set_eval(program, testset, output_type, output_size, num_labels, device='cpu', verbose=False): log_and_print("\n") log_and_print("Evaluating program {} on TEST SET".format(print_program(program, ignore_constants=(not verbose)))) with torch.no_grad(): test_input, test_output = map(list, zip(*testset)) true_vals = torch.tensor(flatten_batch(test_output)).to(device) predicted_vals = process_batch(program, test_input, output_type, output_size, device) metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=num_labels) log_and_print("F1 score achieved is {:.4f}".format(1 - metric)) log_and_print("Additional performance parameters: {}\n".format(additional_params))
def evaluate_neurosymb(self, program): # if self.device == 'cpu': # program = CPU_Unpickler(open("neursym.p" % self.base_program_name, "rb")).load() # else: # program = pickle.load(open("neursym.p" % self.base_program_name, "rb")) # # program= CPU_Unpickler(open("%s.p" % self.base_program_name, "rb")).load() print(print_program(program, ignore_constants=True)) l = [] traverse(program.submodules,l) with torch.no_grad(): test_input, test_output = map(list, zip(*self.testset)) true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device) predicted_vals = self.process_batch(program, test_input, self.output_type, self.output_size, self.device) metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels) log_and_print("Test F1 score achieved is {:.4f}\n".format(1 - metric)) log_and_print(str(additional_params)) return 1- metric
def __init__(self, **kwargs): self.__dict__.update(kwargs) if torch.cuda.is_available(): self.device = 'cuda:0' else: self.device = 'cpu' # load input data self.train_data = np.load(self.train_data) self.test_data = np.load(self.test_data) self.valid_data = None self.train_labels = np.load(self.train_labels) self.test_labels = np.load(self.test_labels) self.valid_labels = None if self.valid_data is not None and self.valid_labels is not None: self.valid_data = np.load(self.valid_data) self.valid_labels = np.load(self.valid_labels) self.batched_trainset, self.validset, self.testset = prepare_datasets(self.train_data, self.valid_data, self.test_data, self.train_labels, self.valid_labels, self.test_labels, normalize=self.normalize, train_valid_split=self.train_valid_split, batch_size=self.batch_size) if self.device == 'cpu': self.base_program = CPU_Unpickler(open("%s.p" % self.base_program_name, "rb")).load() else: self.base_program = pickle.load(open("%s.p" % self.base_program_name, "rb")) data = self.base_program.submodules l = [] traverse(data,l) self.hole_node = l[self.hole_node_ind] #for near on subtree self.curr_iter = 0 self.program_path = None now = datetime.now() self.timestamp = str(datetime.timestamp(now)).split('.')[0] log_and_print(self.timestamp) self.evaluate()
def fix(self): self.real_base = 'results/mars_an_astar-near_1_1605057595/fullprogram' if self.device == 'cpu': base_program = CPU_Unpickler(open("%s.p" % self.real_base, "rb")).load() else: base_program = pickle.load(open("%s.p" % self.real_base, "rb")) if self.device == 'cpu': best_program = CPU_Unpickler(open("%s/subprogram.p" % self.base_program_name, "rb")).load() else: best_program = pickle.load(open("%s/subprogram.p" % self.base_program_name, "rb")) self.full_path = os.path.join(self.base_program_name, "fullprogram.p") #fix this with torch.no_grad(): test_input, test_output = map(list, zip(*self.testset)) true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device) predicted_vals = self.process_batch(base_program, test_input, self.output_type, self.output_size, self.device) metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels) log_and_print("F1 score achieved is {:.4f}".format(1 - metric)) curr_level = 0 l = [] traverse(base_program.submodules,l) curr_program = base_program.submodules change_key(base_program.submodules, [], self.hole_node_ind, best_program.submodules["program"]) with torch.no_grad(): test_input, test_output = map(list, zip(*self.testset)) true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device) predicted_vals = self.process_batch(base_program, test_input, self.output_type, self.output_size, self.device) metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels) log_and_print("F1 score achieved is {:.4f}".format(1 - metric)) log_and_print(str(additional_params)) pickle.dump(base_program, open(self.full_path, "wb"))
def mutation_helper(self, graph, program, total_depth, mutation_point, max_enumeration_depth=6): pointer = 1 queue = [program] while len(queue) != 0: current_function = queue.pop() for submodule, functionclass in current_function.submodules.items( ): if pointer == mutation_point: log_and_print( "Generating replacements for mutation program at depth {} and typesig {}." .format(total_depth - mutation_point, (functionclass.input_type, functionclass.output_type))) enumeration_depth = min(max_enumeration_depth, total_depth - mutation_point) all_replacements = self.enumerative_synthesis( graph, enumeration_depth, (functionclass.input_type, functionclass.output_type), input_size=functionclass.input_size, output_size=functionclass.output_size) if len(all_replacements) == 0: return program else: mutated_replacement = random.choice(all_replacements) current_function.submodules[submodule] = copy.deepcopy( mutated_replacement['program']. submodules['program']) self.reset_program_sizes(program) log_and_print("Mutation completed.") return program pointer += 1 queue.append(functionclass) return program
def execute_and_train_with_full(base_program_name, hole_node_ind, program, validset, trainset, train_config, output_type, output_size, neural=False, device='cpu', use_valid_score=False, print_every=60): #load program # pprint(type(hole_node)) # level_to_replace = hole_node[1] if device == 'cpu': base_program = CPU_Unpickler(open("%s.p" % base_program_name, "rb")).load() else: base_program = pickle.load(open("%s.p" % base_program_name, "rb")) curr_level = 0 l = [] traverse(base_program.submodules, l) # pprint(l) curr_program = base_program.submodules # print(program) # print(program.submodules) # pprint change_key( base_program.submodules, [], hole_node_ind, program.submodules["program"]) #should we just replace with program? log_and_print(print_program(base_program)) # pickle.dump(base_program, open("neursym.p", "wb")) return execute_and_train(base_program, program, validset, trainset, train_config, output_type, output_size, neural, device)
def evaluate(self): # assert os.path.isfile(self.program_path) base_program= CPU_Unpickler(open("%s.p" % self.base_program_name, "rb")).load() program_baby = CPU_Unpickler(open("%s.p" % self.baby_program_name, "rb")).load() data = base_program.submodules l = [] traverse(data,l) # print(l) hole_node = l[self.hole_node_ind] #conditoin node # print(hole_node) change_key(base_program.submodules, hole_node[0], program_baby, hole_node[1]) # pickle.dump(program, open("ite_1603639887.p", "wb")) base_output_type = base_program.program.output_type base_output_size = base_program.program.output_size # program = pickle.load(open(self.program_path, "rb")) with torch.no_grad(): test_input, test_output = map(list, zip(*self.testset)) true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device) predicted_vals = self.process_batch(base_program, test_input, base_output_type, base_output_size, self.device) metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels) log_and_print("F1 score achieved is {:.4f}".format(1 - metric))
def neural_h(self): data = self.base_program.submodules l = [] #populate AST traverse(data,l) train_config = { 'lr' : self.learning_rate, 'neural_epochs' : 15, 'symbolic_epochs' : self.symbolic_epochs, 'optimizer' : optim.Adam, 'lossfxn' : nn.CrossEntropyLoss(weight=self.loss_weight), #todo 'evalfxn' : label_correctness, 'num_labels' : self.num_labels } best_node_ind = 0 best_score = 0 for hole_node_ind in range(1,len(l)): hole_node = l[hole_node_ind] subprogram_str = print_program(hole_node[0]) if subprogram_str.count('(') > self.max_depth: continue near_input_type = hole_node[0].input_type near_output_type = hole_node[0].output_type near_input_size = hole_node[0].input_size near_output_size = hole_node[0].output_size # Initialize program graph starting from trained NN program_graph = ProgramGraph(DSL_DICT, CUSTOM_EDGE_COSTS, near_input_type, near_output_type, near_input_size, near_output_size, self.max_num_units, self.min_num_units, self.max_num_children, 0, self.penalty, ite_beta=self.ite_beta) ## max_depth 0 # Initialize algorithm algorithm = ASTAR_NEAR(frontier_capacity=0) score, new_prog, losses = algorithm.run_init(self.timestamp, self.base_program_name, hole_node_ind, program_graph, self.batched_trainset, self.validset, train_config, self.device) subprogram_str = print_program(hole_node[0]) log_and_print("Subprogram to replace: %s"% subprogram_str) if score > best_score: best_node_ind = hole_node_ind best_score = score log_and_print("New best: RNN Heuristic score at Node %d: %f\n" %( hole_node_ind, score)) else: log_and_print("RNN Heuristic score at Node %d: %f\n" %( hole_node_ind, score)) return best_node_ind
def train_more_epochs(self,program_to_train): log_and_print("starting training more epochs") train_config = { 'lr' : self.learning_rate, 'neural_epochs' : 20, 'symbolic_epochs' : 20, 'optimizer' : optim.Adam, 'lossfxn' : nn.CrossEntropyLoss(weight=self.loss_weight), #todo 'evalfxn' : label_correctness, 'num_labels' : self.num_labels } near_input_type = self.base_program.input_type near_output_type = self.base_program.output_type near_input_size = self.base_program.input_size near_output_size = self.base_program.output_size # Initialize program graph starting from trained NN program_graph = ProgramGraph(DSL_DICT, CUSTOM_EDGE_COSTS, near_input_type, near_output_type, near_input_size, near_output_size, self.max_num_units, self.min_num_units, self.max_num_children, self.max_depth, self.penalty, ite_beta=self.ite_beta) # Initialize algorithm algorithm = ASTAR_NEAR(frontier_capacity=self.frontier_capacity) best_programs = algorithm.run_train_longer(self.timestamp, program_to_train, self.hole_node_ind, program_graph, self.batched_trainset, self.validset, train_config, self.device) best_program_str = [] # Print all best programs found log_and_print("\n") # log_and_print("BEST programs found:") for item in best_programs: program_struct = print_program(item["program"], ignore_constants=True) program_info = " score {:.4f} ".format(item["score"]) best_program_str.append((program_struct, program_info)) print(best_program_str) # print_program_dict(item) best_program = best_programs[-1]["program"] with torch.no_grad(): test_input, test_output = map(list, zip(*self.testset)) true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device) predicted_vals = self.process_batch(best_program, test_input, self.output_type, self.output_size, self.device) metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels) pickle.dump(best_program, open(program_to_train + ".p", "wb")) #overfit?? log_and_print("end training more epochs")
def run(self, graph, trainset, validset, train_config, device, verbose=False): assert isinstance(graph, ProgramGraph) current = copy.deepcopy(graph.root_node) current_avg_f_score = float('inf') best_program = None best_total_cost = float('inf') best_programs_list = [] num_children_trained = 0 start_time = time.time() while not graph.is_fully_symbolic(current.program): log_and_print("CURRENT program has avg fscore {:.4f}: {}".format( current_avg_f_score, print_program(current.program, ignore_constants=(not verbose)))) children = graph.get_all_children(current, in_enumeration=True) children_mapping = { print_program(child.program, ignore_constants=True) : child for child in children } children_scores = { key : [] for key in children_mapping.keys() } costs = [child.cost for child in children] for i in range(self.num_mc_samples): child = random.choices(children, weights=costs)[0] sample = self.mc_sample(graph, child) assert graph.is_fully_symbolic(sample.program) log_and_print("Training sample program: {}".format(print_program(sample.program, ignore_constants=(not verbose)))) sample_score = execute_and_train(sample.program, validset, trainset, train_config, graph.output_type, graph.output_size, neural=False, device=device) num_children_trained += 1 log_and_print("{} total children trained".format(num_children_trained)) sample_f_score = sample.cost + sample_score children_scores[print_program(child.program, ignore_constants=True)].append(sample_f_score) if sample_f_score < best_total_cost: best_program = copy.deepcopy(sample.program) best_total_cost = sample_f_score best_programs_list.append({ "program" : best_program, "struct_cost" : sample.cost, "score" : sample_score, "path_cost" : sample_f_score, "time" : time.time()-start_time }) log_and_print("New BEST program found:") print_program_dict(best_programs_list[-1]) # (Naive) selection operation children_scores = { key : sum(val)/len(val) if len(val) > 0 else float('inf') for key,val in children_scores.items() } best_child_name = min(children_scores, key=children_scores.get) current = children_mapping[best_child_name] current_avg_f_score = children_scores[best_child_name] for key,val in children_scores.items(): log_and_print("Avg score {:.4f} for child {}".format(val,key)) log_and_print("SELECTING {} as best child node\n".format(best_child_name)) log_and_print("DEBUG: time since start is {:.3f}\n".format(time.time()-start_time)) return best_programs_list
def __init__(self, **kwargs): self.__dict__.update(kwargs) if torch.cuda.is_available(): self.device = 'cuda:0' print(self.device) else: self.device = 'cpu' self.loss_weight = torch.tensor([float(w) for w in self.class_weights.split(',')]).to(self.device) if self.exp_name == 'crim13': # load input data self.train_data = np.load(self.train_data) self.test_data = np.load(self.test_data) self.valid_data = None self.train_labels = np.load(self.train_labels) self.test_labels = np.load(self.test_labels) self.valid_labels = None assert self.train_data.shape[-1] == self.test_data.shape[-1] == self.input_size if self.valid_data is not None and self.valid_labels is not None: self.valid_data = np.load(self.valid_data) self.valid_labels = np.load(self.valid_labels) assert valid_data.shape[-1] == self.input_size self.batched_trainset, self.validset, self.testset = prepare_datasets(self.train_data, self.valid_data, self.test_data, self.train_labels, self.valid_labels, self.test_labels, normalize=self.normalize, train_valid_split=self.train_valid_split, batch_size=self.batch_size) elif self.exp_name == 'mars_an': #### start mars train_datasets = self.train_data.split(",") train_raw_features = [] train_raw_annotations = [] for fname in train_datasets: data = np.load(fname, allow_pickle=True) train_raw_features.extend(data["features"]) train_raw_annotations.extend(data["annotations"]) test_data = np.load(self.test_data, allow_pickle=True) test_raw_features = test_data["features"] test_raw_annotations = test_data["annotations"] valid_raw_features = None valid_raw_annotations = None valid_labels = None # Check the # of features of the first frame of the first video assert len(train_raw_features[0][0]) == len(test_raw_features[0][0]) == self.input_size if self.valid_data is not None: valid_data = np.load(self.valid_data, allow_pickle=True) valid_raw_features = valid_data["features"] valid_raw_annotations = valid_data["annotations"] assert len(valid_raw_features[0][0]) == self.input_size behave_dict = read_into_dict('../near_code_7keypoints/data/MARS_data/behavior_assignments_3class.txt') # Reshape the data to trajectories of length 100 train_features, train_labels = preprocess(train_raw_features, train_raw_annotations, self.train_labels, behave_dict) test_features, test_labels = preprocess(test_raw_features, test_raw_annotations, self.train_labels, behave_dict) if valid_raw_features is not None and valid_raw_annotations is not None: valid_features, valid_labels = preprocess(valid_raw_features, valid_raw_annotations, self.train_labels, behave_dict) self.batched_trainset, self.validset, self.testset = prepare_datasets(train_features, valid_features, test_features, train_labels, valid_labels, test_labels, normalize=self.normalize, train_valid_split=self.train_valid_split, batch_size=self.batch_size) ##### END MARS else: log_and_print('bad experiment name') return # self.fix() # add subprogram in # if self.device == 'cpu': # self.base_program = CPU_Unpickler(open("%s/subprogram.p" % self.base_program_name, "rb")).load() # else: # self.base_program = pickle.load(open("%s/subprogram.p" % self.base_program_name, "rb")) if self.device == 'cpu': self.base_program = CPU_Unpickler(open("%s.p" % self.base_program_name, "rb")).load() else: self.base_program = pickle.load(open("%s.p" % self.base_program_name, "rb")) base_folder = os.path.dirname(self.base_program_name) # self.weights_dict = np.load(os.path.join(base_folder,'weights.npy'), allow_pickle=True).item() data = self.base_program.submodules l = [] traverse(data,l) log_and_print(l) # if self.hole_node_ind < 0: # self.hole_node_ind = len(l) + self.hole_node_ind #if negative, make it positive self.hole_node_ind %= len(l) self.hole_node = l[self.hole_node_ind] #for near on subtree self.curr_iter = 0 self.program_path = None if self.exp_id is not None: self.trial = self.exp_id if self.eval: self.evaluate() else: now = datetime.now() self.timestamp = str(datetime.timestamp(now)).split('.')[0][4:] log_and_print(self.timestamp) full_exp_name = "{}_{}_{}_{}".format( self.exp_name, self.algorithm, self.trial, self.timestamp) #unique timestamp for each near run self.save_path = os.path.join(self.save_dir, full_exp_name) if not os.path.exists(self.save_path): os.makedirs(self.save_path) init_logging(self.save_path) if self.neurh: log_and_print(self.base_program_name) self.neural_h() else: self.run_near() self.evaluate_final()
def run(self, graph, trainset, validset, train_config, device, verbose=False): assert isinstance(graph, ProgramGraph) log_and_print("Training root program ...") current = copy.deepcopy(graph.root_node) initial_score = execute_and_train(current.program, validset, trainset, train_config, graph.output_type, graph.output_size, neural=True, device=device) log_and_print( "Initial training complete. Score from program is {:.4f} \n". format(1 - initial_score)) # Branch-and-bound search with iterative deepening current_depth = self.initial_depth current_f_score = float('inf') order = 0 frontier = ProgramNodeFrontier(capacity=self.frontier_capacity) next_frontier = ProgramNodeFrontier(capacity=self.frontier_capacity) num_children_trained = 0 start_time = time.time() best_program = None best_total_cost = float('inf') best_programs_list = [] log_and_print("Starting iterative deepening with depth {}\n".format( current_depth)) while current_depth <= graph.max_depth: log_and_print("CURRENT program has fscore {:.4f}: {}".format( current_f_score, print_program(current.program, ignore_constants=(not verbose)))) log_and_print("Current depth of program is {}".format( current.depth)) log_and_print("Creating children for current node/program") log_and_print("Total time elapsed is {:.3f}".format(time.time() - start_time)) children_nodes = graph.get_all_children(current) # prune if more than self.max_num_children if len(children_nodes) > graph.max_num_children: log_and_print("Sampling {}/{} children".format( graph.max_num_children, len(children_nodes))) children_nodes = random.sample( children_nodes, k=graph.max_num_children) # sample without replacement log_and_print("{} total children to train for current node".format( len(children_nodes))) child_tuples = [] for child_node in children_nodes: child_start_time = time.time() log_and_print("Training child program: {}".format( print_program(child_node.program, ignore_constants=(not verbose)))) is_neural = not graph.is_fully_symbolic(child_node.program) child_node.score = execute_and_train(child_node.program, validset, trainset, train_config, graph.output_type, graph.output_size, neural=is_neural, device=device) log_and_print( "Time to train child {:.3f}".format(time.time() - child_start_time)) num_children_trained += 1 log_and_print( "{} total children trained".format(num_children_trained)) child_node.parent = current child_node.children = [] order -= 1 child_node.order = order # insert order of exploration as tiebreaker for equivalent f-scores # computing path costs (f_scores) child_f_score = child_node.cost + child_node.score # cost + heuristic log_and_print("DEBUG: f-score {}".format(child_f_score)) current.children.append(child_node) child_tuples.append((child_f_score, order, child_node)) if not is_neural and child_f_score < best_total_cost: best_program = copy.deepcopy(child_node.program) best_total_cost = child_f_score best_programs_list.append({ "program": best_program, "struct_cost": child_node.cost, "score": child_node.score, "path_cost": child_f_score, "time": time.time() - start_time }) log_and_print("New BEST program found:") print_program_dict(best_programs_list[-1]) # find next current among children, from best to worst nextfound = False child_tuples.sort(key=lambda x: x[0]) for child_tuple in child_tuples: child = child_tuple[2] if graph.is_fully_symbolic(child.program): continue # don't want to expand symbolic programs (no children) elif child.depth >= current_depth: next_frontier.add(child_tuple) else: if not nextfound: nextfound = True # first child program that's not symbolic and within current_depth current_f_score, current_order, current = child_tuple log_and_print( "Found program among children: {} with f_score {}". format( print_program(current.program, ignore_constants=(not verbose)), current_f_score)) else: frontier.add(child_tuple) # put the rest onto frontier # find next node in frontier if not nextfound: frontier.sort(tup_idx=1) # DFS order log_and_print("Frontier length is: {}".format(len(frontier))) original_depth = current.depth while len(frontier) > 0 and not nextfound: current_f_score, current_order, current = frontier.pop( 0, sort_fscores=False) # DFS order if current_f_score < self.bound_modify( best_total_cost, original_depth, current.depth): nextfound = True log_and_print( "Found program in frontier: {} with f_score {}". format( print_program(current.program, ignore_constants=(not verbose)), current_f_score)) else: log_and_print( "PRUNE from frontier: {} with f_score {}".format( print_program(current.program, ignore_constants=(not verbose)), current_f_score)) log_and_print("Frontier length is now {}".format( len(frontier))) # frontier is empty, go to next stage of iterative deepening if not nextfound: assert len(frontier) == 0 log_and_print("Empty frontier, moving to next depth level") log_and_print( "DEBUG: time since start is {:.3f}\n".format(time.time() - start_time)) current_depth += 1 if current_depth > graph.max_depth: log_and_print("Max depth {} reached. Exiting.\n".format( graph.max_depth)) break elif len(next_frontier) == 0: log_and_print("Next frontier is empty. Exiting.\n") break else: log_and_print( "Starting iterative deepening with depth {}\n".format( current_depth)) frontier = copy.deepcopy(next_frontier) next_frontier = ProgramNodeFrontier( capacity=self.frontier_capacity) current_f_score, current_order, current = frontier.pop(0) if best_program is None: log_and_print("ERROR: no program found") return best_programs_list
if device != 'cpu': lossfxn = lossfxn.cuda() train_config = { 'lr': args.learning_rate, 'neural_epochs': args.neural_epochs, 'symbolic_epochs': args.symbolic_epochs, 'optimizer': optim.Adam, 'lossfxn': lossfxn, 'evalfxn': label_correctness, 'num_labels': args.num_labels } # Initialize logging init_logging(save_path) log_and_print("Starting experiment {}\n".format(full_exp_name)) # Initialize program graph program_graph = ProgramGraph(DSL_DICT, CUSTOM_EDGE_COSTS, args.input_type, args.output_type, args.input_size, args.output_size, args.max_num_units, args.min_num_units, args.max_num_children, args.max_depth, args.penalty, ite_beta=args.ite_beta)
def run(self, graph, trainset, validset, train_config, device, verbose=False, trainset_neural=None): assert isinstance(graph, ProgramGraph) log_and_print("Training root program ...") current = copy.deepcopy(graph.root_node) dataset = trainset_neural if trainset_neural is not None else trainset initial_score = execute_and_train(current.program, validset, dataset, train_config, graph.output_type, graph.output_size, neural=True, device=device) log_and_print( "Initial training complete. Score from program is {:.4f} \n". format(1 - initial_score)) order = 0 frontier = ProgramNodeFrontier(capacity=self.frontier_capacity) frontier.add((float('inf'), order, current)) num_children_trained = 0 start_time = time.time() best_program = None best_total_cost = float('inf') best_programs_list = [] while len(frontier) != 0: current_f_score, _, current = frontier.pop(0) log_and_print("CURRENT program has fscore {:.4f}: {}".format( current_f_score, print_program(current.program, ignore_constants=(not verbose)))) log_and_print("Current depth of program is {}".format( current.depth)) log_and_print("Creating children for current node/program") children_nodes = graph.get_all_children(current) # prune if more than self.max_num_children if len(children_nodes) > graph.max_num_children: children_nodes = random.sample( children_nodes, k=graph.max_num_children) # sample without replacement log_and_print("{} total children to train for current node".format( len(children_nodes))) for child_node in children_nodes: child_start_time = time.time() log_and_print("Training child program: {}".format( print_program(child_node.program, ignore_constants=(not verbose)))) is_neural = not graph.is_fully_symbolic(child_node.program) dataset = trainset_neural if ( is_neural and trainset_neural is not None) else trainset child_node.score = execute_and_train(child_node.program, validset, dataset, train_config, graph.output_type, graph.output_size, neural=is_neural, device=device) log_and_print( "Time to train child {:.3f}".format(time.time() - child_start_time)) num_children_trained += 1 log_and_print( "{} total children trained".format(num_children_trained)) child_node.parent = current child_node.children = [] order -= 1 child_node.order = order # insert order of exploration as tiebreaker for equivalent f-scores current.children.append(child_node) # computing path costs (f_scores) child_f_score = child_node.cost + child_node.score # cost + heuristic log_and_print("DEBUG: f-score {}".format(child_f_score)) if not is_neural and child_f_score < best_total_cost: best_program = copy.deepcopy(child_node.program) best_total_cost = child_f_score best_programs_list.append({ "program": best_program, "struct_cost": child_node.cost, "score": child_node.score, "path_cost": child_f_score, "time": time.time() - start_time }) log_and_print("New BEST program found:") print_program_dict(best_programs_list[-1]) if is_neural: assert child_node.depth < graph.max_depth child_tuple = (child_f_score, order, child_node) frontier.add(child_tuple) # clean up frontier frontier.sort(tup_idx=0) while len(frontier) > 0 and frontier.peek(-1)[0] > best_total_cost: frontier.pop(-1) log_and_print("Frontier length is: {}".format(len(frontier))) log_and_print("Total time elapsed is {:.3f}".format(time.time() - start_time)) if best_program is None: log_and_print("ERROR: no program found") return best_programs_list
def __init__(self, **kwargs): self.__dict__.update(kwargs) if torch.cuda.is_available(): self.device = 'cuda:0' print(self.device) else: self.device = 'cpu' self.loss_weight = torch.tensor([float(w) for w in self.class_weights.split(',')]).to(self.device) if self.exp_name == 'crim13' or 'bball' in self.exp_name: # load input data self.train_data = np.load(self.train_data) self.test_data = np.load(self.test_data) self.valid_data = None self.train_labels = np.load(self.train_labels) self.test_labels = np.load(self.test_labels) self.valid_labels = None assert self.train_data.shape[-1] == self.test_data.shape[-1] == self.input_size if self.valid_data is not None and self.valid_labels is not None: self.valid_data = np.load(self.valid_data) self.valid_labels = np.load(self.valid_labels) assert valid_data.shape[-1] == self.input_size self.batched_trainset, self.validset, self.testset = prepare_datasets(self.train_data, self.valid_data, self.test_data, self.train_labels, self.valid_labels, self.test_labels, normalize=self.normalize, train_valid_split=self.train_valid_split, batch_size=self.batch_size) elif self.exp_name == 'mars_an': #### start mars train_datasets = self.train_data.split(",") train_raw_features = [] train_raw_annotations = [] for fname in train_datasets: data = np.load(fname, allow_pickle=True) train_raw_features.extend(data["features"]) train_raw_annotations.extend(data["annotations"]) test_data = np.load(self.test_data, allow_pickle=True) test_raw_features = test_data["features"] test_raw_annotations = test_data["annotations"] valid_raw_features = None valid_raw_annotations = None valid_labels = None # Check the # of features of the first frame of the first video assert len(train_raw_features[0][0]) == len(test_raw_features[0][0]) == self.input_size if self.valid_data is not None: valid_data = np.load(self.valid_data, allow_pickle=True) valid_raw_features = valid_data["features"] valid_raw_annotations = valid_data["annotations"] assert len(valid_raw_features[0][0]) == self.input_size behave_dict = read_into_dict('../near_code_7keypoints/data/MARS_data/behavior_assignments_3class.txt') # Reshape the data to trajectories of length 100 train_features, train_labels = preprocess(train_raw_features, train_raw_annotations, self.train_labels, behave_dict) test_features, test_labels = preprocess(test_raw_features, test_raw_annotations, self.train_labels, behave_dict) if valid_raw_features is not None and valid_raw_annotations is not None: valid_features, valid_labels = preprocess(valid_raw_features, valid_raw_annotations, self.train_labels, behave_dict) self.batched_trainset, self.validset, self.testset = prepare_datasets(train_features, valid_features, test_features, train_labels, valid_labels, test_labels, normalize=self.normalize, train_valid_split=self.train_valid_split, batch_size=self.batch_size) ##### END MARS else: log_and_print('bad experiment name') return now = datetime.now() self.timestamp = str(datetime.timestamp(now)).split('.')[0][4:] log_and_print(self.timestamp) full_exp_name = "{}_{}_{}_{}".format( self.exp_name, self.algorithm, self.trial, self.timestamp) #unique timestamp for each near run self.save_path = os.path.join(self.save_dir, full_exp_name) if not os.path.exists(self.save_path): os.makedirs(self.save_path) init_logging(self.save_path) visited_nodes = set() #dont visit smth thats already been visited num_iter = 10 #todo make this a parameter later for i in range(num_iter): l = self.load_base_program() #populates self.base_program log_and_print("Base program performance:") self.evaluate_final() self.hole_node_ind = self.neural_h() self.hole_node = l[self.hole_node_ind] log_and_print("Node selected: %d" % self.hole_node_ind) # set up path to save program # self.save_path = os.path.join(self.save_path, str(num_iter)) #run near self.run_near(i) #change base program name self.base_program_name = os.path.join(self.save_path, "fullprogram_%d" % i) #make it rly good self.train_more_epochs(self.base_program_name)
def run_near(self): # print(self.device) train_config = { 'lr' : self.learning_rate, 'neural_epochs' : self.neural_epochs, 'symbolic_epochs' : self.symbolic_epochs, 'optimizer' : optim.Adam, 'lossfxn' : nn.CrossEntropyLoss(weight=self.loss_weight), #todo 'evalfxn' : label_correctness, 'num_labels' : self.num_labels } near_input_type = self.hole_node[0].input_type near_output_type = self.hole_node[0].output_type near_input_size = self.hole_node[0].input_size near_output_size = self.hole_node[0].output_size # Initialize program graph starting from trained NN program_graph = ProgramGraph(DSL_DICT, CUSTOM_EDGE_COSTS, near_input_type, near_output_type, near_input_size, near_output_size, self.max_num_units, self.min_num_units, self.max_num_children, self.max_depth, self.penalty, ite_beta=self.ite_beta) # Initialize algorithm algorithm = ASTAR_NEAR(frontier_capacity=self.frontier_capacity) best_programs = algorithm.run(self.timestamp, self.base_program_name, self.hole_node_ind, program_graph, self.batched_trainset, self.validset, train_config, self.device) best_program_str = [] if self.algorithm == "rnn": # special case for RNN baseline best_program = best_programs else: # Print all best programs found log_and_print("\n") log_and_print("BEST programs found:") for item in best_programs: program_struct = print_program(item["program"], ignore_constants=True) program_info = "struct_cost {:.4f} | score {:.4f} | path_cost {:.4f} | time {:.4f}".format( item["struct_cost"], item["score"], item["path_cost"], item["time"]) best_program_str.append((program_struct, program_info)) print_program_dict(item) best_program = best_programs[-1]["program"] # Save best programs f = open(os.path.join(self.save_path, "best_programs.txt"),"w") f.write( str(best_program_str) ) f.close() self.program_path = os.path.join(self.save_path, "subprogram.p") pickle.dump(best_program, open(self.program_path, "wb")) self.full_path = os.path.join(self.save_path, "fullprogram.p") if self.device == 'cpu': base_program = CPU_Unpickler(open("%s.p" % self.base_program_name, "rb")).load() else: base_program = pickle.load(open("%s.p" % self.base_program_name, "rb")) curr_level = 0 l = [] traverse(base_program.submodules,l) curr_program = base_program.submodules change_key(base_program.submodules, [], self.hole_node_ind, best_program.submodules["program"]) pickle.dump(base_program, open(self.full_path, "wb")) # Save parameters f = open(os.path.join(self.save_path, "parameters.txt"),"w") parameters = ['input_type', 'output_type', 'input_size', 'output_size', 'num_labels', 'neural_units', 'max_num_units', 'min_num_units', 'max_num_children', 'max_depth', 'penalty', 'ite_beta', 'train_valid_split', 'normalize', 'batch_size', 'learning_rate', 'neural_epochs', 'symbolic_epochs', 'lossfxn', 'class_weights', 'num_iter', 'num_f_epochs', 'algorithm', 'frontier_capacity', 'initial_depth', 'performance_multiplier', 'depth_bias', 'exponent_bias', 'num_mc_samples', 'max_num_programs', 'population_size', 'selection_size', 'num_gens', 'total_eval', 'mutation_prob', 'max_enum_depth', 'exp_id', 'base_program_name', 'hole_node_ind'] for p in parameters: f.write( p + ': ' + str(self.__dict__[p]) + '\n' ) f.close()
def run(self, graph, trainset, validset, train_config, device, verbose=False): assert isinstance(graph, ProgramGraph) symbolic_programs = [] enum_depth = 1 while len(symbolic_programs) < self.max_num_programs: print("DEBUG: starting enumerative synthesis with depth {}".format(enum_depth)) symbolic_programs = self.enumerate2depth(graph, enum_depth) print("DEBUG: {} programs found".format(len(symbolic_programs))) enum_depth += 1 if enum_depth > graph.max_depth: break log_and_print("Symbolic Synthesis: generated {}/{} symbolic programs from candidate program.".format( len(symbolic_programs), self.max_num_programs)) total_eval = min(self.max_num_programs, len(symbolic_programs)) symbolic_programs.sort(key=lambda x: x["struct_cost"]) symbolic_programs = symbolic_programs[:total_eval] best_program = None best_total_cost = float('inf') best_programs_list = [] start_time = time.time() num_programs_trained = 1 for prog_dict in symbolic_programs: child_start_time = time.time() candidate = prog_dict["program"] log_and_print("Training candidate program ({}/{}) {}".format( num_programs_trained, total_eval, print_program(candidate, ignore_constants=(not verbose)))) num_programs_trained += 1 score = execute_and_train(candidate, validset, trainset, train_config, graph.output_type, graph.output_size, neural=False, device=device) total_cost = score + prog_dict["struct_cost"] log_and_print("Structural cost is {} with structural penalty {}".format(prog_dict["struct_cost"], graph.penalty)) log_and_print("Time to train child {:.3f}".format(time.time()-child_start_time)) log_and_print("Total time elapsed is: {:.3f}".format(time.time()-start_time)) if total_cost < best_total_cost: best_program = copy.deepcopy(prog_dict["program"]) best_total_cost = total_cost prog_dict["score"] = score prog_dict["path_cost"] = total_cost prog_dict["time"] = time.time()-start_time best_programs_list.append(prog_dict) log_and_print("New BEST program found:") print_program_dict(best_programs_list[-1]) return best_programs_list
def run(self, graph, trainset, validset, train_config, device, verbose=False): assert isinstance(graph, ProgramGraph) self.input_size = graph.input_size self.output_size = graph.output_size best_program = None best_score = float('inf') best_total = float('inf') best_programs_list = [] start_time = time.time() log_and_print("Generating all programs up to specified depth.") all_generated_programs = self.enumerative_synthesis( graph, self.max_enum_depth) random.shuffle(all_generated_programs) current_population = all_generated_programs[:self.population_size] total_trained_programs = 0 def crossover_operation(listofprogramdicts): new_prog_dict = {} parents = random.choices(listofprogramdicts, k=2) prog_dict1, prog_dict2 = parents[0], parents[1] xover1 = random.randint(1, prog_dict1['depth']) xover2 = random.randint(2, prog_dict2['depth']) print('DEBUG: trying new crossover') new_program, crossed_over = self.crossover_helper( prog_dict1['program'], prog_dict2['program'], xover1, xover2) if crossed_over: log_and_print("Crossing over the following programs:") log_and_print( print_program(prog_dict1['program'], ignore_constants=True)) log_and_print( print_program(prog_dict2['program'], ignore_constants=True)) new_prog_dict["program"] = new_program new_prog_dict["struct_cost"], new_prog_dict[ "depth"] = graph.compute_program_cost(new_program) log_and_print("Result has structural cost {:.4f}:".format( new_prog_dict["struct_cost"])) log_and_print( print_program(new_prog_dict['program'], ignore_constants=True)) return new_prog_dict else: return None def mutation_operation(mod_prog_dict): new_prog_dict = {} mutation_point = random.randrange(1, mod_prog_dict['depth']) new_prog_dict['program'] = self.mutation_helper( graph, mod_prog_dict['program'], mod_prog_dict['depth'], mutation_point, max_enumeration_depth=self.max_enum_depth) new_prog_dict["struct_cost"], new_prog_dict[ "depth"] = graph.compute_program_cost(new_prog_dict['program']) return new_prog_dict for gen_idx in range(self.num_gens): #evaluation operation: train each program and evaluate it. log_and_print( "Training generation {}'s population of programs.".format( gen_idx + 1)) for programdict in current_population: total_trained_programs += 1 child_start_time = time.time() log_and_print("Training candidate program ({}/{}) {}".format( total_trained_programs, self.total_eval, print_program(programdict['program'], ignore_constants=(not verbose)))) score = execute_and_train(programdict['program'], validset, trainset, train_config, graph.output_type, graph.output_size, neural=False, device=device) log_and_print( "Structural cost is {} with structural penalty {}".format( programdict["struct_cost"], graph.penalty)) log_and_print( "Time to train child {:.3f}".format(time.time() - child_start_time)) log_and_print("Total time elapsed is: {}".format(time.time() - start_time)) programdict["score"] = score programdict["path_cost"] = score + programdict["struct_cost"] programdict["time"] = time.time() - start_time if programdict["path_cost"] < best_total: best_program = copy.deepcopy(programdict["program"]) best_total = score + programdict["struct_cost"] best_score = score best_cost = programdict["struct_cost"] best_programs_list.append(copy.deepcopy(programdict)) log_and_print("New BEST program found:") print_program_dict(best_programs_list[-1]) if total_trained_programs > self.total_eval: break #select the best programs based on cost + score. current_population.sort(key=lambda x: x["path_cost"]) selected_population = current_population[:self.selection_size] #perform crossover on the selected population crossed_population = [] log_and_print("Beginning crossover operation.") while len(crossed_population) < self.population_size: new_prog_dict = crossover_operation(selected_population) if new_prog_dict is not None: crossed_population.append(new_prog_dict) #perform mutations on the crossed population current_population = [] for crossed_prog_dict in crossed_population: if random.random() < self.mutation_prob: log_and_print("Mutating program.") current_population.append( mutation_operation(crossed_prog_dict)) else: current_population.append(crossed_prog_dict) return best_programs_list
def execute_and_train_og(base_program_name, validset, trainset, train_config, output_type, output_size, neural=False, device='cpu', use_valid_score=False, print_every=60): if device == 'cpu': program = CPU_Unpickler(open("%s.p" % base_program_name, "rb")).load() else: program = pickle.load(open("%s.p" % base_program_name, "rb")) lr = train_config['lr'] neural_epochs = train_config['neural_epochs'] symbolic_epochs = train_config['symbolic_epochs'] optimizer = train_config['optimizer'] lossfxn = train_config['lossfxn'] evalfxn = train_config['evalfxn'] num_labels = train_config['num_labels'] num_epochs = neural_epochs if neural else symbolic_epochs # initialize optimizer curr_optim = init_optimizer(program, optimizer, lr) # prepare validation set validation_input, validation_output = map(list, zip(*validset)) validation_true_vals = torch.flatten( torch.stack(validation_output)).float().to(device) # TODO a little hacky, but easiest solution for now if isinstance(lossfxn, nn.CrossEntropyLoss): validation_true_vals = validation_true_vals.long() best_program = None best_metric = float('inf') best_additional_params = {} for epoch in range(1, num_epochs + 1): for batchidx in range(len(trainset)): batch_input, batch_output = map(list, zip(*trainset[batchidx])) true_vals = torch.flatten( torch.stack(batch_output)).float().to(device) predicted_vals = process_batch(program, batch_input, output_type, output_size, device) # TODO a little hacky, but easiest solution for now if isinstance(lossfxn, nn.CrossEntropyLoss): true_vals = true_vals.long() # print(predicted_vals.shape, true_vals.shape) loss = lossfxn(predicted_vals, true_vals) curr_optim.zero_grad() loss.backward() curr_optim.step() # if batchidx % print_every == 0 or batchidx == 0: # log_and_print('Epoch [{}/{}], Loss: {:.4f}'.format(epoch, num_epochs, loss.item())) # check score on validation set with torch.no_grad(): predicted_vals = process_batch(program, validation_input, output_type, output_size, device) metric, additional_params = evalfxn(predicted_vals, validation_true_vals, num_labels=num_labels) if use_valid_score: if metric < best_metric: best_program = copy.deepcopy(program) best_metric = metric best_additional_params = additional_params else: best_program = copy.deepcopy(program) best_metric = metric best_additional_params = additional_params # select model with best validation score program = copy.deepcopy(best_program) log_and_print("Validation score is: {:.4f}".format(best_metric)) log_and_print("Average f1-score is: {:.4f}".format(1 - best_metric)) log_and_print("Hamming accuracy is: {:.4f}".format( best_additional_params['hamming_accuracy'])) return best_metric, program #mcheng substitute nonterminals with NNs
def execute_and_train(base_program, program, validset, trainset, train_config, output_type, output_size, neural=False, device='cpu', use_valid_score=False, print_every=60): # print('enter training initial') lr = train_config['lr'] neural_epochs = train_config['neural_epochs'] symbolic_epochs = train_config['symbolic_epochs'] optimizer = train_config['optimizer'] lossfxn = train_config['lossfxn'] evalfxn = train_config['evalfxn'] num_labels = train_config['num_labels'] num_epochs = neural_epochs if neural else symbolic_epochs # initialize optimizer curr_optim = init_optimizer(program, optimizer, lr) # prepare validation set validation_input, validation_output = map(list, zip(*validset)) validation_true_vals = torch.flatten( torch.stack(validation_output)).float().to(device) # TODO a little hacky, but easiest solution for now if isinstance(lossfxn, nn.CrossEntropyLoss): validation_true_vals = validation_true_vals.long() best_program = None best_metric = float('inf') best_additional_params = {} original_output_type = base_program.program.output_type original_output_size = base_program.program.output_size losses = [] training_f1 = [] for epoch in range(1, num_epochs + 1): temp_l = 0 # temp_f = 0 for batchidx in range(len(trainset)): batch_input, batch_output = map(list, zip(*trainset[batchidx])) true_vals = torch.flatten( torch.stack(batch_output)).float().to(device) predicted_vals = process_batch(base_program, batch_input, original_output_type, original_output_size, device) #fix lol # TODO a little hacky, but easiest solution for now if isinstance(lossfxn, nn.CrossEntropyLoss): true_vals = true_vals.long() # print(predicted_vals.shape, true_vals.shape) loss = lossfxn(predicted_vals, true_vals) training_metric, _ = evalfxn(predicted_vals, true_vals, num_labels=num_labels) # print('tutu metric') temp_l += float(loss.data) # temp_f += training_metric # print(training_metric) curr_optim.zero_grad() loss.backward() curr_optim.step() # if batchidx % print_every == 0 or batchidx == 0: # log_and_print('Epoch [{}/{}], Loss: {:.4f}'.format(epoch, num_epochs, loss.item())) # check score on validation set losses.append(temp_l / len(trainset)) # training_f1.append(temp_f/len(trainset)) with torch.no_grad(): predicted_vals = process_batch(base_program, validation_input, original_output_type, original_output_size, device) metric, additional_params = evalfxn(predicted_vals, validation_true_vals, num_labels=num_labels) if use_valid_score: if metric < best_metric: best_program = copy.deepcopy(program) best_metric = metric best_additional_params = additional_params else: best_program = copy.deepcopy(program) best_metric = metric best_additional_params = additional_params # select model with best validation score program = copy.deepcopy(best_program) log_and_print("Training loss is: {:.4f}".format(loss)) log_and_print("Validation score is: {:.4f}".format(best_metric)) log_and_print("Average f1-score is: {:.4f}".format(1 - best_metric)) log_and_print("Hamming accuracy is: {:.4f}".format( best_additional_params['hamming_accuracy'])) return best_metric, losses, training_f1
def __init__(self): log_and_print( "Root node is Start(ListToListModule) or Start(ListToAtomModule), both implemented with an RNN." ) log_and_print( "Be sure to set neural_epochs and max_num_units accordingly.\n")