def fix(self): self.real_base = 'results/mars_an_astar-near_1_1605057595/fullprogram' if self.device == 'cpu': base_program = CPU_Unpickler(open("%s.p" % self.real_base, "rb")).load() else: base_program = pickle.load(open("%s.p" % self.real_base, "rb")) if self.device == 'cpu': best_program = CPU_Unpickler(open("%s/subprogram.p" % self.base_program_name, "rb")).load() else: best_program = pickle.load(open("%s/subprogram.p" % self.base_program_name, "rb")) self.full_path = os.path.join(self.base_program_name, "fullprogram.p") #fix this with torch.no_grad(): test_input, test_output = map(list, zip(*self.testset)) true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device) predicted_vals = self.process_batch(base_program, test_input, self.output_type, self.output_size, self.device) metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels) log_and_print("F1 score achieved is {:.4f}".format(1 - metric)) curr_level = 0 l = [] traverse(base_program.submodules,l) curr_program = base_program.submodules change_key(base_program.submodules, [], self.hole_node_ind, best_program.submodules["program"]) with torch.no_grad(): test_input, test_output = map(list, zip(*self.testset)) true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device) predicted_vals = self.process_batch(base_program, test_input, self.output_type, self.output_size, self.device) metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels) log_and_print("F1 score achieved is {:.4f}".format(1 - metric)) log_and_print(str(additional_params)) pickle.dump(base_program, open(self.full_path, "wb"))
def test_set_eval(program, testset, output_type, output_size, num_labels, device='cpu', verbose=False): log_and_print("\n") log_and_print("Evaluating program {} on TEST SET".format(print_program(program, ignore_constants=(not verbose)))) with torch.no_grad(): test_input, test_output = map(list, zip(*testset)) true_vals = torch.tensor(flatten_batch(test_output)).to(device) predicted_vals = process_batch(program, test_input, output_type, output_size, device) metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=num_labels) log_and_print("F1 score achieved is {:.4f}".format(1 - metric)) log_and_print("Additional performance parameters: {}\n".format(additional_params))
def evaluate_final(self): if self.device == 'cpu': program = CPU_Unpickler(open(self.full_path, "rb").load()) else: program = pickle.load(open(self.full_path, "rb")) log_and_print(print_program(program, ignore_constants=True)) l = [] traverse(program.submodules,l) with torch.no_grad(): test_input, test_output = map(list, zip(*self.testset)) true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device) predicted_vals = self.process_batch(program, test_input, self.output_type, self.output_size, self.device) metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels) log_and_print("F1 score achieved is {:.4f}".format(1 - metric))
def evaluate(self): assert os.path.isfile(self.program_path) program = pickle.load(open(self.program_path, "rb")) with torch.no_grad(): test_input, test_output = map(list, zip(*self.testset)) true_vals = torch.flatten(torch.stack(test_output)).float().to( self.device) predicted_vals = self.process_batch(program, test_input, self.output_type, self.output_size, self.device) metric, additional_params = label_correctness( predicted_vals, true_vals, num_labels=self.num_labels) log_and_print("F1 score achieved is {:.4f}".format(1 - metric))
def train_more_epochs(self,program_to_train): log_and_print("starting training more epochs") train_config = { 'lr' : self.learning_rate, 'neural_epochs' : 20, 'symbolic_epochs' : 20, 'optimizer' : optim.Adam, 'lossfxn' : nn.CrossEntropyLoss(weight=self.loss_weight), #todo 'evalfxn' : label_correctness, 'num_labels' : self.num_labels } near_input_type = self.base_program.input_type near_output_type = self.base_program.output_type near_input_size = self.base_program.input_size near_output_size = self.base_program.output_size # Initialize program graph starting from trained NN program_graph = ProgramGraph(DSL_DICT, CUSTOM_EDGE_COSTS, near_input_type, near_output_type, near_input_size, near_output_size, self.max_num_units, self.min_num_units, self.max_num_children, self.max_depth, self.penalty, ite_beta=self.ite_beta) # Initialize algorithm algorithm = ASTAR_NEAR(frontier_capacity=self.frontier_capacity) best_programs = algorithm.run_train_longer(self.timestamp, program_to_train, self.hole_node_ind, program_graph, self.batched_trainset, self.validset, train_config, self.device) best_program_str = [] # Print all best programs found log_and_print("\n") # log_and_print("BEST programs found:") for item in best_programs: program_struct = print_program(item["program"], ignore_constants=True) program_info = " score {:.4f} ".format(item["score"]) best_program_str.append((program_struct, program_info)) print(best_program_str) # print_program_dict(item) best_program = best_programs[-1]["program"] with torch.no_grad(): test_input, test_output = map(list, zip(*self.testset)) true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device) predicted_vals = self.process_batch(best_program, test_input, self.output_type, self.output_size, self.device) metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels) pickle.dump(best_program, open(program_to_train + ".p", "wb")) #overfit?? log_and_print("end training more epochs")
def evaluate_neurosymb(self, program): # if self.device == 'cpu': # program = CPU_Unpickler(open("neursym.p" % self.base_program_name, "rb")).load() # else: # program = pickle.load(open("neursym.p" % self.base_program_name, "rb")) # # program= CPU_Unpickler(open("%s.p" % self.base_program_name, "rb")).load() print(print_program(program, ignore_constants=True)) l = [] traverse(program.submodules,l) with torch.no_grad(): test_input, test_output = map(list, zip(*self.testset)) true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device) predicted_vals = self.process_batch(program, test_input, self.output_type, self.output_size, self.device) metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels) log_and_print("Test F1 score achieved is {:.4f}\n".format(1 - metric)) log_and_print(str(additional_params)) return 1- metric
def evaluate(self): # assert os.path.isfile(self.program_path) base_program= CPU_Unpickler(open("%s.p" % self.base_program_name, "rb")).load() program_baby = CPU_Unpickler(open("%s.p" % self.baby_program_name, "rb")).load() data = base_program.submodules l = [] traverse(data,l) # print(l) hole_node = l[self.hole_node_ind] #conditoin node # print(hole_node) change_key(base_program.submodules, hole_node[0], program_baby, hole_node[1]) # pickle.dump(program, open("ite_1603639887.p", "wb")) base_output_type = base_program.program.output_type base_output_size = base_program.program.output_size # program = pickle.load(open(self.program_path, "rb")) with torch.no_grad(): test_input, test_output = map(list, zip(*self.testset)) true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device) predicted_vals = self.process_batch(base_program, test_input, base_output_type, base_output_size, self.device) metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels) log_and_print("F1 score achieved is {:.4f}".format(1 - metric))