def fix(self):
self.real_base = 'results/mars_an_astar-near_1_1605057595/fullprogram'
if self.device == 'cpu':
base_program = CPU_Unpickler(open("%s.p" % self.real_base, "rb")).load()
else:
base_program = pickle.load(open("%s.p" % self.real_base, "rb"))
if self.device == 'cpu':
best_program = CPU_Unpickler(open("%s/subprogram.p" % self.base_program_name, "rb")).load()
else:
best_program = pickle.load(open("%s/subprogram.p" % self.base_program_name, "rb"))
self.full_path = os.path.join(self.base_program_name, "fullprogram.p") #fix this
with torch.no_grad():
test_input, test_output = map(list, zip(*self.testset))
true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device)
predicted_vals = self.process_batch(base_program, test_input, self.output_type, self.output_size, self.device)
metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels)
log_and_print("F1 score achieved is {:.4f}".format(1 - metric))
curr_level = 0
l = []
traverse(base_program.submodules,l)
curr_program = base_program.submodules
change_key(base_program.submodules, [], self.hole_node_ind, best_program.submodules["program"])
with torch.no_grad():
test_input, test_output = map(list, zip(*self.testset))
true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device)
predicted_vals = self.process_batch(base_program, test_input, self.output_type, self.output_size, self.device)
metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels)
log_and_print("F1 score achieved is {:.4f}".format(1 - metric))
log_and_print(str(additional_params))
pickle.dump(base_program, open(self.full_path, "wb"))
def test_set_eval(program, testset, output_type, output_size, num_labels, device='cpu', verbose=False):
log_and_print("\n")
log_and_print("Evaluating program {} on TEST SET".format(print_program(program, ignore_constants=(not verbose))))
with torch.no_grad():
test_input, test_output = map(list, zip(*testset))
true_vals = torch.tensor(flatten_batch(test_output)).to(device)
predicted_vals = process_batch(program, test_input, output_type, output_size, device)
metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=num_labels)
log_and_print("F1 score achieved is {:.4f}".format(1 - metric))
log_and_print("Additional performance parameters: {}\n".format(additional_params))
def evaluate_final(self):
if self.device == 'cpu':
program = CPU_Unpickler(open(self.full_path, "rb").load())
else:
program = pickle.load(open(self.full_path, "rb"))
log_and_print(print_program(program, ignore_constants=True))
l = []
traverse(program.submodules,l)
with torch.no_grad():
test_input, test_output = map(list, zip(*self.testset))
true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device)
predicted_vals = self.process_batch(program, test_input, self.output_type, self.output_size, self.device)
metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels)
log_and_print("F1 score achieved is {:.4f}".format(1 - metric))
def evaluate(self):
assert os.path.isfile(self.program_path)
program = pickle.load(open(self.program_path, "rb"))
with torch.no_grad():
test_input, test_output = map(list, zip(*self.testset))
true_vals = torch.flatten(torch.stack(test_output)).float().to(
self.device)
predicted_vals = self.process_batch(program, test_input,
self.output_type,
self.output_size, self.device)
metric, additional_params = label_correctness(
predicted_vals, true_vals, num_labels=self.num_labels)
log_and_print("F1 score achieved is {:.4f}".format(1 - metric))
def train_more_epochs(self,program_to_train):
log_and_print("starting training more epochs")
train_config = {
'lr' : self.learning_rate,
'neural_epochs' : 20,
'symbolic_epochs' : 20,
'optimizer' : optim.Adam,
'lossfxn' : nn.CrossEntropyLoss(weight=self.loss_weight), #todo
'evalfxn' : label_correctness,
'num_labels' : self.num_labels
}
near_input_type = self.base_program.input_type
near_output_type = self.base_program.output_type
near_input_size = self.base_program.input_size
near_output_size = self.base_program.output_size
# Initialize program graph starting from trained NN
program_graph = ProgramGraph(DSL_DICT, CUSTOM_EDGE_COSTS, near_input_type, near_output_type, near_input_size, near_output_size,
self.max_num_units, self.min_num_units, self.max_num_children, self.max_depth, self.penalty, ite_beta=self.ite_beta)
# Initialize algorithm
algorithm = ASTAR_NEAR(frontier_capacity=self.frontier_capacity)
best_programs = algorithm.run_train_longer(self.timestamp, program_to_train, self.hole_node_ind,
program_graph, self.batched_trainset, self.validset, train_config, self.device)
best_program_str = []
# Print all best programs found
log_and_print("\n")
# log_and_print("BEST programs found:")
for item in best_programs:
program_struct = print_program(item["program"], ignore_constants=True)
program_info = " score {:.4f} ".format(item["score"])
best_program_str.append((program_struct, program_info))
print(best_program_str)
# print_program_dict(item)
best_program = best_programs[-1]["program"]
with torch.no_grad():
test_input, test_output = map(list, zip(*self.testset))
true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device)
predicted_vals = self.process_batch(best_program, test_input, self.output_type, self.output_size, self.device)
metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels)
pickle.dump(best_program, open(program_to_train + ".p", "wb")) #overfit??
log_and_print("end training more epochs")
def evaluate_neurosymb(self, program):
# if self.device == 'cpu':
# program = CPU_Unpickler(open("neursym.p" % self.base_program_name, "rb")).load()
# else:
# program = pickle.load(open("neursym.p" % self.base_program_name, "rb"))
# # program= CPU_Unpickler(open("%s.p" % self.base_program_name, "rb")).load()
print(print_program(program, ignore_constants=True))
l = []
traverse(program.submodules,l)
with torch.no_grad():
test_input, test_output = map(list, zip(*self.testset))
true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device)
predicted_vals = self.process_batch(program, test_input, self.output_type, self.output_size, self.device)
metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels)
log_and_print("Test F1 score achieved is {:.4f}\n".format(1 - metric))
log_and_print(str(additional_params))
return 1- metric
def evaluate(self):
# assert os.path.isfile(self.program_path)
base_program= CPU_Unpickler(open("%s.p" % self.base_program_name, "rb")).load()
program_baby = CPU_Unpickler(open("%s.p" % self.baby_program_name, "rb")).load()
data = base_program.submodules
l = []
traverse(data,l)
# print(l)
hole_node = l[self.hole_node_ind] #conditoin node
# print(hole_node)
change_key(base_program.submodules, hole_node[0], program_baby, hole_node[1])
# pickle.dump(program, open("ite_1603639887.p", "wb"))
base_output_type = base_program.program.output_type
base_output_size = base_program.program.output_size
# program = pickle.load(open(self.program_path, "rb"))
with torch.no_grad():
test_input, test_output = map(list, zip(*self.testset))
true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device)
predicted_vals = self.process_batch(base_program, test_input, base_output_type, base_output_size, self.device)
metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels)
log_and_print("F1 score achieved is {:.4f}".format(1 - metric))
