def run_init(self, timestamp, base_program_name, hole_node_ind, graph, trainset, validset, train_config, device, verbose=False):
assert isinstance(graph, ProgramGraph)
log_and_print("Training root program ...")
current = copy.deepcopy(graph.root_node)
initial_score, losses, m = execute_and_train_with_full(base_program_name, hole_node_ind, current.program, validset, trainset, train_config,
graph.output_type, graph.output_size, neural=True, device=device)
log_and_print("Initial training complete. Score from program is {:.4f} \n".format(1 - initial_score))
if device == 'cpu':
base_program = CPU_Unpickler(open("%s.p" % base_program_name, "rb")).load()
else:
base_program = pickle.load(open("%s.p" % base_program_name, "rb"))
curr_level = 0
l = []
traverse(base_program.submodules,l)
# pprint(l)
curr_program = base_program.submodules
change_key(base_program.submodules, [], hole_node_ind, current.program.submodules["program"])
new_prog = base_program
return 1 - initial_score, new_prog, losses
def crossover_operation(listofprogramdicts):
new_prog_dict = {}
parents = random.choices(listofprogramdicts, k=2)
prog_dict1, prog_dict2 = parents[0], parents[1]
xover1 = random.randint(1, prog_dict1['depth'])
xover2 = random.randint(2, prog_dict2['depth'])
print('DEBUG: trying new crossover')
new_program, crossed_over = self.crossover_helper(
prog_dict1['program'], prog_dict2['program'], xover1, xover2)
if crossed_over:
log_and_print("Crossing over the following programs:")
log_and_print(
print_program(prog_dict1['program'],
ignore_constants=True))
log_and_print(
print_program(prog_dict2['program'],
ignore_constants=True))
new_prog_dict["program"] = new_program
new_prog_dict["struct_cost"], new_prog_dict[
"depth"] = graph.compute_program_cost(new_program)
log_and_print("Result has structural cost {:.4f}:".format(
new_prog_dict["struct_cost"]))
log_and_print(
print_program(new_prog_dict['program'],
ignore_constants=True))
return new_prog_dict
else:
return None
def run(self, graph, trainset, validset, train_config, device, verbose=False):
assert isinstance(graph, ProgramGraph)
log_and_print("Training RNN baseline with {} LSTM units ...".format(graph.max_num_units))
current = copy.deepcopy(graph.root_node)
score = execute_and_train(current.program, validset, trainset, train_config,
graph.output_type, graph.output_size, neural=True, device=device)
log_and_print("Score of RNN is {:.4f}".format(1-score))
return current.program
def add(self, item):
assert len(item) == 3
assert isinstance(item[2], ProgramNode)
self.prioqueue.append(item)
if len(self.prioqueue) > self.capacity:
# self.sort(tup_idx=0)
popped_f_score, _, popped = self.pop(-1)
log_and_print("POP {} with fscore {:.4f}".format(
print_program(popped.program, ignore_constants=True),
popped_f_score))
def run_train_longer(self, timestamp, base_program_name, hole_node_ind, graph, trainset, validset, train_config, device, verbose=False):
assert isinstance(graph, ProgramGraph)
log_and_print("Training root program ...")
current = copy.deepcopy(graph.root_node)
initial_score, program = execute_and_train_og(base_program_name, validset, trainset, train_config,
graph.output_type, graph.output_size, neural=True, device=device)
log_and_print("Re-training complete. Score from program is {:.4f} \n".format(1 - initial_score))
return [{
"program" : program,
"score" : 1- initial_score,
}]
def load_base_program(self):
print("Loading %s" % self.base_program_name)
if self.device == 'cpu':
self.base_program = CPU_Unpickler(open("%s.p" % self.base_program_name, "rb")).load()
else:
self.base_program = pickle.load(open("%s.p" % self.base_program_name, "rb"))
base_folder = os.path.dirname(self.base_program_name)
data = self.base_program.submodules
l = []
traverse(data,l)
log_and_print(l)
return l
def evaluate_final(self):
if self.device == 'cpu':
program = CPU_Unpickler(open(self.full_path, "rb").load())
else:
program = pickle.load(open(self.full_path, "rb"))
log_and_print(print_program(program, ignore_constants=True))
l = []
traverse(program.submodules,l)
with torch.no_grad():
test_input, test_output = map(list, zip(*self.testset))
true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device)
predicted_vals = self.process_batch(program, test_input, self.output_type, self.output_size, self.device)
metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels)
log_and_print("F1 score achieved is {:.4f}".format(1 - metric))
def evaluate(self):
assert os.path.isfile(self.program_path)
program = pickle.load(open(self.program_path, "rb"))
with torch.no_grad():
test_input, test_output = map(list, zip(*self.testset))
true_vals = torch.flatten(torch.stack(test_output)).float().to(
self.device)
predicted_vals = self.process_batch(program, test_input,
self.output_type,
self.output_size, self.device)
metric, additional_params = label_correctness(
predicted_vals, true_vals, num_labels=self.num_labels)
log_and_print("F1 score achieved is {:.4f}".format(1 - metric))
def run_near(self, model, num_iter):
train_config = {
'lr': self.learning_rate,
'neural_epochs': self.neural_epochs,
'symbolic_epochs': self.symbolic_epochs,
'optimizer': optim.Adam,
'lossfxn': nn.CrossEntropyLoss(), #todo
'evalfxn': label_correctness,
'num_labels': self.num_labels
}
# Initialize program graph starting from trained NN
program_graph = ProgramGraph(model,
DSL_DICT,
CUSTOM_EDGE_COSTS,
self.input_type,
self.output_type,
self.input_size,
self.output_size,
self.max_num_units,
self.min_num_units,
self.max_num_children,
self.max_depth,
self.penalty,
ite_beta=self.ite_beta)
# Initialize algorithm
algorithm = ASTAR_NEAR(frontier_capacity=self.frontier_capacity)
best_programs = algorithm.run(program_graph, self.batched_trainset,
self.validset, train_config, self.device)
if self.algorithm == "rnn":
# special case for RNN baseline
best_program = best_programs
else:
# Print all best programs found
log_and_print("\n")
log_and_print("BEST programs found:")
for item in best_programs:
print_program_dict(item)
best_program = best_programs[-1]["program"]
# Save best program
self.program_path = os.path.join(self.save_path,
"program_%d.p" % num_iter)
pickle.dump(best_program, open(self.program_path, "wb"))
def test_set_eval(program, testset, output_type, output_size, num_labels, device='cpu', verbose=False):
log_and_print("\n")
log_and_print("Evaluating program {} on TEST SET".format(print_program(program, ignore_constants=(not verbose))))
with torch.no_grad():
test_input, test_output = map(list, zip(*testset))
true_vals = torch.tensor(flatten_batch(test_output)).to(device)
predicted_vals = process_batch(program, test_input, output_type, output_size, device)
metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=num_labels)
log_and_print("F1 score achieved is {:.4f}".format(1 - metric))
log_and_print("Additional performance parameters: {}\n".format(additional_params))
def evaluate_neurosymb(self, program):
# if self.device == 'cpu':
# program = CPU_Unpickler(open("neursym.p" % self.base_program_name, "rb")).load()
# else:
# program = pickle.load(open("neursym.p" % self.base_program_name, "rb"))
# # program= CPU_Unpickler(open("%s.p" % self.base_program_name, "rb")).load()
print(print_program(program, ignore_constants=True))
l = []
traverse(program.submodules,l)
with torch.no_grad():
test_input, test_output = map(list, zip(*self.testset))
true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device)
predicted_vals = self.process_batch(program, test_input, self.output_type, self.output_size, self.device)
metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels)
log_and_print("Test F1 score achieved is {:.4f}\n".format(1 - metric))
log_and_print(str(additional_params))
return 1- metric
def __init__(self, **kwargs):
self.__dict__.update(kwargs)
if torch.cuda.is_available():
self.device = 'cuda:0'
else:
self.device = 'cpu'
# load input data
self.train_data = np.load(self.train_data)
self.test_data = np.load(self.test_data)
self.valid_data = None
self.train_labels = np.load(self.train_labels)
self.test_labels = np.load(self.test_labels)
self.valid_labels = None
if self.valid_data is not None and self.valid_labels is not None:
self.valid_data = np.load(self.valid_data)
self.valid_labels = np.load(self.valid_labels)
self.batched_trainset, self.validset, self.testset = prepare_datasets(self.train_data, self.valid_data, self.test_data, self.train_labels, self.valid_labels,
self.test_labels, normalize=self.normalize, train_valid_split=self.train_valid_split, batch_size=self.batch_size)
if self.device == 'cpu':
self.base_program = CPU_Unpickler(open("%s.p" % self.base_program_name, "rb")).load()
else:
self.base_program = pickle.load(open("%s.p" % self.base_program_name, "rb"))
data = self.base_program.submodules
l = []
traverse(data,l)
self.hole_node = l[self.hole_node_ind]
#for near on subtree
self.curr_iter = 0
self.program_path = None
now = datetime.now()
self.timestamp = str(datetime.timestamp(now)).split('.')[0]
log_and_print(self.timestamp)
self.evaluate()
def fix(self):
self.real_base = 'results/mars_an_astar-near_1_1605057595/fullprogram'
if self.device == 'cpu':
base_program = CPU_Unpickler(open("%s.p" % self.real_base, "rb")).load()
else:
base_program = pickle.load(open("%s.p" % self.real_base, "rb"))
if self.device == 'cpu':
best_program = CPU_Unpickler(open("%s/subprogram.p" % self.base_program_name, "rb")).load()
else:
best_program = pickle.load(open("%s/subprogram.p" % self.base_program_name, "rb"))
self.full_path = os.path.join(self.base_program_name, "fullprogram.p") #fix this
with torch.no_grad():
test_input, test_output = map(list, zip(*self.testset))
true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device)
predicted_vals = self.process_batch(base_program, test_input, self.output_type, self.output_size, self.device)
metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels)
log_and_print("F1 score achieved is {:.4f}".format(1 - metric))
curr_level = 0
l = []
traverse(base_program.submodules,l)
curr_program = base_program.submodules
change_key(base_program.submodules, [], self.hole_node_ind, best_program.submodules["program"])
with torch.no_grad():
test_input, test_output = map(list, zip(*self.testset))
true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device)
predicted_vals = self.process_batch(base_program, test_input, self.output_type, self.output_size, self.device)
metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels)
log_and_print("F1 score achieved is {:.4f}".format(1 - metric))
log_and_print(str(additional_params))
pickle.dump(base_program, open(self.full_path, "wb"))
def mutation_helper(self,
graph,
program,
total_depth,
mutation_point,
max_enumeration_depth=6):
pointer = 1
queue = [program]
while len(queue) != 0:
current_function = queue.pop()
for submodule, functionclass in current_function.submodules.items(
):
if pointer == mutation_point:
log_and_print(
"Generating replacements for mutation program at depth {} and typesig {}."
.format(total_depth - mutation_point,
(functionclass.input_type,
functionclass.output_type)))
enumeration_depth = min(max_enumeration_depth,
total_depth - mutation_point)
all_replacements = self.enumerative_synthesis(
graph,
enumeration_depth,
(functionclass.input_type, functionclass.output_type),
input_size=functionclass.input_size,
output_size=functionclass.output_size)
if len(all_replacements) == 0:
return program
else:
mutated_replacement = random.choice(all_replacements)
current_function.submodules[submodule] = copy.deepcopy(
mutated_replacement['program'].
submodules['program'])
self.reset_program_sizes(program)
log_and_print("Mutation completed.")
return program
pointer += 1
queue.append(functionclass)
return program
def execute_and_train_with_full(base_program_name,
hole_node_ind,
program,
validset,
trainset,
train_config,
output_type,
output_size,
neural=False,
device='cpu',
use_valid_score=False,
print_every=60):
#load program
# pprint(type(hole_node))
# level_to_replace = hole_node[1]
if device == 'cpu':
base_program = CPU_Unpickler(open("%s.p" % base_program_name,
"rb")).load()
else:
base_program = pickle.load(open("%s.p" % base_program_name, "rb"))
curr_level = 0
l = []
traverse(base_program.submodules, l)
# pprint(l)
curr_program = base_program.submodules
# print(program)
# print(program.submodules)
# pprint
change_key(
base_program.submodules, [], hole_node_ind,
program.submodules["program"]) #should we just replace with program?
log_and_print(print_program(base_program))
# pickle.dump(base_program, open("neursym.p", "wb"))
return execute_and_train(base_program, program, validset, trainset,
train_config, output_type, output_size, neural,
device)
def evaluate(self):
# assert os.path.isfile(self.program_path)
base_program= CPU_Unpickler(open("%s.p" % self.base_program_name, "rb")).load()
program_baby = CPU_Unpickler(open("%s.p" % self.baby_program_name, "rb")).load()
data = base_program.submodules
l = []
traverse(data,l)
# print(l)
hole_node = l[self.hole_node_ind] #conditoin node
# print(hole_node)
change_key(base_program.submodules, hole_node[0], program_baby, hole_node[1])
# pickle.dump(program, open("ite_1603639887.p", "wb"))
base_output_type = base_program.program.output_type
base_output_size = base_program.program.output_size
# program = pickle.load(open(self.program_path, "rb"))
with torch.no_grad():
test_input, test_output = map(list, zip(*self.testset))
true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device)
predicted_vals = self.process_batch(base_program, test_input, base_output_type, base_output_size, self.device)
metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels)
log_and_print("F1 score achieved is {:.4f}".format(1 - metric))
def neural_h(self):
data = self.base_program.submodules
l = [] #populate AST
traverse(data,l)
train_config = {
'lr' : self.learning_rate,
'neural_epochs' : 15,
'symbolic_epochs' : self.symbolic_epochs,
'optimizer' : optim.Adam,
'lossfxn' : nn.CrossEntropyLoss(weight=self.loss_weight), #todo
'evalfxn' : label_correctness,
'num_labels' : self.num_labels
}
best_node_ind = 0
best_score = 0
for hole_node_ind in range(1,len(l)):
hole_node = l[hole_node_ind]
subprogram_str = print_program(hole_node[0])
if subprogram_str.count('(') > self.max_depth:
continue
near_input_type = hole_node[0].input_type
near_output_type = hole_node[0].output_type
near_input_size = hole_node[0].input_size
near_output_size = hole_node[0].output_size
# Initialize program graph starting from trained NN
program_graph = ProgramGraph(DSL_DICT, CUSTOM_EDGE_COSTS, near_input_type, near_output_type, near_input_size, near_output_size,
self.max_num_units, self.min_num_units, self.max_num_children, 0, self.penalty, ite_beta=self.ite_beta) ## max_depth 0
# Initialize algorithm
algorithm = ASTAR_NEAR(frontier_capacity=0)
score, new_prog, losses = algorithm.run_init(self.timestamp, self.base_program_name, hole_node_ind,
program_graph, self.batched_trainset, self.validset, train_config, self.device)
subprogram_str = print_program(hole_node[0])
log_and_print("Subprogram to replace: %s"% subprogram_str)
if score > best_score:
best_node_ind = hole_node_ind
best_score = score
log_and_print("New best: RNN Heuristic score at Node %d: %f\n" %( hole_node_ind, score))
else:
log_and_print("RNN Heuristic score at Node %d: %f\n" %( hole_node_ind, score))
return best_node_ind
def train_more_epochs(self,program_to_train):
log_and_print("starting training more epochs")
train_config = {
'lr' : self.learning_rate,
'neural_epochs' : 20,
'symbolic_epochs' : 20,
'optimizer' : optim.Adam,
'lossfxn' : nn.CrossEntropyLoss(weight=self.loss_weight), #todo
'evalfxn' : label_correctness,
'num_labels' : self.num_labels
}
near_input_type = self.base_program.input_type
near_output_type = self.base_program.output_type
near_input_size = self.base_program.input_size
near_output_size = self.base_program.output_size
# Initialize program graph starting from trained NN
program_graph = ProgramGraph(DSL_DICT, CUSTOM_EDGE_COSTS, near_input_type, near_output_type, near_input_size, near_output_size,
self.max_num_units, self.min_num_units, self.max_num_children, self.max_depth, self.penalty, ite_beta=self.ite_beta)
# Initialize algorithm
algorithm = ASTAR_NEAR(frontier_capacity=self.frontier_capacity)
best_programs = algorithm.run_train_longer(self.timestamp, program_to_train, self.hole_node_ind,
program_graph, self.batched_trainset, self.validset, train_config, self.device)
best_program_str = []
# Print all best programs found
log_and_print("\n")
# log_and_print("BEST programs found:")
for item in best_programs:
program_struct = print_program(item["program"], ignore_constants=True)
program_info = " score {:.4f} ".format(item["score"])
best_program_str.append((program_struct, program_info))
print(best_program_str)
# print_program_dict(item)
best_program = best_programs[-1]["program"]
with torch.no_grad():
test_input, test_output = map(list, zip(*self.testset))
true_vals = torch.flatten(torch.stack(test_output)).float().to(self.device)
predicted_vals = self.process_batch(best_program, test_input, self.output_type, self.output_size, self.device)
metric, additional_params = label_correctness(predicted_vals, true_vals, num_labels=self.num_labels)
pickle.dump(best_program, open(program_to_train + ".p", "wb")) #overfit??
log_and_print("end training more epochs")
def run(self, graph, trainset, validset, train_config, device, verbose=False):
assert isinstance(graph, ProgramGraph)
current = copy.deepcopy(graph.root_node)
current_avg_f_score = float('inf')
best_program = None
best_total_cost = float('inf')
best_programs_list = []
num_children_trained = 0
start_time = time.time()
while not graph.is_fully_symbolic(current.program):
log_and_print("CURRENT program has avg fscore {:.4f}: {}".format(
current_avg_f_score, print_program(current.program, ignore_constants=(not verbose))))
children = graph.get_all_children(current, in_enumeration=True)
children_mapping = { print_program(child.program, ignore_constants=True) : child for child in children }
children_scores = { key : [] for key in children_mapping.keys() }
costs = [child.cost for child in children]
for i in range(self.num_mc_samples):
child = random.choices(children, weights=costs)[0]
sample = self.mc_sample(graph, child)
assert graph.is_fully_symbolic(sample.program)
log_and_print("Training sample program: {}".format(print_program(sample.program, ignore_constants=(not verbose))))
sample_score = execute_and_train(sample.program, validset, trainset, train_config,
graph.output_type, graph.output_size, neural=False, device=device)
num_children_trained += 1
log_and_print("{} total children trained".format(num_children_trained))
sample_f_score = sample.cost + sample_score
children_scores[print_program(child.program, ignore_constants=True)].append(sample_f_score)
if sample_f_score < best_total_cost:
best_program = copy.deepcopy(sample.program)
best_total_cost = sample_f_score
best_programs_list.append({
"program" : best_program,
"struct_cost" : sample.cost,
"score" : sample_score,
"path_cost" : sample_f_score,
"time" : time.time()-start_time
})
log_and_print("New BEST program found:")
print_program_dict(best_programs_list[-1])
# (Naive) selection operation
children_scores = { key : sum(val)/len(val) if len(val) > 0 else float('inf') for key,val in children_scores.items() }
best_child_name = min(children_scores, key=children_scores.get)
current = children_mapping[best_child_name]
current_avg_f_score = children_scores[best_child_name]
for key,val in children_scores.items():
log_and_print("Avg score {:.4f} for child {}".format(val,key))
log_and_print("SELECTING {} as best child node\n".format(best_child_name))
log_and_print("DEBUG: time since start is {:.3f}\n".format(time.time()-start_time))
return best_programs_list
def __init__(self, **kwargs):
self.__dict__.update(kwargs)
if torch.cuda.is_available():
self.device = 'cuda:0'
print(self.device)
else:
self.device = 'cpu'
self.loss_weight = torch.tensor([float(w) for w in self.class_weights.split(',')]).to(self.device)
if self.exp_name == 'crim13':
# load input data
self.train_data = np.load(self.train_data)
self.test_data = np.load(self.test_data)
self.valid_data = None
self.train_labels = np.load(self.train_labels)
self.test_labels = np.load(self.test_labels)
self.valid_labels = None
assert self.train_data.shape[-1] == self.test_data.shape[-1] == self.input_size
if self.valid_data is not None and self.valid_labels is not None:
self.valid_data = np.load(self.valid_data)
self.valid_labels = np.load(self.valid_labels)
assert valid_data.shape[-1] == self.input_size
self.batched_trainset, self.validset, self.testset = prepare_datasets(self.train_data, self.valid_data, self.test_data, self.train_labels, self.valid_labels,
self.test_labels, normalize=self.normalize, train_valid_split=self.train_valid_split, batch_size=self.batch_size)
elif self.exp_name == 'mars_an':
#### start mars
train_datasets = self.train_data.split(",")
train_raw_features = []
train_raw_annotations = []
for fname in train_datasets:
data = np.load(fname, allow_pickle=True)
train_raw_features.extend(data["features"])
train_raw_annotations.extend(data["annotations"])
test_data = np.load(self.test_data, allow_pickle=True)
test_raw_features = test_data["features"]
test_raw_annotations = test_data["annotations"]
valid_raw_features = None
valid_raw_annotations = None
valid_labels = None
# Check the # of features of the first frame of the first video
assert len(train_raw_features[0][0]) == len(test_raw_features[0][0]) == self.input_size
if self.valid_data is not None:
valid_data = np.load(self.valid_data, allow_pickle=True)
valid_raw_features = valid_data["features"]
valid_raw_annotations = valid_data["annotations"]
assert len(valid_raw_features[0][0]) == self.input_size
behave_dict = read_into_dict('../near_code_7keypoints/data/MARS_data/behavior_assignments_3class.txt')
# Reshape the data to trajectories of length 100
train_features, train_labels = preprocess(train_raw_features, train_raw_annotations, self.train_labels, behave_dict)
test_features, test_labels = preprocess(test_raw_features, test_raw_annotations, self.train_labels, behave_dict)
if valid_raw_features is not None and valid_raw_annotations is not None:
valid_features, valid_labels = preprocess(valid_raw_features, valid_raw_annotations, self.train_labels, behave_dict)
self.batched_trainset, self.validset, self.testset = prepare_datasets(train_features, valid_features, test_features,
train_labels, valid_labels, test_labels,
normalize=self.normalize, train_valid_split=self.train_valid_split, batch_size=self.batch_size)
##### END MARS
else:
log_and_print('bad experiment name')
return
# self.fix()
# add subprogram in
# if self.device == 'cpu':
# self.base_program = CPU_Unpickler(open("%s/subprogram.p" % self.base_program_name, "rb")).load()
# else:
# self.base_program = pickle.load(open("%s/subprogram.p" % self.base_program_name, "rb"))
if self.device == 'cpu':
self.base_program = CPU_Unpickler(open("%s.p" % self.base_program_name, "rb")).load()
else:
self.base_program = pickle.load(open("%s.p" % self.base_program_name, "rb"))
base_folder = os.path.dirname(self.base_program_name)
# self.weights_dict = np.load(os.path.join(base_folder,'weights.npy'), allow_pickle=True).item()
data = self.base_program.submodules
l = []
traverse(data,l)
log_and_print(l)
# if self.hole_node_ind < 0:
# self.hole_node_ind = len(l) + self.hole_node_ind
#if negative, make it positive
self.hole_node_ind %= len(l)
self.hole_node = l[self.hole_node_ind]
#for near on subtree
self.curr_iter = 0
self.program_path = None
if self.exp_id is not None:
self.trial = self.exp_id
if self.eval:
self.evaluate()
else:
now = datetime.now()
self.timestamp = str(datetime.timestamp(now)).split('.')[0][4:]
log_and_print(self.timestamp)
full_exp_name = "{}_{}_{}_{}".format(
self.exp_name, self.algorithm, self.trial, self.timestamp) #unique timestamp for each near run
self.save_path = os.path.join(self.save_dir, full_exp_name)
if not os.path.exists(self.save_path):
os.makedirs(self.save_path)
init_logging(self.save_path)
if self.neurh:
log_and_print(self.base_program_name)
self.neural_h()
else:
self.run_near()
self.evaluate_final()
def run(self,
graph,
trainset,
validset,
train_config,
device,
verbose=False):
assert isinstance(graph, ProgramGraph)
log_and_print("Training root program ...")
current = copy.deepcopy(graph.root_node)
initial_score = execute_and_train(current.program,
validset,
trainset,
train_config,
graph.output_type,
graph.output_size,
neural=True,
device=device)
log_and_print(
"Initial training complete. Score from program is {:.4f} \n".
format(1 - initial_score))
# Branch-and-bound search with iterative deepening
current_depth = self.initial_depth
current_f_score = float('inf')
order = 0
frontier = ProgramNodeFrontier(capacity=self.frontier_capacity)
next_frontier = ProgramNodeFrontier(capacity=self.frontier_capacity)
num_children_trained = 0
start_time = time.time()
best_program = None
best_total_cost = float('inf')
best_programs_list = []
log_and_print("Starting iterative deepening with depth {}\n".format(
current_depth))
while current_depth <= graph.max_depth:
log_and_print("CURRENT program has fscore {:.4f}: {}".format(
current_f_score,
print_program(current.program,
ignore_constants=(not verbose))))
log_and_print("Current depth of program is {}".format(
current.depth))
log_and_print("Creating children for current node/program")
log_and_print("Total time elapsed is {:.3f}".format(time.time() -
start_time))
children_nodes = graph.get_all_children(current)
# prune if more than self.max_num_children
if len(children_nodes) > graph.max_num_children:
log_and_print("Sampling {}/{} children".format(
graph.max_num_children, len(children_nodes)))
children_nodes = random.sample(
children_nodes,
k=graph.max_num_children) # sample without replacement
log_and_print("{} total children to train for current node".format(
len(children_nodes)))
child_tuples = []
for child_node in children_nodes:
child_start_time = time.time()
log_and_print("Training child program: {}".format(
print_program(child_node.program,
ignore_constants=(not verbose))))
is_neural = not graph.is_fully_symbolic(child_node.program)
child_node.score = execute_and_train(child_node.program,
validset,
trainset,
train_config,
graph.output_type,
graph.output_size,
neural=is_neural,
device=device)
log_and_print(
"Time to train child {:.3f}".format(time.time() -
child_start_time))
num_children_trained += 1
log_and_print(
"{} total children trained".format(num_children_trained))
child_node.parent = current
child_node.children = []
order -= 1
child_node.order = order # insert order of exploration as tiebreaker for equivalent f-scores
# computing path costs (f_scores)
child_f_score = child_node.cost + child_node.score # cost + heuristic
log_and_print("DEBUG: f-score {}".format(child_f_score))
current.children.append(child_node)
child_tuples.append((child_f_score, order, child_node))
if not is_neural and child_f_score < best_total_cost:
best_program = copy.deepcopy(child_node.program)
best_total_cost = child_f_score
best_programs_list.append({
"program": best_program,
"struct_cost": child_node.cost,
"score": child_node.score,
"path_cost": child_f_score,
"time": time.time() - start_time
})
log_and_print("New BEST program found:")
print_program_dict(best_programs_list[-1])
# find next current among children, from best to worst
nextfound = False
child_tuples.sort(key=lambda x: x[0])
for child_tuple in child_tuples:
child = child_tuple[2]
if graph.is_fully_symbolic(child.program):
continue # don't want to expand symbolic programs (no children)
elif child.depth >= current_depth:
next_frontier.add(child_tuple)
else:
if not nextfound:
nextfound = True # first child program that's not symbolic and within current_depth
current_f_score, current_order, current = child_tuple
log_and_print(
"Found program among children: {} with f_score {}".
format(
print_program(current.program,
ignore_constants=(not verbose)),
current_f_score))
else:
frontier.add(child_tuple) # put the rest onto frontier
# find next node in frontier
if not nextfound:
frontier.sort(tup_idx=1) # DFS order
log_and_print("Frontier length is: {}".format(len(frontier)))
original_depth = current.depth
while len(frontier) > 0 and not nextfound:
current_f_score, current_order, current = frontier.pop(
0, sort_fscores=False) # DFS order
if current_f_score < self.bound_modify(
best_total_cost, original_depth, current.depth):
nextfound = True
log_and_print(
"Found program in frontier: {} with f_score {}".
format(
print_program(current.program,
ignore_constants=(not verbose)),
current_f_score))
else:
log_and_print(
"PRUNE from frontier: {} with f_score {}".format(
print_program(current.program,
ignore_constants=(not verbose)),
current_f_score))
log_and_print("Frontier length is now {}".format(
len(frontier)))
# frontier is empty, go to next stage of iterative deepening
if not nextfound:
assert len(frontier) == 0
log_and_print("Empty frontier, moving to next depth level")
log_and_print(
"DEBUG: time since start is {:.3f}\n".format(time.time() -
start_time))
current_depth += 1
if current_depth > graph.max_depth:
log_and_print("Max depth {} reached. Exiting.\n".format(
graph.max_depth))
break
elif len(next_frontier) == 0:
log_and_print("Next frontier is empty. Exiting.\n")
break
else:
log_and_print(
"Starting iterative deepening with depth {}\n".format(
current_depth))
frontier = copy.deepcopy(next_frontier)
next_frontier = ProgramNodeFrontier(
capacity=self.frontier_capacity)
current_f_score, current_order, current = frontier.pop(0)
if best_program is None:
log_and_print("ERROR: no program found")
return best_programs_list
if device != 'cpu':
lossfxn = lossfxn.cuda()
train_config = {
'lr': args.learning_rate,
'neural_epochs': args.neural_epochs,
'symbolic_epochs': args.symbolic_epochs,
'optimizer': optim.Adam,
'lossfxn': lossfxn,
'evalfxn': label_correctness,
'num_labels': args.num_labels
}
# Initialize logging
init_logging(save_path)
log_and_print("Starting experiment {}\n".format(full_exp_name))
# Initialize program graph
program_graph = ProgramGraph(DSL_DICT,
CUSTOM_EDGE_COSTS,
args.input_type,
args.output_type,
args.input_size,
args.output_size,
args.max_num_units,
args.min_num_units,
args.max_num_children,
args.max_depth,
args.penalty,
ite_beta=args.ite_beta)
def run(self,
graph,
trainset,
validset,
train_config,
device,
verbose=False,
trainset_neural=None):
assert isinstance(graph, ProgramGraph)
log_and_print("Training root program ...")
current = copy.deepcopy(graph.root_node)
dataset = trainset_neural if trainset_neural is not None else trainset
initial_score = execute_and_train(current.program,
validset,
dataset,
train_config,
graph.output_type,
graph.output_size,
neural=True,
device=device)
log_and_print(
"Initial training complete. Score from program is {:.4f} \n".
format(1 - initial_score))
order = 0
frontier = ProgramNodeFrontier(capacity=self.frontier_capacity)
frontier.add((float('inf'), order, current))
num_children_trained = 0
start_time = time.time()
best_program = None
best_total_cost = float('inf')
best_programs_list = []
while len(frontier) != 0:
current_f_score, _, current = frontier.pop(0)
log_and_print("CURRENT program has fscore {:.4f}: {}".format(
current_f_score,
print_program(current.program,
ignore_constants=(not verbose))))
log_and_print("Current depth of program is {}".format(
current.depth))
log_and_print("Creating children for current node/program")
children_nodes = graph.get_all_children(current)
# prune if more than self.max_num_children
if len(children_nodes) > graph.max_num_children:
children_nodes = random.sample(
children_nodes,
k=graph.max_num_children) # sample without replacement
log_and_print("{} total children to train for current node".format(
len(children_nodes)))
for child_node in children_nodes:
child_start_time = time.time()
log_and_print("Training child program: {}".format(
print_program(child_node.program,
ignore_constants=(not verbose))))
is_neural = not graph.is_fully_symbolic(child_node.program)
dataset = trainset_neural if (
is_neural and trainset_neural is not None) else trainset
child_node.score = execute_and_train(child_node.program,
validset,
dataset,
train_config,
graph.output_type,
graph.output_size,
neural=is_neural,
device=device)
log_and_print(
"Time to train child {:.3f}".format(time.time() -
child_start_time))
num_children_trained += 1
log_and_print(
"{} total children trained".format(num_children_trained))
child_node.parent = current
child_node.children = []
order -= 1
child_node.order = order # insert order of exploration as tiebreaker for equivalent f-scores
current.children.append(child_node)
# computing path costs (f_scores)
child_f_score = child_node.cost + child_node.score # cost + heuristic
log_and_print("DEBUG: f-score {}".format(child_f_score))
if not is_neural and child_f_score < best_total_cost:
best_program = copy.deepcopy(child_node.program)
best_total_cost = child_f_score
best_programs_list.append({
"program": best_program,
"struct_cost": child_node.cost,
"score": child_node.score,
"path_cost": child_f_score,
"time": time.time() - start_time
})
log_and_print("New BEST program found:")
print_program_dict(best_programs_list[-1])
if is_neural:
assert child_node.depth < graph.max_depth
child_tuple = (child_f_score, order, child_node)
frontier.add(child_tuple)
# clean up frontier
frontier.sort(tup_idx=0)
while len(frontier) > 0 and frontier.peek(-1)[0] > best_total_cost:
frontier.pop(-1)
log_and_print("Frontier length is: {}".format(len(frontier)))
log_and_print("Total time elapsed is {:.3f}".format(time.time() -
start_time))
if best_program is None:
log_and_print("ERROR: no program found")
return best_programs_list
def __init__(self, **kwargs):
self.__dict__.update(kwargs)
if torch.cuda.is_available():
self.device = 'cuda:0'
print(self.device)
else:
self.device = 'cpu'
self.loss_weight = torch.tensor([float(w) for w in self.class_weights.split(',')]).to(self.device)
if self.exp_name == 'crim13' or 'bball' in self.exp_name:
# load input data
self.train_data = np.load(self.train_data)
self.test_data = np.load(self.test_data)
self.valid_data = None
self.train_labels = np.load(self.train_labels)
self.test_labels = np.load(self.test_labels)
self.valid_labels = None
assert self.train_data.shape[-1] == self.test_data.shape[-1] == self.input_size
if self.valid_data is not None and self.valid_labels is not None:
self.valid_data = np.load(self.valid_data)
self.valid_labels = np.load(self.valid_labels)
assert valid_data.shape[-1] == self.input_size
self.batched_trainset, self.validset, self.testset = prepare_datasets(self.train_data, self.valid_data, self.test_data, self.train_labels, self.valid_labels,
self.test_labels, normalize=self.normalize, train_valid_split=self.train_valid_split, batch_size=self.batch_size)
elif self.exp_name == 'mars_an':
#### start mars
train_datasets = self.train_data.split(",")
train_raw_features = []
train_raw_annotations = []
for fname in train_datasets:
data = np.load(fname, allow_pickle=True)
train_raw_features.extend(data["features"])
train_raw_annotations.extend(data["annotations"])
test_data = np.load(self.test_data, allow_pickle=True)
test_raw_features = test_data["features"]
test_raw_annotations = test_data["annotations"]
valid_raw_features = None
valid_raw_annotations = None
valid_labels = None
# Check the # of features of the first frame of the first video
assert len(train_raw_features[0][0]) == len(test_raw_features[0][0]) == self.input_size
if self.valid_data is not None:
valid_data = np.load(self.valid_data, allow_pickle=True)
valid_raw_features = valid_data["features"]
valid_raw_annotations = valid_data["annotations"]
assert len(valid_raw_features[0][0]) == self.input_size
behave_dict = read_into_dict('../near_code_7keypoints/data/MARS_data/behavior_assignments_3class.txt')
# Reshape the data to trajectories of length 100
train_features, train_labels = preprocess(train_raw_features, train_raw_annotations, self.train_labels, behave_dict)
test_features, test_labels = preprocess(test_raw_features, test_raw_annotations, self.train_labels, behave_dict)
if valid_raw_features is not None and valid_raw_annotations is not None:
valid_features, valid_labels = preprocess(valid_raw_features, valid_raw_annotations, self.train_labels, behave_dict)
self.batched_trainset, self.validset, self.testset = prepare_datasets(train_features, valid_features, test_features,
train_labels, valid_labels, test_labels,
normalize=self.normalize, train_valid_split=self.train_valid_split, batch_size=self.batch_size)
##### END MARS
else:
log_and_print('bad experiment name')
return
now = datetime.now()
self.timestamp = str(datetime.timestamp(now)).split('.')[0][4:]
log_and_print(self.timestamp)
full_exp_name = "{}_{}_{}_{}".format(
self.exp_name, self.algorithm, self.trial, self.timestamp) #unique timestamp for each near run
self.save_path = os.path.join(self.save_dir, full_exp_name)
if not os.path.exists(self.save_path):
os.makedirs(self.save_path)
init_logging(self.save_path)
visited_nodes = set() #dont visit smth thats already been visited
num_iter = 10 #todo make this a parameter later
for i in range(num_iter):
l = self.load_base_program() #populates self.base_program
log_and_print("Base program performance:")
self.evaluate_final()
self.hole_node_ind = self.neural_h()
self.hole_node = l[self.hole_node_ind]
log_and_print("Node selected: %d" % self.hole_node_ind)
# set up path to save program
# self.save_path = os.path.join(self.save_path, str(num_iter))
#run near
self.run_near(i)
#change base program name
self.base_program_name = os.path.join(self.save_path, "fullprogram_%d" % i)
#make it rly good
self.train_more_epochs(self.base_program_name)
def run_near(self):
# print(self.device)
train_config = {
'lr' : self.learning_rate,
'neural_epochs' : self.neural_epochs,
'symbolic_epochs' : self.symbolic_epochs,
'optimizer' : optim.Adam,
'lossfxn' : nn.CrossEntropyLoss(weight=self.loss_weight), #todo
'evalfxn' : label_correctness,
'num_labels' : self.num_labels
}
near_input_type = self.hole_node[0].input_type
near_output_type = self.hole_node[0].output_type
near_input_size = self.hole_node[0].input_size
near_output_size = self.hole_node[0].output_size
# Initialize program graph starting from trained NN
program_graph = ProgramGraph(DSL_DICT, CUSTOM_EDGE_COSTS, near_input_type, near_output_type, near_input_size, near_output_size,
self.max_num_units, self.min_num_units, self.max_num_children, self.max_depth, self.penalty, ite_beta=self.ite_beta)
# Initialize algorithm
algorithm = ASTAR_NEAR(frontier_capacity=self.frontier_capacity)
best_programs = algorithm.run(self.timestamp, self.base_program_name, self.hole_node_ind,
program_graph, self.batched_trainset, self.validset, train_config, self.device)
best_program_str = []
if self.algorithm == "rnn":
# special case for RNN baseline
best_program = best_programs
else:
# Print all best programs found
log_and_print("\n")
log_and_print("BEST programs found:")
for item in best_programs:
program_struct = print_program(item["program"], ignore_constants=True)
program_info = "struct_cost {:.4f} | score {:.4f} | path_cost {:.4f} | time {:.4f}".format(
item["struct_cost"], item["score"], item["path_cost"], item["time"])
best_program_str.append((program_struct, program_info))
print_program_dict(item)
best_program = best_programs[-1]["program"]
# Save best programs
f = open(os.path.join(self.save_path, "best_programs.txt"),"w")
f.write( str(best_program_str) )
f.close()
self.program_path = os.path.join(self.save_path, "subprogram.p")
pickle.dump(best_program, open(self.program_path, "wb"))
self.full_path = os.path.join(self.save_path, "fullprogram.p")
if self.device == 'cpu':
base_program = CPU_Unpickler(open("%s.p" % self.base_program_name, "rb")).load()
else:
base_program = pickle.load(open("%s.p" % self.base_program_name, "rb"))
curr_level = 0
l = []
traverse(base_program.submodules,l)
curr_program = base_program.submodules
change_key(base_program.submodules, [], self.hole_node_ind, best_program.submodules["program"])
pickle.dump(base_program, open(self.full_path, "wb"))
# Save parameters
f = open(os.path.join(self.save_path, "parameters.txt"),"w")
parameters = ['input_type', 'output_type', 'input_size', 'output_size', 'num_labels', 'neural_units', 'max_num_units',
'min_num_units', 'max_num_children', 'max_depth', 'penalty', 'ite_beta', 'train_valid_split', 'normalize', 'batch_size',
'learning_rate', 'neural_epochs', 'symbolic_epochs', 'lossfxn', 'class_weights', 'num_iter', 'num_f_epochs', 'algorithm',
'frontier_capacity', 'initial_depth', 'performance_multiplier', 'depth_bias', 'exponent_bias', 'num_mc_samples', 'max_num_programs',
'population_size', 'selection_size', 'num_gens', 'total_eval', 'mutation_prob', 'max_enum_depth', 'exp_id', 'base_program_name', 'hole_node_ind']
for p in parameters:
f.write( p + ': ' + str(self.__dict__[p]) + '\n' )
f.close()
def run(self, graph, trainset, validset, train_config, device, verbose=False):
assert isinstance(graph, ProgramGraph)
symbolic_programs = []
enum_depth = 1
while len(symbolic_programs) < self.max_num_programs:
print("DEBUG: starting enumerative synthesis with depth {}".format(enum_depth))
symbolic_programs = self.enumerate2depth(graph, enum_depth)
print("DEBUG: {} programs found".format(len(symbolic_programs)))
enum_depth += 1
if enum_depth > graph.max_depth:
break
log_and_print("Symbolic Synthesis: generated {}/{} symbolic programs from candidate program.".format(
len(symbolic_programs), self.max_num_programs))
total_eval = min(self.max_num_programs, len(symbolic_programs))
symbolic_programs.sort(key=lambda x: x["struct_cost"])
symbolic_programs = symbolic_programs[:total_eval]
best_program = None
best_total_cost = float('inf')
best_programs_list = []
start_time = time.time()
num_programs_trained = 1
for prog_dict in symbolic_programs:
child_start_time = time.time()
candidate = prog_dict["program"]
log_and_print("Training candidate program ({}/{}) {}".format(
num_programs_trained, total_eval, print_program(candidate, ignore_constants=(not verbose))))
num_programs_trained += 1
score = execute_and_train(candidate, validset, trainset, train_config,
graph.output_type, graph.output_size, neural=False, device=device)
total_cost = score + prog_dict["struct_cost"]
log_and_print("Structural cost is {} with structural penalty {}".format(prog_dict["struct_cost"], graph.penalty))
log_and_print("Time to train child {:.3f}".format(time.time()-child_start_time))
log_and_print("Total time elapsed is: {:.3f}".format(time.time()-start_time))
if total_cost < best_total_cost:
best_program = copy.deepcopy(prog_dict["program"])
best_total_cost = total_cost
prog_dict["score"] = score
prog_dict["path_cost"] = total_cost
prog_dict["time"] = time.time()-start_time
best_programs_list.append(prog_dict)
log_and_print("New BEST program found:")
print_program_dict(best_programs_list[-1])
return best_programs_list
def run(self,
graph,
trainset,
validset,
train_config,
device,
verbose=False):
assert isinstance(graph, ProgramGraph)
self.input_size = graph.input_size
self.output_size = graph.output_size
best_program = None
best_score = float('inf')
best_total = float('inf')
best_programs_list = []
start_time = time.time()
log_and_print("Generating all programs up to specified depth.")
all_generated_programs = self.enumerative_synthesis(
graph, self.max_enum_depth)
random.shuffle(all_generated_programs)
current_population = all_generated_programs[:self.population_size]
total_trained_programs = 0
def crossover_operation(listofprogramdicts):
new_prog_dict = {}
parents = random.choices(listofprogramdicts, k=2)
prog_dict1, prog_dict2 = parents[0], parents[1]
xover1 = random.randint(1, prog_dict1['depth'])
xover2 = random.randint(2, prog_dict2['depth'])
print('DEBUG: trying new crossover')
new_program, crossed_over = self.crossover_helper(
prog_dict1['program'], prog_dict2['program'], xover1, xover2)
if crossed_over:
log_and_print("Crossing over the following programs:")
log_and_print(
print_program(prog_dict1['program'],
ignore_constants=True))
log_and_print(
print_program(prog_dict2['program'],
ignore_constants=True))
new_prog_dict["program"] = new_program
new_prog_dict["struct_cost"], new_prog_dict[
"depth"] = graph.compute_program_cost(new_program)
log_and_print("Result has structural cost {:.4f}:".format(
new_prog_dict["struct_cost"]))
log_and_print(
print_program(new_prog_dict['program'],
ignore_constants=True))
return new_prog_dict
else:
return None
def mutation_operation(mod_prog_dict):
new_prog_dict = {}
mutation_point = random.randrange(1, mod_prog_dict['depth'])
new_prog_dict['program'] = self.mutation_helper(
graph,
mod_prog_dict['program'],
mod_prog_dict['depth'],
mutation_point,
max_enumeration_depth=self.max_enum_depth)
new_prog_dict["struct_cost"], new_prog_dict[
"depth"] = graph.compute_program_cost(new_prog_dict['program'])
return new_prog_dict
for gen_idx in range(self.num_gens):
#evaluation operation: train each program and evaluate it.
log_and_print(
"Training generation {}'s population of programs.".format(
gen_idx + 1))
for programdict in current_population:
total_trained_programs += 1
child_start_time = time.time()
log_and_print("Training candidate program ({}/{}) {}".format(
total_trained_programs, self.total_eval,
print_program(programdict['program'],
ignore_constants=(not verbose))))
score = execute_and_train(programdict['program'],
validset,
trainset,
train_config,
graph.output_type,
graph.output_size,
neural=False,
device=device)
log_and_print(
"Structural cost is {} with structural penalty {}".format(
programdict["struct_cost"], graph.penalty))
log_and_print(
"Time to train child {:.3f}".format(time.time() -
child_start_time))
log_and_print("Total time elapsed is: {}".format(time.time() -
start_time))
programdict["score"] = score
programdict["path_cost"] = score + programdict["struct_cost"]
programdict["time"] = time.time() - start_time
if programdict["path_cost"] < best_total:
best_program = copy.deepcopy(programdict["program"])
best_total = score + programdict["struct_cost"]
best_score = score
best_cost = programdict["struct_cost"]
best_programs_list.append(copy.deepcopy(programdict))
log_and_print("New BEST program found:")
print_program_dict(best_programs_list[-1])
if total_trained_programs > self.total_eval:
break
#select the best programs based on cost + score.
current_population.sort(key=lambda x: x["path_cost"])
selected_population = current_population[:self.selection_size]
#perform crossover on the selected population
crossed_population = []
log_and_print("Beginning crossover operation.")
while len(crossed_population) < self.population_size:
new_prog_dict = crossover_operation(selected_population)
if new_prog_dict is not None:
crossed_population.append(new_prog_dict)
#perform mutations on the crossed population
current_population = []
for crossed_prog_dict in crossed_population:
if random.random() < self.mutation_prob:
log_and_print("Mutating program.")
current_population.append(
mutation_operation(crossed_prog_dict))
else:
current_population.append(crossed_prog_dict)
return best_programs_list
def execute_and_train_og(base_program_name,
validset,
trainset,
train_config,
output_type,
output_size,
neural=False,
device='cpu',
use_valid_score=False,
print_every=60):
if device == 'cpu':
program = CPU_Unpickler(open("%s.p" % base_program_name, "rb")).load()
else:
program = pickle.load(open("%s.p" % base_program_name, "rb"))
lr = train_config['lr']
neural_epochs = train_config['neural_epochs']
symbolic_epochs = train_config['symbolic_epochs']
optimizer = train_config['optimizer']
lossfxn = train_config['lossfxn']
evalfxn = train_config['evalfxn']
num_labels = train_config['num_labels']
num_epochs = neural_epochs if neural else symbolic_epochs
# initialize optimizer
curr_optim = init_optimizer(program, optimizer, lr)
# prepare validation set
validation_input, validation_output = map(list, zip(*validset))
validation_true_vals = torch.flatten(
torch.stack(validation_output)).float().to(device)
# TODO a little hacky, but easiest solution for now
if isinstance(lossfxn, nn.CrossEntropyLoss):
validation_true_vals = validation_true_vals.long()
best_program = None
best_metric = float('inf')
best_additional_params = {}
for epoch in range(1, num_epochs + 1):
for batchidx in range(len(trainset)):
batch_input, batch_output = map(list, zip(*trainset[batchidx]))
true_vals = torch.flatten(
torch.stack(batch_output)).float().to(device)
predicted_vals = process_batch(program, batch_input, output_type,
output_size, device)
# TODO a little hacky, but easiest solution for now
if isinstance(lossfxn, nn.CrossEntropyLoss):
true_vals = true_vals.long()
# print(predicted_vals.shape, true_vals.shape)
loss = lossfxn(predicted_vals, true_vals)
curr_optim.zero_grad()
loss.backward()
curr_optim.step()
# if batchidx % print_every == 0 or batchidx == 0:
# log_and_print('Epoch [{}/{}], Loss: {:.4f}'.format(epoch, num_epochs, loss.item()))
# check score on validation set
with torch.no_grad():
predicted_vals = process_batch(program, validation_input,
output_type, output_size, device)
metric, additional_params = evalfxn(predicted_vals,
validation_true_vals,
num_labels=num_labels)
if use_valid_score:
if metric < best_metric:
best_program = copy.deepcopy(program)
best_metric = metric
best_additional_params = additional_params
else:
best_program = copy.deepcopy(program)
best_metric = metric
best_additional_params = additional_params
# select model with best validation score
program = copy.deepcopy(best_program)
log_and_print("Validation score is: {:.4f}".format(best_metric))
log_and_print("Average f1-score is: {:.4f}".format(1 - best_metric))
log_and_print("Hamming accuracy is: {:.4f}".format(
best_additional_params['hamming_accuracy']))
return best_metric, program
#mcheng substitute nonterminals with NNs
def execute_and_train(base_program,
program,
validset,
trainset,
train_config,
output_type,
output_size,
neural=False,
device='cpu',
use_valid_score=False,
print_every=60):
# print('enter training initial')
lr = train_config['lr']
neural_epochs = train_config['neural_epochs']
symbolic_epochs = train_config['symbolic_epochs']
optimizer = train_config['optimizer']
lossfxn = train_config['lossfxn']
evalfxn = train_config['evalfxn']
num_labels = train_config['num_labels']
num_epochs = neural_epochs if neural else symbolic_epochs
# initialize optimizer
curr_optim = init_optimizer(program, optimizer, lr)
# prepare validation set
validation_input, validation_output = map(list, zip(*validset))
validation_true_vals = torch.flatten(
torch.stack(validation_output)).float().to(device)
# TODO a little hacky, but easiest solution for now
if isinstance(lossfxn, nn.CrossEntropyLoss):
validation_true_vals = validation_true_vals.long()
best_program = None
best_metric = float('inf')
best_additional_params = {}
original_output_type = base_program.program.output_type
original_output_size = base_program.program.output_size
losses = []
training_f1 = []
for epoch in range(1, num_epochs + 1):
temp_l = 0
# temp_f = 0
for batchidx in range(len(trainset)):
batch_input, batch_output = map(list, zip(*trainset[batchidx]))
true_vals = torch.flatten(
torch.stack(batch_output)).float().to(device)
predicted_vals = process_batch(base_program, batch_input,
original_output_type,
original_output_size,
device) #fix lol
# TODO a little hacky, but easiest solution for now
if isinstance(lossfxn, nn.CrossEntropyLoss):
true_vals = true_vals.long()
# print(predicted_vals.shape, true_vals.shape)
loss = lossfxn(predicted_vals, true_vals)
training_metric, _ = evalfxn(predicted_vals,
true_vals,
num_labels=num_labels)
# print('tutu metric')
temp_l += float(loss.data)
# temp_f += training_metric
# print(training_metric)
curr_optim.zero_grad()
loss.backward()
curr_optim.step()
# if batchidx % print_every == 0 or batchidx == 0:
# log_and_print('Epoch [{}/{}], Loss: {:.4f}'.format(epoch, num_epochs, loss.item()))
# check score on validation set
losses.append(temp_l / len(trainset))
# training_f1.append(temp_f/len(trainset))
with torch.no_grad():
predicted_vals = process_batch(base_program, validation_input,
original_output_type,
original_output_size, device)
metric, additional_params = evalfxn(predicted_vals,
validation_true_vals,
num_labels=num_labels)
if use_valid_score:
if metric < best_metric:
best_program = copy.deepcopy(program)
best_metric = metric
best_additional_params = additional_params
else:
best_program = copy.deepcopy(program)
best_metric = metric
best_additional_params = additional_params
# select model with best validation score
program = copy.deepcopy(best_program)
log_and_print("Training loss is: {:.4f}".format(loss))
log_and_print("Validation score is: {:.4f}".format(best_metric))
log_and_print("Average f1-score is: {:.4f}".format(1 - best_metric))
log_and_print("Hamming accuracy is: {:.4f}".format(
best_additional_params['hamming_accuracy']))
return best_metric, losses, training_f1
def __init__(self):
log_and_print(
"Root node is Start(ListToListModule) or Start(ListToAtomModule), both implemented with an RNN."
)
log_and_print(
"Be sure to set neural_epochs and max_num_units accordingly.\n")
