def step(self): self.nnet.eval() #Find lowest cost state lowest_cost_state: State = list(self.opened.keys())[0] for item in self.opened: if self.opened[lowest_cost_state][0] + self.opened[lowest_cost_state][1] > self.opened[item][0] + self.opened[item][1]: lowest_cost_state = item self.curr_state = lowest_cost_state #Add Children to open children = self.env.expand([lowest_cost_state])[0][0] children_output = self.nnet(states_nnet_to_pytorch_input(self.env.state_to_nnet_input(children), 'cpu').float()).cpu().data.numpy() for i, child in enumerate(children): adjusted_huer: float = h_prime(children_output[i][0], self.o_c_max, self.cutoffs) #print("Adjusted huer: ",adjusted_huer) if child in self.closed: if (adjusted_huer + self.opened[lowest_cost_state][1]+1.0) < (self.closed[child][0] + self.closed[child][1]): self.opened[child] = adjusted_huer, self.opened[lowest_cost_state][1]+1.0 del self.closed[child] else: continue elif child in self.opened: if (adjusted_huer + self.opened[lowest_cost_state][1]+1.0) < (self.opened[child][0] + self.opened[child][1]): self.opened[child] = adjusted_huer, self.opened[lowest_cost_state][1]+1.0 else: self.opened[child] = adjusted_huer, self.opened[lowest_cost_state][1]+1.0
def main(): # get environment env: Environment = env_utils.get_environment("puzzle8") # get nnet model nnet: nn.Module = get_nnet_model() device = torch.device('cpu') batch_size: int = 100 num_itrs: int = 10000 # get data print("Preparing Data\n") data = pickle.load(open("data/data.pkl", "rb")) states_nnet, outputs = sample_training_data(data['states'], data['output'], env, batch_size*num_itrs) # train with supervised learning print("Training DNN\n") nnet.train() train_nnet(nnet, states_nnet, outputs, batch_size, num_itrs, 0) # get performance print("Evaluating DNN\n") nnet.eval() for cost_to_go in np.unique(data["output"]): idxs_targ: np.array = np.where(data["output"] == cost_to_go)[0] states_targ: List[State] = [data["states"][idx] for idx in idxs_targ] states_targ_nnet: np.ndarray = env.state_to_nnet_input(states_targ) out_nnet = nnet(states_nnet_to_pytorch_input(states_targ_nnet, device)).cpu().data.numpy() mse = float(np.mean((out_nnet - cost_to_go) ** 2)) print("Cost-To-Go: %i, Ave DNN Output: %f, MSE: %f" % (cost_to_go, float(np.mean(out_nnet)), mse))
def evaluate_cost_to_go(nnet, device, env: Environment, states: List[State], outputs: np.array, f): for cost_to_go in np.unique(outputs): idxs_targ: np.array = np.where(outputs == cost_to_go)[0] states_targ: List[State] = [states[idx] for idx in idxs_targ] states_targ_nnet: np.ndarray = env.state_to_nnet_input(states_targ) out_nnet = nnet(states_nnet_to_pytorch_input( states_targ_nnet, device).float()).cpu().data.numpy() mse = float(np.mean((out_nnet - cost_to_go) ** 2)) f.write("Cost-To-Go: %i, Ave DNN Output: %f, MSE: %f \n" % (cost_to_go, float(np.mean(out_nnet)), mse))
def main(): torch.set_num_threads(1) # get environment env: Environment = env_utils.get_environment("puzzle8") # get nnet model nnet: nn.Module = get_nnet_model() # get optimizer and lr scheduler optimizer = torch.optim.Adam(nnet.parameters(), lr=1e-3) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=100, gamma=0.996) criterion = nn.MSELoss() device = torch.device('cpu') batch_size: int = 100 num_itrs: int = 10000 with open("sample_outputs/exercise_1_akash.txt", 'w') as f: # get data f.write("Preparing Data\n") data = pickle.load(open("data/data.pkl", "rb")) states_nnet, outputs = sample_training_data(data['states'], data['output'], env, batch_size * num_itrs) # train with supervised learning f.write("Training DNN\n") nnet.train() train_nnet(nnet, states_nnet, outputs, batch_size, num_itrs, 0, criterion, optimizer, scheduler, f) # get performance f.write("Evaluating DNN\n") nnet.eval() for cost_to_go in np.unique(data["output"]): idxs_targ: np.array = np.where(data["output"] == cost_to_go)[0] states_targ: List[State] = [ data["states"][idx] for idx in idxs_targ ] states_targ_nnet: np.ndarray = env.state_to_nnet_input(states_targ) out_nnet = nnet( states_nnet_to_pytorch_input( states_targ_nnet, device).float()).cpu().data.numpy() mse = float(np.mean((out_nnet - cost_to_go)**2)) f.write("Cost-To-Go: %i, Ave DNN Output: %f, MSE: %f \n" % (cost_to_go, float(np.mean(out_nnet)), mse))
def generate_plot(nnet: nn.Module(), device, env: Environment, states: List[State], outputs: np.array): nnet.eval() states_targ_nnet: np.ndarray = env.state_to_nnet_input(states) out_nnet = nnet(states_nnet_to_pytorch_input(states_targ_nnet, device).float()).cpu().data.numpy() out_nnet, _ = flatten(out_nnet) outputs, _ = flatten(outputs) out_nnet_array = np.array(out_nnet) outputs_array = np.array(outputs) random_indexs = list(range(len(out_nnet_array))) random.shuffle(random_indexs) random_states: np.ndarray = [] sample_expected: np.ndarray = [] sample_outputs: np.ndarray = [] for i in range(100): random_states.append(states[random_indexs[i]]) sample_expected.append(outputs_array[random_indexs[i]]) sample_outputs.append(out_nnet_array[random_indexs[i]]) h_new: np.ndarray = approx_admissible_conv(env, nnet, out_nnet_array, outputs_array, states, random_states, sample_outputs, sample_expected) #before, after = plt.subplots() plt.scatter(sample_expected, sample_outputs, c = '000000', linewidths = 0.1) #plt.plot([0,0],[30,30], c = 'g') plt.axline([0,0],[30,30], linewidth =3, c = 'g') plt.ylabel('NNet output') plt.xlabel('Expected value') plt.title("Output vs Expected") plt.show() #before.savefig("preconversion.pdf") plt.scatter(sample_expected, h_new, c = '000000', linewidths = 0.1) plt.axline([0,0],[30,30], linewidth =3, c = 'g') plt.ylabel('Converted output') plt.xlabel('Expected value') plt.title("Converted Output vs Expected") plt.show()