def test(model): game_state = GameState() # initial action is do nothing action = torch.zeros([model.number_of_actions], dtype=torch.float32) action[0] = 1 image_data, reward, terminal = game_state.frame_step(action) image_data = resize_and_bgr2gray(image_data) image_data = image_to_tensor(image_data) state = torch.cat( (image_data, image_data, image_data, image_data)).unsqueeze(0) while True: # get output from the neural network output = model(state)[0] action = torch.zeros([model.number_of_actions], dtype=torch.float32) if torch.cuda.is_available(): # put on GPU if CUDA is available action = action.cuda() # get action action_index = torch.argmax(output) if torch.cuda.is_available(): # put on GPU if CUDA is available action_index = action_index.cuda() action[action_index] = 1 # get next state image_data_1, reward, terminal = game_state.frame_step(action) image_data_1 = resize_and_bgr2gray(image_data_1) image_data_1 = image_to_tensor(image_data_1) state_1 = torch.cat( (state.squeeze(0)[1:, :, :], image_data_1)).unsqueeze(0) # set state to be state_1 state = state_1
def test(model): game_state = GameState() action = torch.zeros([model.number_of_actions], dtype=torch.float32) action[0] = 1 img_data, reward, done = game_state.frame_step(action) img_data = preprocess(img_data) img_data = convert_img_to_tensor(img_data) state = torch.cat((img_data, img_data, img_data, img_data)).unsqueeze(0) while True: output = model(state)[0] action = torch.zeros([model.number_of_actions], dtype=torch.float32) action_index = torch.argmax(output) action[action_index] = 1 next_img_data, reward, done = game_state.frame_step(action) next_img_data = preprocess(next_img_data) next_img_data = convert_img_to_tensor(img_data) next_state = torch.cat( (state.squeeze(0)[1:, :, :], next_img_data)).unsqueeze(0) state = next_state
def test(model): game_state = GameState() # initial action is do nothing action = torch.zeros([model.number_of_actions], dtype=torch.float32) action[0] = 1 image_data, reward, terminal = game_state.frame_step(action) image_data = resize_and_bgr2gray(image_data) image_data = image_to_tensor(image_data) state = torch.cat( (image_data, image_data, image_data, image_data)).unsqueeze(0) epsilon = 0 total_reward = 0 max_reward = 0 cur_reward = 0 rewards = [] while True: if epsilon >= 10: break # get output from the neural network output = model(state)[0] action = torch.zeros([model.number_of_actions], dtype=torch.float32) if torch.cuda.is_available(): # put on GPU if CUDA is available action = action.cuda() # get action action_index = torch.argmax(output) if torch.cuda.is_available(): # put on GPU if CUDA is available action_index = action_index.cuda() action[action_index] = 1 # get next state image_data_1, reward, terminal = game_state.frame_step(action) if terminal: epsilon += 1 if cur_reward > max_reward: max_reward = cur_reward rewards.append(cur_reward) cur_reward = 0 if reward > 0.1: total_reward += reward cur_reward += reward image_data_1 = resize_and_bgr2gray(image_data_1) image_data_1 = image_to_tensor(image_data_1) state_1 = torch.cat( (state.squeeze(0)[1:, :, :], image_data_1)).unsqueeze(0) # set state to be state_1 state = state_1 print('reward:', total_reward / 10.0) print('max reward:', max_reward) print('standard deviation:', np.std(rewards, axis=0))
def test(model): game_state = GameState() # initial action is do nothing action = torch.zeros([model.number_of_actions], dtype=torch.float32) action[0] = 1 image_data, reward, terminal = game_state.frame_step(action) image_origin = resize_and_bgr(image_data) image_data = resize_and_bgr2gray(image_data) image_data = image_to_tensor(image_data) state = torch.cat((image_data, image_data, image_data, image_data)).unsqueeze(0) while True: # get output from the neural network output, conv_output = model(state) heatmap = np.mean(np.mean(conv_output.detach().numpy(), axis=0), axis=0) heatmap = np.maximum(heatmap, 0) heatmap /= np.max(heatmap) heatmap = cv2.resize(heatmap, (image_origin.shape[1], image_origin.shape[0])) heatmap = np.fliplr(rotate(heatmap, 90*3)) image_origin = np.fliplr(rotate(image_origin, 90*3)) plt.imshow(image_origin) plt.imshow(heatmap, cmap=plt.cm.jet, alpha=0.5, interpolation='nearest', vmin=0, vmax=1) plt.show() action = torch.zeros([model.number_of_actions], dtype=torch.float32) if torch.cuda.is_available(): # put on GPU if CUDA is available action = action.cuda() # get action action_index = torch.argmax(output) if torch.cuda.is_available(): # put on GPU if CUDA is available action_index = action_index.cuda() action[action_index] = 1 # get next state image_data_1, reward, terminal = game_state.frame_step(action) image_origin = resize_and_bgr(image_data_1) image_data_1 = resize_and_bgr2gray(image_data_1) image_data_1 = image_to_tensor(image_data_1) state_1 = torch.cat((state.squeeze(0)[1:, :, :], image_data_1)).unsqueeze(0) # set state to be state_1 state = state_1
def train(model, start): # define Adam optimizer optimizer = optim.Adam(model.parameters(), lr=1e-6) # initialize mean squared error loss criterion = nn.MSELoss() # instantiate game game_state = GameState() # initialize replay memory replay_memory = [] # initial action is do nothing action = torch.zeros([model.number_of_actions], dtype=torch.float32) action[0] = 1 image_data, reward, terminal = game_state.frame_step(action) image_data = resize_and_bgr2gray(image_data) image_data = image_to_tensor(image_data) state = torch.cat( (image_data, image_data, image_data, image_data)).unsqueeze(0) # initialize epsilon value epsilon = model.initial_epsilon iteration = 0 epsilon_decrements = np.linspace(model.initial_epsilon, model.final_epsilon, model.number_of_iterations) # main infinite loop while iteration < model.number_of_iterations: # get output from the neural network output = model(state)[0] # initialize action action = torch.zeros([model.number_of_actions], dtype=torch.float32) if torch.cuda.is_available(): # put on GPU if CUDA is available action = action.cuda() # epsilon greedy exploration random_action = random.random() <= epsilon if random_action: print("Performed random action!") action_index = [ torch.randint( model.number_of_actions, torch.Size([]), dtype=torch.int) if random_action else torch.argmax(output) ][0] if torch.cuda.is_available(): # put on GPU if CUDA is available action_index = action_index.cuda() action[action_index] = 1 # get next state and reward image_data_1, reward, terminal = game_state.frame_step(action) image_data_1 = resize_and_bgr2gray(image_data_1) image_data_1 = image_to_tensor(image_data_1) state_1 = torch.cat( (state.squeeze(0)[1:, :, :], image_data_1)).unsqueeze(0) action = action.unsqueeze(0) reward = torch.from_numpy(np.array([reward], dtype=np.float32)).unsqueeze(0) # save transition to replay memory replay_memory.append((state, action, reward, state_1, terminal)) # if replay memory is full, remove the oldest transition if len(replay_memory) > model.replay_memory_size: replay_memory.pop(0) # epsilon annealing epsilon = epsilon_decrements[iteration] # sample random minibatch minibatch = random.sample( replay_memory, min(len(replay_memory), model.minibatch_size)) # unpack minibatch state_batch = torch.cat(tuple(d[0] for d in minibatch)) action_batch = torch.cat(tuple(d[1] for d in minibatch)) reward_batch = torch.cat(tuple(d[2] for d in minibatch)) state_1_batch = torch.cat(tuple(d[3] for d in minibatch)) if torch.cuda.is_available(): # put on GPU if CUDA is available state_batch = state_batch.cuda() action_batch = action_batch.cuda() reward_batch = reward_batch.cuda() state_1_batch = state_1_batch.cuda() # get output for the next state output_1_batch = model(state_1_batch) # set y_j to r_j for terminal state, otherwise to r_j + gamma*max(Q) y_batch = torch.cat( tuple(reward_batch[i] if minibatch[i][4] else reward_batch[i] + model.gamma * torch.max(output_1_batch[i]) for i in range(len(minibatch)))) # extract Q-value q_value = torch.sum(model(state_batch) * action_batch, dim=1) # PyTorch accumulates gradients by default, so they need to be reset in each pass optimizer.zero_grad() # returns a new Tensor, detached from the current graph, the result will never require gradient y_batch = y_batch.detach() # calculate loss loss = criterion(q_value, y_batch) # do backward pass loss.backward() optimizer.step() # set state to be state_1 state = state_1 iteration += 1 if iteration % 25000 == 0: torch.save( model, "pretrained_model/current_model_" + str(iteration) + ".pth") print("iteration:", iteration, "elapsed time:", time.time() - start, "epsilon:", epsilon, "action:", action_index.cpu().detach().numpy(), "reward:", reward.numpy()[0][0], "Q max:", np.max(output.cpu().detach().numpy()))
def train(): env = GameState() # num_inputs = env.observation_space.shape[0] num_inputs = 3136 num_actions = 2 print('state size:', num_inputs) print('action size:', num_actions) online_net = DRQN(num_inputs, num_actions) target_net = DRQN(num_inputs, num_actions) update_target_model(online_net, target_net) optimizer = optim.Adam(online_net.parameters(), lr=lr) if torch.cuda.is_available(): # put on GPU if CUDA is available online_net = online_net.cuda() target_net = target_net.cuda() online_net.train() target_net.train() memory = Memory(replay_memory_capacity) epsilon = 1.0 loss = 0 iteration = 0 while iteration < 2000000: done = False action = torch.zeros([2], dtype=torch.float32) action[0] = 1 image_data, reward, done = env.frame_step(action) image_data = resize_and_bgr2gray(image_data) image_data = image_to_tensor(image_data) state = image_data state = torch.Tensor(state) if torch.cuda.is_available(): state = state.cuda() hidden = None while not done: action, hidden, action_index = get_action(state, target_net, epsilon, env, hidden) image_data, reward, done = env.frame_step(action) image_data = resize_and_bgr2gray(image_data) image_data = image_to_tensor(image_data) next_state = image_data next_state = torch.Tensor(next_state) if torch.cuda.is_available(): next_state = next_state.cuda() mask = 0 if done else 1 reward = reward if not done else -1 memory.push(state, next_state, action_index, reward, mask) state = next_state if iteration > initial_exploration and len(memory) > batch_size: epsilon -= 0.00005 epsilon = max(epsilon, 0.1) batch = memory.sample(batch_size) loss = DRQN.train_model(online_net, target_net, optimizer, batch) if iteration % update_target == 0: print('iteration: {}, update model'.format(iteration)) update_target_model(online_net, target_net) iteration += 1 if iteration % 25000 == 0: torch.save(online_net, "pretrained_model/current_model_" + str(iteration) + ".pth") print('iteration: {}'.format(iteration))
def test(): cuda_is_available = torch.cuda.is_available() env = GameState() # num_inputs = env.observation_space.shape[0] num_inputs = 3136 num_actions = 2 print('state size:', num_inputs) print('action size:', num_actions) model = torch.load( 'pretrained_model/current_model_2000000.pth', map_location='cpu' if not cuda_is_available else None ).eval() if torch.cuda.is_available(): # put on GPU if CUDA is available model = model.cuda() action = torch.zeros([2], dtype=torch.float32) action[0] = 1 image_data, reward, done = env.frame_step(action) image_data = resize_and_bgr2gray(image_data) image_data = image_to_tensor(image_data) state = image_data state = torch.Tensor(state) if torch.cuda.is_available(): state = state.cuda() hidden = None epsilon = 0 total_reward = 0 max_reward = 0 cur_reward = 0 rewards = [] while True: if epsilon >= 10: break action, hidden, action_index = get_action(state, model, 0, env, hidden) image_data, reward, done = env.frame_step(action) image_data = resize_and_bgr2gray(image_data) image_data = image_to_tensor(image_data) next_state = image_data next_state = torch.Tensor(next_state) if torch.cuda.is_available(): next_state = next_state.cuda() state = next_state if done: epsilon += 1 if cur_reward > max_reward: max_reward = cur_reward rewards.append(cur_reward) cur_reward = 0 if reward > 0.1: total_reward += reward cur_reward += reward print('reward:', total_reward/10.0) print('max reward:', max_reward) print('standard deviation:', np.std(rewards, axis=0))
def train(model): device = torch.device( "cuda") if torch.cuda.is_available() else torch.device("cpu") # instantiate game game_state = GameState() agentBird = AgentBird(ACTION_NUM, model) # initial action is do nothing # [1, 0] represents "Do nothing" # [0, 1] represents "Fly up" action = torch.zeros([ACTION_NUM], dtype=torch.float32) action[0] = 1 state_image, reward, terminal, state_score = game_state.frame_step(action) image = preprocess_image(state_image) state = torch.cat((image, image, image, image)).unsqueeze(0) run = 1 i = 0 while True: #for i in range(NUM_ITERATIONS): # Select and perform an action action_idx, q_values = agentBird.predict_action(state) action = torch.zeros([ACTION_NUM], dtype=torch.float32) action[action_idx] = 1 # get next state and reward state_image_1, reward, terminal, state_score = game_state.frame_step( action) image_1 = preprocess_image(state_image_1) state_1 = torch.cat((state.squeeze(0)[1:, :, :], image_1)).unsqueeze(0) score = reward reward = torch.from_numpy(np.array([reward], dtype=np.float32)).unsqueeze(0) reward = torch.tensor([reward]) reward = reward.to(device) action = action.unsqueeze(0) action = action.to(device) # Store the transition in memory agentBird.add_memory(state, action, reward, state_1, terminal) agentBird.update_exploration_rate(i) # Perform one step of the optimization (on the target network) loss = agentBird.experience_replay() print("iteration " + str(i) + ", exploration: " + str(agentBird.exploration_rate) + ", Q max:" + str(np.max(q_values.cpu().detach().numpy())) + ", action:" + str(action_idx) + ", reward:" + str(score)) writer.add_scalar('Q_value', np.max(q_values.cpu().detach().numpy()), i) writer.add_scalar('loss', float(loss), i) # Move to the next state state = state_1 i += 1 if terminal: run += 1 print("episode " + str(run) + ", Score: " + str(state_score)) writer.add_scalar('Score', state_score, run) if i % 50000 == 0: date_today = date.today() curr_time = datetime.now() formatted_time = curr_time.strftime('%H%M%S') save_model = agentBird.return_model() torch.save( save_model, "pretrained_model/easy_model_" + str(i) + "_" + str(date_today) + "_" + str(formatted_time) + ".pth")
def train(model, start): optimizer = optim.Adam(model.parameters(), lr=1e-6) criterion = nn.MSELoss() game_state = GameState() replay_memory = [] action = torch.zeros([model.number_of_actions], dtype=torch.float32) action[0] = 1 img_data, reward, done = game_state.frame_step(action) img_data = preprocess(img_data) img_data = convert_img_to_tensor(img_data) state = torch.cat((img_data, img_data, img_data, img_data)).unsqueeze(0) epsilon = model.epsilon1 iteration = 0 epsilon_decrements = np.linspace(model.epsilon1, model.epsilon2, model.number_of_iterations) while iteration < model.number_of_iterations: output = model(state)[0] action = torch.zeros([model.number_of_actions], dtype=torch.float32) random_action = random.random() <= epsilon if random_action: print("Performed random action!") action_index = [ torch.randint( model.number_of_actions, torch.Size([]), dtype=torch.int) if random_action else torch.argmax(output) ][0] action[action_index] = 1 next_img_data, reward, done = game_state.frame_step(action) next_img_data = preprocess(next_img_data) next_img_data = convert_img_to_tensor(next_img_data) next_state = torch.cat( (state.squeeze(0)[1:, :, :], next_img_data)).unsqueeze(0) action = action.unsqueeze(0) reward = torch.from_numpy(np.array([reward], dtype=np.float32)).unsqueeze(0) replay_memory.append((state, action, reward, next_state, terminal)) if len(replay_memory) > model.replay_memory_size: replay_memory.pop(0) epsilon = epsilon_decrements[iteration] batch = random.sample(replay_memory, min(len(replay_memory), model.minibatch_size)) state_memory = torch.cat(tuple(d[0] for d in batch)) action_memory = torch.cat(tuple(d[1] for d in batch)) reward_memory = torch.cat(tuple(d[2] for d in batch)) next_state_memory = torch.cat(tuple(d[3] for d in batch)) output_memory = model(next_state_memory) y_memory = torch.cat( tuple(reward_memory[i] if minibatch[i][4] else reward_memory[i] + model.gamma * torch.max(output_memory[i]) for i in range(len(minibatch)))) q_value = torch.sum(model(state_memory) * action_memory, dim=1) optimizer.zero_grad() y_memory = y_memory.detach() loss = criterion(q_value, y_memory) loss.backward() optimizer.step() state = next_state iteration += 1 print("iteration:", iteration, "elapsed time:", time.time() - start, "epsilon:", epsilon, "action:", action_index.cpu().detach().numpy(), "reward:", reward.numpy()[0][0], "Q max:", np.max(output.cpu().detach().numpy()))