def __init__(self): self.game = Snake() self.p = PLE(self.game, fps=30, display_screen=True) # self.actions = self.p.getActionSet() # self._action_space = list(range(self.actions[0])) # self._action_space.append(self.actions[-1]) self.action_space = self.p.getActionSet()
def main(): game = Snake() game = PLE(game, display_screen=True) game.init() action_space = game.getActionSet() game.act(0) agent = Agent(game, MEMORY_MAX_SIZE, EXPLORATION_RATE, DISCOUNT, INITIALIZATION_SIZE) return game, agent
def __init__(self, height=32, width=32, fps=15, frame_history_size=4): # create the game environment and initialize the attribute values self.game = Snake(height=height,width=width,init_length=4) reward_dict = {"positive": 1.0, "negative": -1.0, "tick": 0.0, "loss": -1.0, "win": 1.0} self.environment = PLE(self.game, fps=fps, reward_values=reward_dict, num_steps=2) self.init_env() # initialize the game self.allowed_actions = self.environment.getActionSet() # the list of allowed actions to be taken by an agent self.num_actions = len(self.allowed_actions) - 1 # number of actions that are allowed in this env self.frame_hist = frame_history(height=height,width=width,frame_history_size=frame_history_size,num_channels=3); self.input_shape = self.frame_hist.get_history().shape # shape of the game input screen
def test(): game = Snake(600, 600) p = PLE(game, fps=60, state_preprocessor=process_state, force_fps=True, display_screen=True, frame_skip=2, reward_values={ "positive": 100.0, "negative": -50.0, "tick": -0.1, "loss": -70.0, "win": 5.0 }) agent = Agent(alpha=float(sys.argv[1]), gamma=float(sys.argv[2]), n_actions=3, epsilon=0.01, batch_size=100, input_shape=6, epsilon_dec=0.99999, epsilon_end=0.001, memory_size=500000, file_name=sys.argv[3], activations=[str(sys.argv[4]), str(sys.argv[5])]) p.init() agent.load_game() scores = [] for _ in range(200): if p.game_over(): p.reset_game() apples = 0 initial_direction = "Right" while not p.game_over(): old_state = np.array( vision(list(p.getGameState()[0]), initial_direction)) action = agent.choose_action(old_state) possible_directions = prepare_corect_directions(initial_direction) possible_directions_tuples = list( zip(possible_directions.keys(), possible_directions.values())) direction = possible_directions_tuples[action] initial_direction = direction[1] reward = p.act(direction[0]) if reward > 50.0: apples += reward scores.append(apples) return scores
def main(): # 创建环境 game = Snake(width=200, height=200, init_length=5) p = PLE(game, fps=30, display_screen=False, force_fps=True) # 根据parl框架构建agent print(p.getActionSet()) act_dim = len(p.getActionSet()) obs_dim = 200 * 200 rpm = ReplayMemory(MEMORY_SIZE) # DQN的经验回放池 model = Model(act_dim=act_dim) alg = DQN(model, act_dim=act_dim, gamma=GAMMA, lr=LEARNING_RATE) agent = Agent(alg, obs_dim=obs_dim, act_dim=act_dim, e_greed_decrement=1e-6, e_greed=0.2) # e_greed有一定概率随机选取动作,探索 # 加载模型 # if os.path.exists('./dqn_snake_400.ckpt'): # agent.restore('./dqn_snake_400.ckpt') # 先往经验池里存一些数据,避免最开始训练的时候样本丰富度不够 while len(rpm) < MEMORY_WARMUP_SIZE: run_episode(p, agent, rpm) max_episode = 2000000 # 开始训练 episode = 0 best_reward = -float('inf') while episode < max_episode: # 训练max_episode个回合,test部分不计算入episode数量 # train part for i in range(0, 100): total_reward = run_episode(p, agent, rpm) episode += 1 # test part eval_reward = evaluate(p, agent, render=True) # render=True 查看显示效果 if eval_reward > best_reward: best_reward = eval_reward agent.save('model_dir/dqn_snake_{}.ckpt'.format(episode)) logger.info('episode:{} e_greed:{} test_reward:{}'.format( episode, agent.e_greed, eval_reward))
def main(): # 创建环境 game = Snake(width=224, height=224, init_length=7) p = PLE(game, fps=30, display_screen=True, force_fps=True) # 根据parl框架构建agent print(p.getActionSet()) act_dim = len(p.getActionSet()) #rpm = ReplayMemory(MEMORY_SIZE) # DQN的经验回放池 model = Model(act_dim=act_dim) alg = DQN(model, act_dim=act_dim, gamma=GAMMA, lr=LEARNING_RATE) agent = Agent(alg, act_dim=act_dim, e_greed_decrement=1e-6, e_greed=0.2) # e_greed有一定概率随机选取动作,探索 # 加载模型 if os.path.exists('./dqn_snake_7.ckpt'): agent.restore('./dqn_snake_7.ckpt') evaluate(p, agent, False)
def get_envs(): envs = [ EnvWrapper('cartpole', gym.make('CartPole-v1'), 500, 550, 4000), EnvWrapper('catcher', PLE(Catcher(init_lives=1), display_screen=False, reward_values={ "positive": 1, "negative": -1, "loss": -1, }), 100, 110, 3000), EnvWrapper('snake', PLE(Snake(height=256, width=256), display_screen=False, reward_values={ "tick": -0.01, "positive": 5, "loss": -1, }), 100, 110, 3000), EnvWrapper('flappybird', PLE(FlappyBird(), display_screen=False, reward_values={ "positive": 1, "tick": 0.1, "loss": -1, }), 100, 110, 3000), ] return envs
def main(): # 创建环境 game = Snake(width=256, height=256, init_length=10) p = PLE(game, fps=30, display_screen=True, force_fps=True) # 根据parl框架构建agent p.reset_game() print(p.getActionSet()) act_dim = len(p.getActionSet()) obs_dim = 256 * 256 logger.info('obs_dim {}, act_dim {}'.format(obs_dim, act_dim)) # 根据parl框架构建agent model = Model(act_dim=act_dim) alg = PolicyGradient(model, lr=LEARNING_RATE) agent = Agent(alg, obs_dim=obs_dim, act_dim=act_dim) # # 加载模型 # if os.path.exists('model_dir/pg_pong_episode_19.ckpt'): # agent.restore('model_dir/pg_pong_episode_19.ckpt') best_total_reward = -float('inf') for i in range(50000): obs_list, action_list, reward_list = run_episode(p, agent) if i % 10 == 0: logger.info("Episode {}, Reward Sum {}.".format( i, sum(reward_list))) batch_obs = np.array(obs_list) batch_action = np.array(action_list) batch_reward = calc_reward_to_go(reward_list) agent.learn(batch_obs, batch_action, batch_reward) if (i + 1) % 50 == 0: total_reward = evaluate(p, agent, render=True) if total_reward > best_total_reward: best_total_reward = total_reward agent.save( 'model_dir/pg_pong_episode_{}_reward_{}.ckpt'.format( i, total_reward)) logger.info('Test reward: {}'.format(total_reward))
def __init__(self, game, display_screen=False): from ple import PLE assert game in [ 'catcher', 'monsterkong', 'flappybird', 'pixelcopter', 'pong', 'puckworld', 'raycastmaze', 'snake', 'waterworld' ] if game == 'catcher': from ple.games.catcher import Catcher env = Catcher() elif game == 'monsterkong': from ple.games.monsterkong import MonsterKong env = MonsterKong() elif game == 'flappybird': from ple.games.flappybird import FlappyBird env = FlappyBird() elif game == 'pixelcopter': from ple.games.pixelcopter import Pixelcopter env = Pixelcopter() elif game == 'pong': from ple.games.pong import Pong env = Pong() elif game == 'puckworld': from ple.games.puckworld import PuckWorld env = PuckWorld() elif game == 'raycastmaze': from ple.games.raycastmaze import RaycastMaze env = RaycastMaze() elif game == 'snake': from ple.games.snake import Snake env = Snake() elif game == 'waterworld': from ple.games.waterworld import WaterWorld env = WaterWorld() self.p = PLE(env, fps=30, display_screen=display_screen) self.action_set = self.p.getActionSet() self.action_size = len(self.action_set) self.screen_dims = self.p.getScreenDims() self.p.init()
target = reward + self.gamma * \ np.amax(self.model.predict(np.array([next_state]))[0]) target_f = self.model.predict(np.array([state])) target_f[0][self.actions.index(action)] = target self.model.fit(np.array([state]), np.array(target_f), epochs=1, verbose=0) if self.epsilon > self.epsilon_min: self.epsilon *= self.epsilon_decay game = Snake(width=320, height=320) p = PLE(game, fps=30, display_screen=True) agent = NaiveAgent(p.getActionSet()) p.init() reward = 0.0 learning_time = 100000 actions = [] rewards = [] snake_head_x = [] snake_head_y = [] food_x = [] food_y = []
# obs = [] # game_state = env.getGameState() # """ # {'snake_head_x': 32.0, 'snake_head_y': 32.0, 'food_x': 24, 'food_y': 30, # 'snake_body': [0.0, 3.0, 6.0], # 'snake_body_pos': [[32.0, 32.0], [29.0, 32.0], [26.0, 32.0]]} # # {'snake_head_x': 33.93333333333334, 'snake_head_y': 32.0, 'food_x': 36, 'food_y': 30, # 'snake_body': [0.0, 1.0933333333333337, 2.5333333333333314, 5.233333333333334], # 'snake_body_pos': [[33.93333333333334, 32.0], [32.84, 32.0], [31.400000000000006, 32.0], [28.700000000000003, 32.0]]} # """ # obs.append(game_state['snake_head_x']) # obs.append(game_state['snake_head_y']) # obs.append(game_state['food_x']) # obs.append(game_state['food_y']) # body_positions = [] # obs.append(np.mean(np.array(game_state['snake_body']))) # for body_pos in game_state['snake_body_pos']: # body_positions.extend(body_pos) # obs.append(np.mean(np.array(body_positions))) return obs if __name__ == '__main__': game = Snake(width=64, height=64, init_length=20) p = PLE(game, fps=30, display_screen=True, force_fps=True) # 根据parl框架构建agent print(p.getActionSet()) act_dim = len(p.getActionSet()) game_state = p.getGameState() print(game_state)
def a3c_main(save_path, shared_model,\ model,\ select_action,\ perform_action,\ save_model,\ optimizer=None,\ train=True,\ display=False,\ gamma =.99,\ tau=1.): tresh = False fps = 30 # fps we want to run at frame_skip = 2 num_steps = 1 force_fps = False # slower speed game = Snake(width=256, height=256) p = PLE(game, fps=fps, frame_skip=frame_skip, num_steps=num_steps, force_fps=force_fps, display_screen=display) p.init() def p_action(action): # reward, action return p.act(action) def main(lstm_shape, steps): reward_alive = 0 values = [] log_probs = [] rewards = [] entropies = [] x_t = extract_image(p.getScreenRGB(), (80, 80), tresh=tresh) stack_x = np.stack((x_t, x_t, x_t, x_t), axis=0) model.load_state_dict(shared_model.state_dict()) cx = Variable(torch.zeros(1, lstm_shape[-1])) hx = Variable(torch.zeros(1, lstm_shape[-1])) try: while p.game_over() == False and steps > 0: steps -= 1 x_t = extract_image(p.getScreenRGB(), (80, 80), tresh=tresh) x_t = np.reshape(x_t, (1, 80, 80)) st = np.append(stack_x[1:4, :, :], x_t, axis=0) if train: # print() reward, action, hx, cx, info_dict = train_and_play(p_action, st,\ select_action, perform_action,\ possible_actions, opt_nothing, \ model, {"isTrain":True, "hx":hx,"cx":cx}) reward_alive += 0.1 reward += reward_alive rewards.append(reward) # reward += r entropies.append(info_dict["entropies"]) values.append(info_dict["values"]) log_probs.append(info_dict["log_probs"]) else: _, _, hx, cx, _ = play(p_action, st, select_action,\ perform_action, possible_actions, model, {"hx":hx,"cx":cx, "isTrain":False}) stack_x = st if train: state = torch.from_numpy(stack_x) R = torch.zeros(1, 1) if steps > 0: value, _, _ = model( (Variable(state.unsqueeze(0).float()), (hx, cx))) values.append(Variable(R)) policy_loss = 0 value_loss = 0 R = Variable(R) gae = torch.zeros(1, 1) for i in reversed(range(len(rewards))): R = gamma * R + rewards[i] advantage = R - values[i] value_loss = value_loss + 0.5 * advantage.pow(2) # Generalized Advantage Estimataion delta_t = rewards[i] + gamma * \ values[i + 1].data - values[i].data gae = gae * gamma * tau + delta_t policy_loss = policy_loss - \ log_probs[i] * Variable(gae) - 0.01 * entropies[i] optimizer.zero_grad() (policy_loss + 0.5 * value_loss).backward() torch.nn.utils.clip_grad_norm(model.parameters(), 40) ensure_shared_grads(model, shared_model) optimizer.step() except KeyboardInterrupt as e: print("KeyboardInterrupt >>", e) print("Saving model") if train: save_model(shared_model, save_path) print("Model saved") sys.exit() score = p.score() p.reset_game() if train: save_model(shared_model, save_path) return score return main
def init_main(save_path, model, train=True, display=False): """The application's entry point. If someone executes this module (instead of importing it, for example), this function is called. """ push_to_memory, select_action, perform_action, optimize, save_model = model tresh = False fps = 30 # fps we want to run at frame_skip = 2 num_steps = 1 force_fps = False # slower speed game = Snake(width=256, height=256) p = PLE(game, fps=fps, frame_skip=frame_skip, num_steps=num_steps, force_fps=force_fps, display_screen=display) p.init() def p_action(action): # reward, action return p.act(action) def main(steps): reward_alive = 0 x_t = extract_image(p.getScreenRGB(), (80, 80), tresh=tresh) stack_x = np.stack((x_t, x_t, x_t, x_t), axis=0) try: while p.game_over() == False and steps > 0: steps -= 1 x_t = extract_image(p.getScreenRGB(), (80, 80), tresh=tresh) x_t = np.reshape(x_t, (1, 80, 80)) st = np.append(stack_x[1:4, :, :], x_t, axis=0) if train: r, action, _, _, _ = train_and_play( p_action, st, select_action, perform_action, possible_actions, optimize, None, {}) reward_alive += 0.1 r += reward_alive push_to_memory(stack_x, action, st, r) else: play(p_action, st, select_action, perform_action, possible_actions, None, {}) stack_x = st except KeyboardInterrupt as e: print("KeyboardInterrupt >>", e) print("Saving model") if train: save_model(save_path) print("Model saved") sys.exit() score = p.score() p.reset_game() if train: save_model(save_path) return score return main
fy = state['food_y'] turns = individual.activate([sx,sy,fx,fy]) best_turn = max(turns) if turns[0] == best_turn: fitness += snake_game.act(UP) elif turns[1] == best_turn: fitness += snake_game.act(LEFT) elif turns[2] == best_turn: fitness += snake_game.act(RIGHT) else: fitness += snake_game.act(DOWN) return fitness if __name__ == '__main__': # set up flappybird game game = Snake(width=400, height=400) # NOTE- if training: set force_fps = true, if testing: set force_fps to false snake_game = PLE(game, fps=30, display_screen=True, force_fps=False) snake_game.init() # uncomment this block to train a solution #model = neat.NEAT(config_file="snake.config") #best_genome = model.run(fitness_function=test_snake) #pickle.dump( best_genome, open( "snek", "wb" ) ) # uncomment this block to test solution LEFT = 119 DOWN = 97 UP = 100 RIGHT = 115 individual = pickle.load(open("snek", "rb"))
def test_snake(self): from ple.games.snake import Snake game = Snake() self.run_a_game(game)
def train(): game = Snake(600, 600) p = PLE(game, fps=60, state_preprocessor=process_state, force_fps=True, display_screen=False, frame_skip=2, reward_values={ "positive": 100.0, "negative": -50.0, "tick": -0.1, "loss": -110.0, "win": 5.0 }) agent = Agent(alpha=float(sys.argv[1]), gamma=float(sys.argv[2]), n_actions=3, epsilon=0.99, batch_size=100, input_shape=6, epsilon_dec=0.99999, epsilon_end=0.001, memory_size=500000, file_name=sys.argv[3], activations=[str(sys.argv[4]), str(sys.argv[5])]) p.init() # agent.load_game() scores = [] for _ in range(100000): if p.game_over(): p.reset_game() score = 0 initial_direction = "Right" while not p.game_over(): old_state = np.array( vision(list(p.getGameState()[0]), initial_direction)) action = agent.choose_action(old_state) possible_directions = prepare_corect_directions(initial_direction) possible_directions_tuples = list( zip(possible_directions.keys(), possible_directions.values())) direction = possible_directions_tuples[action] initial_direction = direction[1] reward = p.act(direction[0]) new_state = np.array( vision(list(p.getGameState()[0]), initial_direction)) agent.add_experience(old_state, action, reward, new_state) agent.learn() score = p.score() scores.append(score) print( f"Score for model iteration number _ {str(sys.argv[3])} with learning_rate {sys.argv[1]}, gama {sys.argv[2]}, activations: {sys.argv[4], sys.argv[5]} is score {score}. Epsilon is {agent.epsilon}" ) agent.save_game()
import numpy as np from ple import PLE from ple.games.snake import Snake import random def getReward(env, state, action): return(env.act(action)) agent = Snake(width=360, height=360) Q = {} gama = 0.9 alpha = 0.1 explore = 0.75 env = PLE(agent, fps=15, force_fps=False, display_screen=True) env.init() for i in range(100000): if explore != 0.25: if i % 30000 == 0: explore -= 0.25 if env.game_over(): env.reset_game() state = env.getGameState() del(state["snake_body_pos"])
RMS_EPSILON = 0.0001 MOMENTUM = 0 CLIP_DELTA = 1.0 EPSILON_START = 1.0 EPSILON_MIN = .1 EPSILON_DECAY = 100000 UPDATE_FREQUENCY = 1 REPLAY_MEMORY_SIZE = 1000000 BATCH_SIZE = 32 NETWORK_TYPE = "General_DQN_0" FREEZE_INTERVAL = 10000 DETERMINISTIC = True if __name__ == "__main__": game = Snake(width=64, height=64) logging.basicConfig(level=logging.INFO) # --- Parse parameters --- parameters = process_args(sys.argv[1:], Defaults) if parameters.deterministic: rng = np.random.RandomState(123456) else: rng = np.random.RandomState() # --- Instantiate environment --- env = PLE_env(rng, game=game, frame_skip=parameters.frame_skip, ple_options={ "display_screen": True,
return self.Q[fs[0], fs[1], fs[2], fs[3]] def pickAction(self, reward, obs): self.testenv(reward) #print(str(self.findState())) self.slightRandDic = {} for i in self.findQ(self.findState()): self.slightRandDic[i] = self.findQ(self.findState( ))[i] + self.pathr * np.random.randint(-10, 10) * self.steps #print(self.slightRandDic) self.act = max(self.slightRandDic, key=self.slightRandDic.get) self.rememberStep(self.act) return self.act #highest value action game = Snake(width=size, height=size, init_length=3) fps = 30 # fps we want to run at frame_skip = 2 num_steps = 1 force_fps = True # True == MegaSpeed display_screen = True reward = 0.0 max_noops = 20 nb_frames = 1500000 # make a PLE instance. p = PLE(game, fps=fps, frame_skip=frame_skip,
def get_grille(game, snake_location, food_location): s_x, s_y = snake_location f_x, f_x = food_location grille = [[0] * int(game.width / 10) for i in range(int(game.height / 10))] grille[s_x][s_y] += 1 grille[f_x][f_x] += 1 return grille # NE PAS CHANGER CETTE VARIABLE case_size = 20 size = 10 # Initialisation du jeu game = Snake(height=case_size * size, width=case_size * size) p = PLE(game, fps=30, display_screen=True) agent = Trainer(allowed_actions=p.getActionSet(), height=game.height, width=game.width) p.init() reward = 0.0 nb_frames = 10000000000000000 bestScore = 0 for i in range(nb_frames): if (p.score() > bestScore): bestScore = int(p.score())
target_f = self.model.predict(state) target_f[0][action] = target self.model.fit(state, target_f, nb_epoch=1, verbose=0) if self.epsilon > self.epsilon_min: self.epsilon *= self.epsilon_decay def load(self, name): self.model.load_weights(name) def save(self, name): self.model.save_weights(name) if __name__ == "__main__": game = Snake(width=256, height=256) env = PLE(game, display_screen=True, fps=10, state_preprocessor=process_state) agent = DQNAgent(env) agent.load('./save/snake.h5') env.init() for e in range(EPISODE): env.reset_game() score = 0 state = game.getGameState() state = sorted(state.items(), key=operator.itemgetter(0)) for i in range(len(state)):
def __init__(self, game_name, rewards, state_as_image = True, fps = 30, force_fps=True, frame_skip=2, hold_action=2, visualize=False, width=84, height=84, lives=1): """ Initialize Pygame Learning Environment https://github.com/ntasfi/PyGame-Learning-Environment Args: env_name: PLE environment fps: frames per second force_fps: False for slower speeds frame_skip: number of env frames to skip hold_action: number of env frames to hold each action for isRGB: get color or greyscale version of statespace #isRGB = False, game_height,game_width: height and width of environment visualize: If set True, the program will visualize the trainings, will slow down training lives: number of lives in game. Game resets on game over (ie lives = 0). only in Catcher and Pong (score) """ self.env_name = game_name self.rewards = rewards self.lives = lives self.state_as_image = state_as_image self.fps = fps #30 # frames per second self.force_fps = force_fps #True # False for slower speeds self.frame_skip = frame_skip # frames to skip self.ple_num_steps = hold_action # frames to continue action for #self.isRGB = isRGB #always returns color, lets tensorforce due the processing self.visualize = visualize self.width = width self.height = height #testing self.reached_terminal = 0 self.episode_time_steps = 0 self.episode_reward = 0 self.total_time_steps = 0 if self.env_name == 'catcher': self.game = Catcher(width=self.width, height=self.height,init_lives=self.lives) elif self.env_name == 'pixelcopter': self.game = Pixelcopter(width=self.width, height=self.height) elif self.env_name == 'pong': self.game = Pong(width=self.width, height=self.height,MAX_SCORE=self.lives) elif self.env_name == 'puckworld': self.game = PuckWorld(width=self.width, height=self.height) elif self.env_name == 'raycastmaze': self.game = RaycastMaze(width=self.width, height=self.height) elif self.env_name == 'snake': self.game = Snake(width=self.width, height=self.height) elif self.env_name == 'waterworld': self.game = WaterWorld(width=self.width, height=self.height) elif self.env_name == 'monsterkong': self.game = MonsterKong() elif self.env_name == 'flappybird': self.game = FlappyBird(width=144, height=256) # limitations on height and width for flappy bird else: raise TensorForceError('Unknown Game Environement.') if self.state_as_image: process_state = None else: #create a preprocessor to read the state dictionary as a numpy array def process_state(state): # ret_value = np.fromiter(state.values(),dtype=float,count=len(state)) ret_value = np.array(list(state.values()), dtype=np.float32) return ret_value # make a PLE instance self.env = PLE(self.game,reward_values=self.rewards,fps=self.fps, frame_skip=self.frame_skip, num_steps=self.ple_num_steps,force_fps=self.force_fps,display_screen=self.visualize, state_preprocessor = process_state) #self.env.init() #self.env.act(self.env.NOOP) #game starts on black screen #self.env.reset_game() #self.env.act(self.env.NOOP) #self.env.act(self.env.NOOP) #self.env.act(self.env.NOOP) #self.env.act(self.env.NOOP) #self.env.reset_game() # setup gamescreen object if state_as_image: w, h = self.env.getScreenDims() self.gamescreen = np.empty((h, w, 3), dtype=np.uint8) else: self.gamescreen = np.empty(self.env.getGameStateDims(), dtype=np.float32) # if isRGB: # self.gamescreen = np.empty((h, w, 3), dtype=np.uint8) # else: # self.gamescreen = np.empty((h, w), dtype=np.uint8) # setup action converter # PLE returns legal action indexes, convert these to just numbers self.action_list = self.env.getActionSet() self.action_list = sorted(self.action_list, key=lambda x: (x is None, x))
import numpy as np from ple import PLE from ple.games.snake import Snake agent = Snake(width=256, height=256) env = PLE(agent, fps=15, force_fps=False, display_screen=True) env.init() actions = env.getActionSet() for i in range(1000): if env.game_over(): env.reset_game() action = actions[np.random.randint(0, len(actions))] env.act(action)
def discounted_rewards(rewards, gamma=0.99): res = [] for r in reversed(rewards): cum_reward = res[0] if res else 0 res.insert(0, gamma * cum_reward + r) return res def train(env, agent): optimizer = torch.optim.Adam(agent.parameters()) while True: agent.zero_grad() p, r = play_episode(env, agent) r = torch.tensor(discounted_rewards(r), device=agent.device) loss = -r * p loss = loss.mean() loss.backward() optimizer.step() if __name__ == '__main__': env = PLE(Snake(), fps=30, display_screen=True) env.init() agent = Agent(env.getScreenDims(), 16, env.getActionSet()) train(env, agent)