def __init__(self, pool, observation_shape, n_actions, n_parallel_games=1, replay_seq_len=20, replay_batch_size=20, pool_size=None, n_steps=3, gamma=0.99): """ :type n_parallel_games: int n_actions: int """ # Parameters for training self.n_parallel_games = n_parallel_games self.replay_seq_len = replay_seq_len self.replay_batch_size = replay_batch_size self.pool_size = pool_size self.n_steps = n_steps self.gamma = gamma self.loss = None # image observation self.observation_layer = InputLayer(observation_shape) self.n_actions = n_actions self.resolver, self.agent = self.build_model() weights = lasagne.layers.get_all_params(self.resolver, trainable=True) self.applier_fun = self.agent.get_react_function() # Prepare replay pool env = SessionPoolEnvironment(observations=self.observation_layer, actions=self.resolver, agent_memories=self.agent.state_variables) preceding_memory_states = list(pool.prev_memory_states) # get interaction sessions observation_tensor, action_tensor, reward_tensor, _, is_alive_tensor, _ = \ pool.interact(self.step, n_steps=self.replay_seq_len) env.load_sessions(observation_tensor, action_tensor, reward_tensor, is_alive_tensor, preceding_memory_states) if pool_size is None: batch_env = env else: batch_env = env.sample_session_batch(self.replay_batch_size) self.loss = self.build_loss(batch_env) self.eval_fun = self.build_eval_fun(batch_env) updates = lasagne.updates.adadelta(self.loss, weights, learning_rate=0.01) train_fun = theano.function([], [self.loss], updates=updates) super(BasicRLAgent, self).__init__(env, pool, train_fun, pool_size, replay_seq_len)
def __init__(self, pool, observation_shape, n_actions, n_parallel_games=1, replay_seq_len=20, replay_batch_size=20, pool_size=None, n_steps=3, gamma=0.99): """ :type n_parallel_games: int n_actions: int """ # Parameters for training self.n_parallel_games = n_parallel_games self.replay_seq_len = replay_seq_len self.replay_batch_size = replay_batch_size self.pool_size = pool_size self.n_steps = n_steps self.gamma = gamma self.loss = None # image observation self.observation_layer = InputLayer(observation_shape) self.n_actions = n_actions self.resolver, self.agent = self.build_model() weights = lasagne.layers.get_all_params(self.resolver, trainable=True) self.applier_fun = self.agent.get_react_function() # Prepare replay pool env = SessionPoolEnvironment(observations=self.observation_layer, actions=self.resolver, agent_memories=self.agent.state_variables) preceding_memory_states = list(pool.prev_memory_states) # get interaction sessions observation_tensor, action_tensor, reward_tensor, _, is_alive_tensor, _ = \ pool.interact(self.step, n_steps=self.replay_seq_len) env.load_sessions(observation_tensor, action_tensor, reward_tensor, is_alive_tensor, preceding_memory_states) if pool_size is None: batch_env = env else: batch_env = env.sample_session_batch(self.replay_batch_size) self.loss = self.build_loss(batch_env) self.eval_fun = self.build_eval_fun(batch_env) updates = lasagne.updates.adadelta(self.loss, weights, learning_rate=0.01) train_fun = theano.function([], [self.loss], updates=updates) super(BasicRLAgent, self).__init__(env, pool, train_fun, pool_size, replay_seq_len)
def __init__(self, pool, observation_shape, n_actions, n_parallel_games=1, replay_seq_len=20, replay_batch_size=20, pool_size=None, n_steps=3, gamma=0.99, split_into=1,): #gru0_size=128): self.n_parallel_games = n_parallel_games self.replay_seq_len = replay_seq_len self.replay_batch_size = replay_batch_size self.pool_size = pool_size self.n_steps = n_steps self.n_actions = n_actions self.gamma = gamma self.split_into = split_into self.controller = Controller(observation_shape, n_actions) self.metacontroller = MetaController(self.controller)#, gru0_size) # Prepare replay pool self.controller_env = SessionPoolEnvironment(observations=self.controller.agent.observation_layers, actions=self.controller.resolver, agent_memories=self.controller.agent.agent_states) self.metacontroller_env = SessionPoolEnvironment(observations=self.metacontroller.agent.observation_layers, actions=self.metacontroller.resolver, agent_memories=self.metacontroller.agent.agent_states) # get interaction sessions observation_log, action_tensor, extrinsic_reward_log, memory_log, is_alive_tensor, _ = \ pool.interact(self.step, n_steps=self.replay_seq_len) preceding_memory_states = list(pool.prev_memory_states) self.reload_pool(observation_log, action_tensor, extrinsic_reward_log, is_alive_tensor, memory_log, preceding_memory_states) if pool_size is None: controller_batch_env = self.controller_env metacontroller_batch_env = self.metacontroller_env else: controller_batch_env = self.controller_env.sample_session_batch(self.replay_batch_size) metacontroller_batch_env = self.metacontroller_env.sample_session_batch(self.replay_batch_size) self.loss = self.build_loss(controller_batch_env, self.controller.agent, 50) + \ self.build_loss(metacontroller_batch_env, self.metacontroller.agent, 10) self.eval_fun = self.build_eval_fun(metacontroller_batch_env) weights = self.controller.weights + self.metacontroller.weights updates = lasagne.updates.adadelta(self.loss, weights, learning_rate=0.01) mean_session_reward = metacontroller_batch_env.rewards.sum(axis=1).mean() train_fun = theano.function([], [self.loss, mean_session_reward], updates=updates) super(HierarchicalAgent, self).__init__([self.controller_env, self.metacontroller_env], pool, train_fun, pool_size, replay_seq_len)
def __init__(self, agent, game_title, n_games, max_size=None, **kwargs): """ A pool that stores several - game states (gym environment) - prev_observations - last agent observations - prev memory states - last agent hidden states :param game_title: name of the game. See here http://yavar.naddaf.name/ale/list_of_current_games.html :param n_games: number of parallel games :param kwargs: options passed to Atari when creating a game. See Atari __init__ """ #create atari games self.game_kwargs = kwargs self.game_title = game_title self.games = [ Atari(self.game_title, **self.game_kwargs) for _ in range(n_games) ] #initial observations self.prev_observations = [atari.reset() for atari in self.games] #agent memory variables (if you use recurrent networks self.prev_memory_states = [ np.zeros((n_games, ) + tuple(mem.output_shape[1:]), dtype=get_layer_dtype(mem)) for mem in agent.agent_states ] #save agent self.agent = agent self.agent_step = agent.get_react_function() # Create experience replay environment self.experience_replay = SessionPoolEnvironment( observations=agent.observation_layers, actions=agent.action_layers, agent_memories=agent.agent_states) self.max_size = max_size
def test_space_invaders( game_title='SpaceInvaders-v0', n_parallel_games=3, replay_seq_len=2, ): """ :param game_title: name of atari game in Gym :param n_parallel_games: how many games we run in parallel :param replay_seq_len: how long is one replay session from a batch """ atari = gym.make(game_title) atari.reset() # Game Parameters n_actions = atari.action_space.n observation_shape = (None, ) + atari.observation_space.shape action_names = atari.get_action_meanings() del atari # ##### Agent observations # image observation at current tick goes here observation_layer = InputLayer(observation_shape, name="images input") # reshape to [batch, color, x, y] to allow for convolutional layers to work correctly observation_reshape = DimshuffleLayer(observation_layer, (0, 3, 1, 2)) # Agent memory states window_size = 3 # prev state input prev_window = InputLayer( (None, window_size) + tuple(observation_reshape.output_shape[1:]), name="previous window state") # our window window = WindowAugmentation(observation_reshape, prev_window, name="new window state") memory_dict = {window: prev_window} # ##### Neural network body # you may use any other lasagne layers, including convolutions, batch_norms, maxout, etc # pixel-wise maximum over the temporal window (to avoid flickering) window_max = ExpressionLayer(window, lambda a: a.max(axis=1), output_shape=(None, ) + window.output_shape[2:]) # a simple lasagne network (try replacing with any other lasagne network and see what works best) nn = DenseLayer(window_max, num_units=50, name='dense0') # Agent policy and action picking q_eval = DenseLayer(nn, num_units=n_actions, nonlinearity=lasagne.nonlinearities.linear, name="QEvaluator") #fakes for a2c policy_eval = DenseLayer(nn, num_units=n_actions, nonlinearity=lasagne.nonlinearities.softmax, name="a2c action probas") state_value_eval = DenseLayer(nn, num_units=1, nonlinearity=None, name="a2c state values") # resolver resolver = ProbabilisticResolver(policy_eval, name="resolver") # agent agent = Agent(observation_layer, memory_dict, (q_eval, policy_eval, state_value_eval), resolver) # Since it's a single lasagne network, one can get it's weights, output, etc weights = lasagne.layers.get_all_params(resolver, trainable=True) # Agent step function print('compiling react') applier_fun = agent.get_react_function() # a nice pythonic interface def step(observation, prev_memories='zeros', batch_size=n_parallel_games): """ returns actions and new states given observation and prev state Prev state in default setup should be [prev window,]""" # default to zeros if prev_memories == 'zeros': prev_memories = [ np.zeros((batch_size, ) + tuple(mem.output_shape[1:]), dtype='float32') for mem in agent.agent_states ] res = applier_fun(np.array(observation), *prev_memories) action = res[0] memories = res[1:] return action, memories # # Create and manage a pool of atari sessions to play with pool = GamePool(game_title, n_parallel_games) observation_log, action_log, reward_log, _, _, _ = pool.interact(step, 50) print(np.array(action_names)[np.array(action_log)[:3, :5]]) # # experience replay pool # Create an environment with all default parameters env = SessionPoolEnvironment(observations=observation_layer, actions=resolver, agent_memories=agent.agent_states) def update_pool(env, pool, n_steps=100): """ a function that creates new sessions and ads them into the pool throwing the old ones away entirely for simplicity""" preceding_memory_states = list(pool.prev_memory_states) # get interaction sessions observation_tensor, action_tensor, reward_tensor, _, is_alive_tensor, _ = pool.interact( step, n_steps=n_steps) # load them into experience replay environment env.load_sessions(observation_tensor, action_tensor, reward_tensor, is_alive_tensor, preceding_memory_states) # load first sessions update_pool(env, pool, replay_seq_len) # A more sophisticated way of training is to store a large pool of sessions and train on random batches of them. # ### Training via experience replay # get agent's Q-values, policy, etc obtained via experience replay _env_states, _observations, _memories, _imagined_actions, estimators = agent.get_sessions( env, session_length=replay_seq_len, batch_size=env.batch_size, optimize_experience_replay=True, ) (q_values_sequence, policy_sequence, value_sequence) = estimators # Evaluating loss function scaled_reward_seq = env.rewards # For SpaceInvaders, however, not scaling rewards is at least working elwise_mse_loss = 0. #1-step algos for algo in qlearning, sarsa: elwise_mse_loss += algo.get_elementwise_objective( q_values_sequence, env.actions[0], scaled_reward_seq, env.is_alive, gamma_or_gammas=0.99, ) #qlearning_n_step for n in (1, 3, replay_seq_len - 1, replay_seq_len, replay_seq_len + 1, None): elwise_mse_loss += qlearning_n_step.get_elementwise_objective( q_values_sequence, env.actions[0], scaled_reward_seq, env.is_alive, gamma_or_gammas=0.99, n_steps=n) #a2c n_step elwise_mse_loss += a2c_n_step.get_elementwise_objective( policy_sequence, value_sequence[:, :, 0], env.actions[0], scaled_reward_seq, env.is_alive, gamma_or_gammas=0.99, n_steps=3) # compute mean over "alive" fragments mse_loss = elwise_mse_loss.sum() / env.is_alive.sum() # regularize network weights reg_l2 = regularize_network_params(resolver, l2) * 10**-4 loss = mse_loss + reg_l2 # Compute weight updates updates = lasagne.updates.adadelta(loss, weights, learning_rate=0.01) # mean session reward mean_session_reward = env.rewards.sum(axis=1).mean() # # Compile train and evaluation functions print('compiling') train_fun = theano.function([], [loss, mean_session_reward], updates=updates) evaluation_fun = theano.function( [], [loss, mse_loss, reg_l2, mean_session_reward]) print("I've compiled!") # # Training loop for epoch_counter in range(10): update_pool(env, pool, replay_seq_len) loss, avg_reward = train_fun() full_loss, q_loss, l2_penalty, avg_reward_current = evaluation_fun() print("epoch %i,loss %.5f, rewards: %.5f " % (epoch_counter, full_loss, avg_reward_current)) print("rec %.3f reg %.3f" % (q_loss, l2_penalty))
def __init__( self, pool, observation_shape, n_actions, n_parallel_games=1, replay_seq_len=20, replay_batch_size=20, pool_size=None, n_steps=3, gamma=0.99, split_into=1, ): #gru0_size=128): self.n_parallel_games = n_parallel_games self.replay_seq_len = replay_seq_len self.replay_batch_size = replay_batch_size self.pool_size = pool_size self.n_steps = n_steps self.n_actions = n_actions self.gamma = gamma self.split_into = split_into self.controller = Controller(observation_shape, n_actions) self.metacontroller = MetaController(self.controller) #, gru0_size) # Prepare replay pool self.controller_env = SessionPoolEnvironment( observations=self.controller.agent.observation_layers, actions=self.controller.resolver, agent_memories=self.controller.agent.agent_states) self.metacontroller_env = SessionPoolEnvironment( observations=self.metacontroller.agent.observation_layers, actions=self.metacontroller.resolver, agent_memories=self.metacontroller.agent.agent_states) # get interaction sessions observation_log, action_tensor, extrinsic_reward_log, memory_log, is_alive_tensor, _ = \ pool.interact(self.step, n_steps=self.replay_seq_len) preceding_memory_states = list(pool.prev_memory_states) self.reload_pool(observation_log, action_tensor, extrinsic_reward_log, is_alive_tensor, memory_log, preceding_memory_states) if pool_size is None: controller_batch_env = self.controller_env metacontroller_batch_env = self.metacontroller_env else: controller_batch_env = self.controller_env.sample_session_batch( self.replay_batch_size) metacontroller_batch_env = self.metacontroller_env.sample_session_batch( self.replay_batch_size) self.loss = self.build_loss(controller_batch_env, self.controller.agent, 50) + \ self.build_loss(metacontroller_batch_env, self.metacontroller.agent, 10) self.eval_fun = self.build_eval_fun(metacontroller_batch_env) weights = self.controller.weights + self.metacontroller.weights updates = lasagne.updates.adadelta(self.loss, weights, learning_rate=0.01) mean_session_reward = metacontroller_batch_env.rewards.sum( axis=1).mean() train_fun = theano.function([], [self.loss, mean_session_reward], updates=updates) super(HierarchicalAgent, self).__init__([self.controller_env, self.metacontroller_env], pool, train_fun, pool_size, replay_seq_len)
class HierarchicalAgent(MdpAgent): def __init__( self, pool, observation_shape, n_actions, n_parallel_games=1, replay_seq_len=20, replay_batch_size=20, pool_size=None, n_steps=3, gamma=0.99, split_into=1, ): #gru0_size=128): self.n_parallel_games = n_parallel_games self.replay_seq_len = replay_seq_len self.replay_batch_size = replay_batch_size self.pool_size = pool_size self.n_steps = n_steps self.n_actions = n_actions self.gamma = gamma self.split_into = split_into self.controller = Controller(observation_shape, n_actions) self.metacontroller = MetaController(self.controller) #, gru0_size) # Prepare replay pool self.controller_env = SessionPoolEnvironment( observations=self.controller.agent.observation_layers, actions=self.controller.resolver, agent_memories=self.controller.agent.agent_states) self.metacontroller_env = SessionPoolEnvironment( observations=self.metacontroller.agent.observation_layers, actions=self.metacontroller.resolver, agent_memories=self.metacontroller.agent.agent_states) # get interaction sessions observation_log, action_tensor, extrinsic_reward_log, memory_log, is_alive_tensor, _ = \ pool.interact(self.step, n_steps=self.replay_seq_len) preceding_memory_states = list(pool.prev_memory_states) self.reload_pool(observation_log, action_tensor, extrinsic_reward_log, is_alive_tensor, memory_log, preceding_memory_states) if pool_size is None: controller_batch_env = self.controller_env metacontroller_batch_env = self.metacontroller_env else: controller_batch_env = self.controller_env.sample_session_batch( self.replay_batch_size) metacontroller_batch_env = self.metacontroller_env.sample_session_batch( self.replay_batch_size) self.loss = self.build_loss(controller_batch_env, self.controller.agent, 50) + \ self.build_loss(metacontroller_batch_env, self.metacontroller.agent, 10) self.eval_fun = self.build_eval_fun(metacontroller_batch_env) weights = self.controller.weights + self.metacontroller.weights updates = lasagne.updates.adadelta(self.loss, weights, learning_rate=0.01) mean_session_reward = metacontroller_batch_env.rewards.sum( axis=1).mean() train_fun = theano.function([], [self.loss, mean_session_reward], updates=updates) super(HierarchicalAgent, self).__init__([self.controller_env, self.metacontroller_env], pool, train_fun, pool_size, replay_seq_len) # raise NotImplementedError def reload_pool(self, observation_tensor, action_tensor, extrinsic_reward_tensor, is_alive_tensor, memory_tensor, preceding_memory_states): batch_size = observation_tensor.shape[0] # print observation_tensor.shape meta_obs_log, goal_log, meta_V, itrs = memory_tensor[-4:] itr = itrs[0] pivot = len(self.controller.agent.state_variables) controller_preceding_states = preceding_memory_states[:pivot] metacontroller_preceding_states = preceding_memory_states[pivot:-4] ###CONTROLLER### # load them into experience replay environment for controller # controller_preceding_states =!!!!!!!!!!!!!!!!!!!!!!!!!!! ctrl_shape = (batch_size * self.split_into, self.replay_seq_len / self.split_into) intrinsic_rewards = np.concatenate( [np.zeros([meta_V.shape[0], 1]), np.diff(meta_V, axis=1)], axis=1) # print [observation_tensor.reshape(ctrl_shape + self.controller.observation_shape[1:]), # goal_log.reshape(ctrl_shape)][0].shape self.controller_env.load_sessions( [ observation_tensor.reshape( ctrl_shape + self.controller.observation_shape[1:]), goal_log.reshape(ctrl_shape) ], action_tensor.reshape(ctrl_shape), intrinsic_rewards.reshape(ctrl_shape), is_alive_tensor.reshape(ctrl_shape), # controller_preceding_states ) ###METACONTROLLER### # separate case for metacontroller extrinsic_reward_sums = np.diff( np.concatenate([ np.zeros_like(extrinsic_reward_tensor[:, 0, None]), extrinsic_reward_tensor.cumsum(axis=-1)[:, itr == 0] ], axis=1)) self.metacontroller_env.load_sessions( meta_obs_log[:, itr == 0][:, :10], goal_log[:, itr == 0][:, :10], extrinsic_reward_sums[:, :10], is_alive_tensor[:, itr == 0][:, :10], metacontroller_preceding_states) def update_pool(self, observation_tensor, action_tensor, extrinsic_reward_tensor, is_alive_tensor, memory_tensor, preceding_memory_states): batch_size = observation_tensor.shape[0] meta_obs_log, goal_log, meta_V, itrs = memory_tensor[-4:] itr = itrs[0] pivot = len(self.controller.agent.state_variables) controller_preceding_states = preceding_memory_states[:pivot] metacontroller_preceding_states = preceding_memory_states[pivot:-4] ###CONTROLLER### # load them into experience replay environment for controller # controller_preceding_states =!!!!!!!!!!!!!!!!!!!!!!!!!!! ctrl_shape = (batch_size * self.split_into, self.replay_seq_len / self.split_into) intrinsic_rewards = np.concatenate( [np.zeros([meta_V.shape[0], 1]), np.diff(meta_V, axis=1)], axis=1) self.controller_env.append_sessions( [ observation_tensor.reshape( ctrl_shape + self.controller.observation_shape[1:]), goal_log.reshape(ctrl_shape) ], action_tensor.reshape(ctrl_shape), intrinsic_rewards.reshape(ctrl_shape), is_alive_tensor.reshape(ctrl_shape), controller_preceding_states, max_pool_size=self.pool_size, ) ###METACONTROLLER### # separate case for metacontroller extrinsic_reward_sums = np.diff( np.concatenate([ np.zeros_like(extrinsic_reward_tensor[:, 0, None]), extrinsic_reward_tensor.cumsum(axis=-1)[:, itr == 0] ], axis=1)) self.metacontroller_env.append_sessions( meta_obs_log[:, itr == 0][:, :10], goal_log[:, itr == 0][:, :10], extrinsic_reward_sums[:, :10], is_alive_tensor[:, itr == 0][:, :10], metacontroller_preceding_states, max_pool_size=self.pool_size) def step(self, env_observation, prev_memories='zeros'): """ returns actions and new states given observation and prev state Prev state in default setup should be [prev window,]""" batch_size = self.n_parallel_games if prev_memories == 'zeros': controller_mem = metacontroller_mem = 'zeros' meta_inp = np.zeros( (batch_size, ) + tuple(self.metacontroller.observation_shape[1:]), dtype='float32') itr = -1 # goal will be defined by "if itr ==0" clause else: pivot = len(self.controller.agent.state_variables) controller_mem, metacontroller_mem = prev_memories[: pivot], prev_memories[ pivot:-4] meta_inp, goal, meta_V, itrs = prev_memories[-4:] itr = itrs[0] itr = (itr + 1) % self.metacontroller.period if itr == 0: goal, metacontroller_mem, meta_V = self.metacontroller.step( meta_inp, metacontroller_mem, batch_size) #print env_observation.shape action, controller_mem, meta_inp = self.controller.step( env_observation, goal, controller_mem, batch_size) new_memories = controller_mem + metacontroller_mem + [ meta_inp, goal, meta_V, [itr] * self.n_parallel_games ] return action, new_memories def build_loss(self, env, agent, replay_seq_len): # get agent's Qvalues obtained via experience replay _, _, _, _, qvalues_seq = agent.get_sessions( env, # initial_hidden = env.preceding_agent_memories, session_length=replay_seq_len, batch_size=env.batch_size, optimize_experience_replay=True, ) scaled_reward_seq = env.rewards elwise_mse_loss = qlearning_n_step.get_elementwise_objective( qvalues_seq, env.actions[0], scaled_reward_seq, env.is_alive, gamma_or_gammas=self.gamma, n_steps=self.n_steps) # compute mean over "alive" fragments mse_loss = elwise_mse_loss.sum() / env.is_alive.sum() # regularize network weights reg_l2 = regularize_network_params(agent.state_variables.keys(), l2) * 10**-5 return mse_loss + reg_l2 def build_eval_fun(self, env): mean_session_reward = env.rewards.sum( axis=1).mean() / self.replay_seq_len eval_fun = theano.function([], [mean_session_reward]) return eval_fun
def test_memory( game_title='SpaceInvaders-v0', n_parallel_games=3, replay_seq_len=2, ): """ :param game_title: name of atari game in Gym :param n_parallel_games: how many games we run in parallel :param replay_seq_len: how long is one replay session from a batch """ atari = gym.make(game_title) atari.reset() # Game Parameters n_actions = atari.action_space.n observation_shape = (None, ) + atari.observation_space.shape action_names = atari.get_action_meanings() del atari # ##### Agent observations # image observation at current tick goes here observation_layer = InputLayer(observation_shape, name="images input") # reshape to [batch, color, x, y] to allow for convolutional layers to work correctly observation_reshape = DimshuffleLayer(observation_layer, (0, 3, 1, 2)) # Agent memory states memory_dict = OrderedDict([]) ###Window window_size = 3 # prev state input prev_window = InputLayer( (None, window_size) + tuple(observation_reshape.output_shape[1:]), name="previous window state") # our window window = WindowAugmentation(observation_reshape, prev_window, name="new window state") # pixel-wise maximum over the temporal window (to avoid flickering) window_max = ExpressionLayer(window, lambda a: a.max(axis=1), output_shape=(None, ) + window.output_shape[2:]) memory_dict[window] = prev_window ###Stack #prev stack stack_w, stack_h = 4, 5 stack_inputs = DenseLayer(observation_reshape, stack_w, name="prev_stack") stack_controls = DenseLayer(observation_reshape, 3, nonlinearity=lasagne.nonlinearities.softmax, name="prev_stack") prev_stack = InputLayer((None, stack_h, stack_w), name="previous stack state") stack = StackAugmentation(stack_inputs, prev_stack, stack_controls) memory_dict[stack] = prev_stack stack_top = lasagne.layers.SliceLayer(stack, 0, 1) ###RNN preset prev_rnn = InputLayer((None, 16), name="previous RNN state") new_rnn = RNNCell(prev_rnn, observation_reshape) memory_dict[new_rnn] = prev_rnn ###GRU preset prev_gru = InputLayer((None, 16), name="previous GRUcell state") new_gru = GRUCell(prev_gru, observation_reshape) memory_dict[new_gru] = prev_gru ###GRUmemorylayer prev_gru1 = InputLayer((None, 15), name="previous GRUcell state") new_gru1 = GRUMemoryLayer(15, observation_reshape, prev_gru1) memory_dict[new_gru1] = prev_gru1 #LSTM with peepholes prev_lstm0_cell = InputLayer( (None, 13), name="previous LSTMCell hidden state [with peepholes]") prev_lstm0_out = InputLayer( (None, 13), name="previous LSTMCell output state [with peepholes]") new_lstm0_cell, new_lstm0_out = LSTMCell( prev_lstm0_cell, prev_lstm0_out, input_or_inputs=observation_reshape, peepholes=True, name="newLSTM1 [with peepholes]") memory_dict[new_lstm0_cell] = prev_lstm0_cell memory_dict[new_lstm0_out] = prev_lstm0_out #LSTM without peepholes prev_lstm1_cell = InputLayer( (None, 14), name="previous LSTMCell hidden state [no peepholes]") prev_lstm1_out = InputLayer( (None, 14), name="previous LSTMCell output state [no peepholes]") new_lstm1_cell, new_lstm1_out = LSTMCell( prev_lstm1_cell, prev_lstm1_out, input_or_inputs=observation_reshape, peepholes=False, name="newLSTM1 [no peepholes]") memory_dict[new_lstm1_cell] = prev_lstm1_cell memory_dict[new_lstm1_out] = prev_lstm1_out ##concat everything for i in [flatten(window_max), stack_top, new_rnn, new_gru, new_gru1]: print(i.output_shape) all_memory = concat([ flatten(window_max), stack_top, new_rnn, new_gru, new_gru1, new_lstm0_out, new_lstm1_out, ]) # ##### Neural network body # you may use any other lasagne layers, including convolutions, batch_norms, maxout, etc # a simple lasagne network (try replacing with any other lasagne network and see what works best) nn = DenseLayer(all_memory, num_units=50, name='dense0') # Agent policy and action picking q_eval = DenseLayer(nn, num_units=n_actions, nonlinearity=lasagne.nonlinearities.linear, name="QEvaluator") # resolver resolver = EpsilonGreedyResolver(q_eval, epsilon=0.1, name="resolver") # agent agent = Agent(observation_layer, memory_dict, q_eval, resolver) # Since it's a single lasagne network, one can get it's weights, output, etc weights = lasagne.layers.get_all_params(resolver, trainable=True) # Agent step function print('compiling react') applier_fun = agent.get_react_function() # a nice pythonic interface def step(observation, prev_memories='zeros', batch_size=n_parallel_games): """ returns actions and new states given observation and prev state Prev state in default setup should be [prev window,]""" # default to zeros if prev_memories == 'zeros': prev_memories = [ np.zeros((batch_size, ) + tuple(mem.output_shape[1:]), dtype='float32') for mem in agent.agent_states ] res = applier_fun(np.array(observation), *prev_memories) action = res[0] memories = res[1:] return action, memories # # Create and manage a pool of atari sessions to play with pool = GamePool(game_title, n_parallel_games) observation_log, action_log, reward_log, _, _, _ = pool.interact(step, 50) print(np.array(action_names)[np.array(action_log)[:3, :5]]) # # experience replay pool # Create an environment with all default parameters env = SessionPoolEnvironment(observations=observation_layer, actions=resolver, agent_memories=agent.agent_states) def update_pool(env, pool, n_steps=100): """ a function that creates new sessions and ads them into the pool throwing the old ones away entirely for simplicity""" preceding_memory_states = list(pool.prev_memory_states) # get interaction sessions observation_tensor, action_tensor, reward_tensor, _, is_alive_tensor, _ = pool.interact( step, n_steps=n_steps) # load them into experience replay environment env.load_sessions(observation_tensor, action_tensor, reward_tensor, is_alive_tensor, preceding_memory_states) # load first sessions update_pool(env, pool, replay_seq_len) # A more sophisticated way of training is to store a large pool of sessions and train on random batches of them. # ### Training via experience replay # get agent's Q-values obtained via experience replay _env_states, _observations, _memories, _imagined_actions, q_values_sequence = agent.get_sessions( env, session_length=replay_seq_len, batch_size=env.batch_size, optimize_experience_replay=True, ) # Evaluating loss function scaled_reward_seq = env.rewards # For SpaceInvaders, however, not scaling rewards is at least working elwise_mse_loss = qlearning.get_elementwise_objective( q_values_sequence, env.actions[0], scaled_reward_seq, env.is_alive, gamma_or_gammas=0.99, ) # compute mean over "alive" fragments mse_loss = elwise_mse_loss.sum() / env.is_alive.sum() # regularize network weights reg_l2 = regularize_network_params(resolver, l2) * 10**-4 loss = mse_loss + reg_l2 # Compute weight updates updates = lasagne.updates.adadelta(loss, weights, learning_rate=0.01) # mean session reward mean_session_reward = env.rewards.sum(axis=1).mean() # # Compile train and evaluation functions print('compiling') train_fun = theano.function([], [loss, mean_session_reward], updates=updates) evaluation_fun = theano.function( [], [loss, mse_loss, reg_l2, mean_session_reward]) print("I've compiled!") # # Training loop for epoch_counter in range(10): update_pool(env, pool, replay_seq_len) loss, avg_reward = train_fun() full_loss, q_loss, l2_penalty, avg_reward_current = evaluation_fun() print("epoch %i,loss %.5f, rewards: %.5f " % (epoch_counter, full_loss, avg_reward_current)) print("rec %.3f reg %.3f" % (q_loss, l2_penalty))
class HierarchicalAgent(MdpAgent): def __init__(self, pool, observation_shape, n_actions, n_parallel_games=1, replay_seq_len=20, replay_batch_size=20, pool_size=None, n_steps=3, gamma=0.99, split_into=1,): #gru0_size=128): self.n_parallel_games = n_parallel_games self.replay_seq_len = replay_seq_len self.replay_batch_size = replay_batch_size self.pool_size = pool_size self.n_steps = n_steps self.n_actions = n_actions self.gamma = gamma self.split_into = split_into self.controller = Controller(observation_shape, n_actions) self.metacontroller = MetaController(self.controller)#, gru0_size) # Prepare replay pool self.controller_env = SessionPoolEnvironment(observations=self.controller.agent.observation_layers, actions=self.controller.resolver, agent_memories=self.controller.agent.agent_states) self.metacontroller_env = SessionPoolEnvironment(observations=self.metacontroller.agent.observation_layers, actions=self.metacontroller.resolver, agent_memories=self.metacontroller.agent.agent_states) # get interaction sessions observation_log, action_tensor, extrinsic_reward_log, memory_log, is_alive_tensor, _ = \ pool.interact(self.step, n_steps=self.replay_seq_len) preceding_memory_states = list(pool.prev_memory_states) self.reload_pool(observation_log, action_tensor, extrinsic_reward_log, is_alive_tensor, memory_log, preceding_memory_states) if pool_size is None: controller_batch_env = self.controller_env metacontroller_batch_env = self.metacontroller_env else: controller_batch_env = self.controller_env.sample_session_batch(self.replay_batch_size) metacontroller_batch_env = self.metacontroller_env.sample_session_batch(self.replay_batch_size) self.loss = self.build_loss(controller_batch_env, self.controller.agent, 50) + \ self.build_loss(metacontroller_batch_env, self.metacontroller.agent, 10) self.eval_fun = self.build_eval_fun(metacontroller_batch_env) weights = self.controller.weights + self.metacontroller.weights updates = lasagne.updates.adadelta(self.loss, weights, learning_rate=0.01) mean_session_reward = metacontroller_batch_env.rewards.sum(axis=1).mean() train_fun = theano.function([], [self.loss, mean_session_reward], updates=updates) super(HierarchicalAgent, self).__init__([self.controller_env, self.metacontroller_env], pool, train_fun, pool_size, replay_seq_len) # raise NotImplementedError def reload_pool(self, observation_tensor, action_tensor, extrinsic_reward_tensor, is_alive_tensor, memory_tensor, preceding_memory_states): batch_size = observation_tensor.shape[0] # print observation_tensor.shape meta_obs_log, goal_log, meta_V, itrs = memory_tensor[-4:] itr = itrs[0] pivot = len(self.controller.agent.state_variables) controller_preceding_states = preceding_memory_states[:pivot] metacontroller_preceding_states = preceding_memory_states[pivot:-4] ###CONTROLLER### # load them into experience replay environment for controller # controller_preceding_states =!!!!!!!!!!!!!!!!!!!!!!!!!!! ctrl_shape = (batch_size * self.split_into, self.replay_seq_len / self.split_into) intrinsic_rewards = np.concatenate([np.zeros([meta_V.shape[0], 1]), np.diff(meta_V, axis=1)], axis=1) # print [observation_tensor.reshape(ctrl_shape + self.controller.observation_shape[1:]), # goal_log.reshape(ctrl_shape)][0].shape self.controller_env.load_sessions([observation_tensor.reshape(ctrl_shape + self.controller.observation_shape[1:]), goal_log.reshape(ctrl_shape)], action_tensor.reshape(ctrl_shape), intrinsic_rewards.reshape(ctrl_shape), is_alive_tensor.reshape(ctrl_shape), # controller_preceding_states ) ###METACONTROLLER### # separate case for metacontroller extrinsic_reward_sums = np.diff( np.concatenate( [np.zeros_like(extrinsic_reward_tensor[:, 0, None]), extrinsic_reward_tensor.cumsum(axis=-1)[:, itr == 0]], axis=1 ) ) self.metacontroller_env.load_sessions(meta_obs_log[:, itr == 0][:, :10], goal_log[:, itr == 0][:, :10], extrinsic_reward_sums[:, :10], is_alive_tensor[:, itr == 0][:, :10], metacontroller_preceding_states) def update_pool(self, observation_tensor, action_tensor, extrinsic_reward_tensor, is_alive_tensor, memory_tensor, preceding_memory_states): batch_size = observation_tensor.shape[0] meta_obs_log, goal_log, meta_V, itrs = memory_tensor[-4:] itr = itrs[0] pivot = len(self.controller.agent.state_variables) controller_preceding_states = preceding_memory_states[:pivot] metacontroller_preceding_states = preceding_memory_states[pivot:-4] ###CONTROLLER### # load them into experience replay environment for controller # controller_preceding_states =!!!!!!!!!!!!!!!!!!!!!!!!!!! ctrl_shape = (batch_size * self.split_into, self.replay_seq_len / self.split_into) intrinsic_rewards = np.concatenate([np.zeros([meta_V.shape[0], 1]), np.diff(meta_V, axis=1)], axis=1) self.controller_env.append_sessions([observation_tensor.reshape(ctrl_shape + self.controller.observation_shape[1:]), goal_log.reshape(ctrl_shape)], action_tensor.reshape(ctrl_shape), intrinsic_rewards.reshape(ctrl_shape), is_alive_tensor.reshape(ctrl_shape), controller_preceding_states, max_pool_size=self.pool_size, ) ###METACONTROLLER### # separate case for metacontroller extrinsic_reward_sums = np.diff( np.concatenate( [np.zeros_like(extrinsic_reward_tensor[:, 0, None]), extrinsic_reward_tensor.cumsum(axis=-1)[:, itr == 0]], axis=1 ) ) self.metacontroller_env.append_sessions(meta_obs_log[:, itr == 0][:, :10], goal_log[:, itr == 0][:, :10], extrinsic_reward_sums[:, :10], is_alive_tensor[:, itr == 0][:, :10], metacontroller_preceding_states, max_pool_size=self.pool_size) def step(self, env_observation, prev_memories='zeros'): """ returns actions and new states given observation and prev state Prev state in default setup should be [prev window,]""" batch_size = self.n_parallel_games if prev_memories == 'zeros': controller_mem = metacontroller_mem = 'zeros' meta_inp = np.zeros((batch_size,) + tuple(self.metacontroller.observation_shape[1:]), dtype='float32') itr = -1 # goal will be defined by "if itr ==0" clause else: pivot = len(self.controller.agent.state_variables) controller_mem, metacontroller_mem = prev_memories[:pivot], prev_memories[pivot:-4] meta_inp, goal, meta_V, itrs = prev_memories[-4:] itr = itrs[0] itr = (itr + 1) % self.metacontroller.period if itr == 0: goal, metacontroller_mem, meta_V = self.metacontroller.step(meta_inp, metacontroller_mem, batch_size) #print env_observation.shape action, controller_mem, meta_inp = self.controller.step(env_observation, goal, controller_mem, batch_size) new_memories = controller_mem + metacontroller_mem + [meta_inp, goal, meta_V, [itr] * self.n_parallel_games] return action, new_memories def build_loss(self, env, agent, replay_seq_len): # get agent's Qvalues obtained via experience replay _, _, _, _, qvalues_seq = agent.get_sessions( env, # initial_hidden = env.preceding_agent_memories, session_length=replay_seq_len, batch_size=env.batch_size, optimize_experience_replay=True, ) scaled_reward_seq = env.rewards elwise_mse_loss = qlearning_n_step.get_elementwise_objective(qvalues_seq, env.actions[0], scaled_reward_seq, env.is_alive, gamma_or_gammas=self.gamma, n_steps=self.n_steps) # compute mean over "alive" fragments mse_loss = elwise_mse_loss.sum() / env.is_alive.sum() # regularize network weights reg_l2 = regularize_network_params(agent.state_variables.keys(), l2) * 10 ** -5 return mse_loss + reg_l2 def build_eval_fun(self, env): mean_session_reward = env.rewards.sum(axis=1).mean() / self.replay_seq_len eval_fun = theano.function([], [mean_session_reward]) return eval_fun
class AtariGamePool(object): def __init__(self, agent, game_title, n_games, max_size=None, **kwargs): """ A pool that stores several - game states (gym environment) - prev_observations - last agent observations - prev memory states - last agent hidden states :param game_title: name of the game. See here http://yavar.naddaf.name/ale/list_of_current_games.html :param n_games: number of parallel games :param kwargs: options passed to Atari when creating a game. See Atari __init__ """ #create atari games self.game_kwargs = kwargs self.game_title = game_title self.games = [ Atari(self.game_title, **self.game_kwargs) for _ in range(n_games) ] #initial observations self.prev_observations = [atari.reset() for atari in self.games] #agent memory variables (if you use recurrent networks self.prev_memory_states = [ np.zeros((n_games, ) + tuple(mem.output_shape[1:]), dtype=get_layer_dtype(mem)) for mem in agent.agent_states ] #save agent self.agent = agent self.agent_step = agent.get_react_function() # Create experience replay environment self.experience_replay = SessionPoolEnvironment( observations=agent.observation_layers, actions=agent.action_layers, agent_memories=agent.agent_states) self.max_size = max_size def interact(self, n_steps=100, verbose=False): """generate interaction sessions with ataries (openAI gym atari environments) Sessions will have length n_steps. Each time one of games is finished, it is immediately getting reset params: agent_step: a function(observations,memory_states) -> actions,new memory states for agent update n_steps: length of an interaction verbose: if True, prints small debug message whenever a game gets reloaded after end. returns: observation_log,action_log,reward_log,[memory_logs],is_alive_log,info_log a bunch of tensors [batch, tick, size...] the only exception is info_log, which is a list of infos for [time][batch] """ history_log = [] for i in range(n_steps): res = self.agent_step(self.prev_observations, *self.prev_memory_states) actions, new_memory_states = res[0], res[1:] new_observations, cur_rewards, is_done, infos = \ zip(*map( lambda atari, action: atari.step(action), self.games, actions) ) new_observations = np.array(new_observations) for i in range(len(self.games)): if is_done[i]: new_observations[i] = self.games[i].reset() for m_i in range(len(new_memory_states)): new_memory_states[m_i][i] = 0 if verbose: print("atari %i reloaded" % i) # append observation -> action -> reward tuple history_log.append((self.prev_observations, actions, cur_rewards, new_memory_states, is_done, infos)) self.prev_observations = new_observations self.prev_memory_states = new_memory_states # cast to numpy arrays observation_log, action_log, reward_log, memories_log, is_done_log, info_log = zip( *history_log) # tensor dimensions # [batch_i, time_i, observation_size...] observation_log = np.array(observation_log).swapaxes(0, 1) # [batch, time, units] for each memory tensor memories_log = map(lambda mem: np.array(mem).swapaxes(0, 1), zip(*memories_log)) # [batch_i,time_i] action_log = np.array(action_log).swapaxes(0, 1) # [batch_i, time_i] reward_log = np.array(reward_log).swapaxes(0, 1) # [batch_i, time_i] is_alive_log = 1 - np.array(is_done_log, dtype='int8').swapaxes(0, 1) return observation_log, action_log, reward_log, memories_log, is_alive_log, info_log def update(self, n_steps=100, append=False, max_size=None): """ a function that creates new sessions and ads them into the pool throwing the old ones away entirely for simplicity""" preceding_memory_states = list(self.prev_memory_states) # get interaction sessions observation_tensor, action_tensor, reward_tensor, _, is_alive_tensor, _ = self.interact( n_steps=n_steps) # load them into experience replay environment if not append: self.experience_replay.load_sessions(observation_tensor, action_tensor, reward_tensor, is_alive_tensor, preceding_memory_states) else: self.experience_replay.append_sessions(observation_tensor, action_tensor, reward_tensor, is_alive_tensor, preceding_memory_states, max_pool_size=max_size or self.max_size) def evaluate(self, n_games=1, save_path="./records", record_video=True, verbose=True, t_max=10000): """ Plays an entire game start to end, records the logs(and possibly mp4 video), returns reward :param save_path: where to save the report :param record_video: if True, records mp4 video :return: total reward (scalar) """ env = Atari(self.game_title, **self.game_kwargs) if record_video: env.monitor.start(save_path, force=True) else: env.monitor.start(save_path, lambda i: False, force=True) game_rewards = [] for _ in range(n_games): # initial observation observation = env.reset() # initial memory prev_memories = [ np.zeros((1, ) + tuple(mem.output_shape[1:]), dtype=get_layer_dtype(mem)) for mem in self.agent.agent_states ] t = 0 total_reward = 0 while True: res = self.agent_step(observation[None, ...], *prev_memories) action, new_memories = res[0], res[1:] observation, reward, done, info = env.step(action[0]) total_reward += reward prev_memories = new_memories if done or t >= t_max: if verbose: print( "Episode finished after {} timesteps with reward={}" .format(t + 1, total_reward)) break t += 1 game_rewards.append(total_reward) env.monitor.close() del env return np.mean(game_rewards)