def __init__(self, hyperparams, policy, sess, actions): self.hyperparams = hyperparams self.sess = sess with tf.device('/cpu:0'): self.policy = policy.create_policy(trainable=False) self.sess.run(tf.initialize_all_variables()) self.input_vars = get_input_vars(self.policy) self.y_hat = get_output(self.policy) self.experience_cache = [] self.episode_cache = [] self.prepare_epsilon() self.training = True self.actions = actions self.greedy_ind = None
def __init__(self, hyperparams, policy, sess, actions): self.hyperparams = hyperparams self.sess = sess with tf.device('/cpu:0'): self.policy = policy.create_policy(trainable=False) self.sess.run(tf.initialize_all_variables()) self.input_vars = get_input_vars(self.policy) self.y_hat = get_output(self.policy) self.experience_cache = [] self.episode_cache = [] self.prepare_epsilon() self.training = True self.actions = actions self.greedy_ind = None
def __init__(self, hyperparams, q_model, sess, actions): self.hyperparams = hyperparams self.sess = sess with tf.device('/cpu:0'): self.net = q_model.create_net(trainable=False) self.sess.run(tf.initialize_all_variables()) self.input_vars = get_input_vars(self.net) self.y_hat = get_output(self.net) self.experience_cache = [] self.recent_train_q = deque( maxlen=self.hyperparams['num_recent_steps']) self.recent_eval_q = deque(maxlen=self.hyperparams['num_recent_steps']) self.prepare_epsilon() self.training = True self.actions = actions self.greedy_ind = None
def __init__(self, hyperparams, q_model, sess, actions): self.hyperparams = hyperparams self.sess = sess with tf.device('/cpu:0'): self.net = q_model.create_net(trainable=False) self.sess.run(tf.initialize_all_variables()) self.input_vars = get_input_vars(self.net) self.y_hat = get_output(self.net) self.experience_cache = [] self.recent_train_q = deque( maxlen=self.hyperparams['num_recent_steps']) self.recent_eval_q = deque( maxlen=self.hyperparams['num_recent_steps']) self.prepare_epsilon() self.training = True self.actions = actions self.greedy_ind = None
def build(self): print('Building agent ...') self.action_model.setup_net() self.state_model.setup_net() self.state_y = tf.placeholder( dtype=tf.float32, shape=(None, np.prod(self.state_shape)), name='expected_state_y') self.state_loss.build(self.state_model.y_hat, self.state_y) state_params = flatten_params(get_all_params( self.state_model.get_net())) # print(state_params) # self.state_train_step = self.state_optim.get_train_step( # self.state_loss.loss, state_params) self.reward_model.setup_net() self.reward_y = tf.placeholder( dtype=tf.float32, shape=(None,), name='expected_reward_y') self.reward_loss.build(self.reward_model.y_hat, self.reward_y) reward_params = flatten_params(get_all_params( self.reward_model.get_net())) # self.reward_train_step = self.reward_optim.get_train_step( # self.reward_loss.loss, reward_params) self.value_model.setup_net() self.value_y = tf.placeholder( dtype=tf.float32, shape=(None,), name='expected_value_y') self.value_loss.build(self.value_model.y_hat, self.value_y) value_params = flatten_params(get_all_params( self.value_model.get_net())) # self.value_train_step = self.value_optim.get_train_step( # self.value_loss.loss, value_params) partial_params = state_params + reward_params + value_params partial_loss = (self.state_loss.loss + self.reward_loss.loss + self.value_loss.loss) self.partial_train_step = self.state_optim.get_train_step( partial_loss, partial_params) reward_discount = self.hyperparams['reward_discount'] batch_size = self.hyperparams['batch_size'] self.seed_train_state = tf.placeholder( tf.float32, shape=(batch_size,) + self.state_shape, name='seed_train_state') # scale = self.hyperparams['action_train_scale'] value_rollout_length = self.hyperparams['value_rollout_length'] next_state = self.seed_train_state next_conv_state = tf.concat(3, [next_state] * value_rollout_length) total_reward = tf.zeros((batch_size,)) for timestep in range(self.hyperparams['rollout_length']): state = next_state conv_state = next_conv_state action = get_output(self.action_model.get_net(), {'state': tf.expand_dims(state, 1)}, timestep=True) # evil softmax to closer-to-one-hot magic # action_max = tf.reduce_max(action, reduction_indices=1) # action_max = tf.expand_dims(action_max, 1) # action_min = tf.reduce_min(action, reduction_indices=1) # action_min = tf.expand_dims(action_min, 1) # action = tf.pow((1 - (action_max - action) - # (1 - (action_max - action_min))) / # (action_max - action_min), scale) # print('action shape') # print(action.get_shape()) next_state = get_output(self.state_model.get_net(), {'state': conv_state, 'action': action}) next_state = tf.reshape(next_state, (-1, *self.state_shape)) next_conv_state = tf.concat( 3, [next_conv_state[:, :, :, :value_rollout_length - 1], next_state]) reward = get_output(self.reward_model.net, {'state': next_conv_state, 'action': action}) total_reward += reward_discount * tf.squeeze(reward, [1]) value = get_output(self.value_model.get_net(), {'state': next_conv_state}) print('reward shape') print(reward.get_shape()) print('value shape') print(value.get_shape()) total_reward += reward_discount * tf.squeeze(value, [1]) print('Total reward shape') print(total_reward.get_shape()) self.exp_returns = tf.reduce_mean(total_reward) print('Flattening params ...') action_params = flatten_params(get_trainable_params( self.action_model.get_net())) print('Action params:') print(get_trainable_params(self.action_model.get_net())) self.action_train_step = self.action_optim.get_train_step( -self.exp_returns, action_params) self.action_preds = get_output(self.action_model.get_net(), None, timestep=True, input_hidden=True) # self.assign_hidden_ops = get_assign_hidden_ops( # self.action_model.get_net()) # self.zero_hidden_ops = get_assign_hidden_ops( # self.action_model.get_net(), # zero=True) # self.hidden_states = get_input_hidden_vars( # self.action_model.get_net(), # timestep=True) self.hidden_states = get_input_hidden_vars( self.action_model.get_net(), timestep=True) self.hidden_output_states = get_output_hidden_vars( self.action_model.get_net()) self.hidden_state_vals = {} self.init_hidden = get_init_hidden( self.action_model.get_net()) # for hidden_name, hidden_state in self.hidden_states.items(): # self.hidden_state_vals[hidden_state] = np.zeros( # hidden_state.eval(session=self.sess).shape) # self.hidden_state_vals[hidden_state] = None self.sess = tf.Session() self.sess.run(tf.initialize_all_variables()) self.update_value_target_weights_ops = set_all_params_ops( get_all_params(self.value_model.get_target_net()), get_all_params(self.value_model.get_net())) self.update_value_target_weights() self.prepare_epsilon() self.training = True self.part_experiences = [] self.experience_replay = RDRLMem(self.hyperparams) self.experience_replay.build() self.greedy_ind = None
def output_fn(self, state): return get_output(self.net, {get_input_name(self.net): state})
def setup_net(self): self.build_net() self.input_vars = get_input_vars(self.get_net()) self.y_hat = get_output(self.get_net())
def setup_net(self): self.build_net() self.y_hat = get_output(self.get_net()) self.target_y_hat = get_output(self.get_target_net())
def setup_net(self): self.build_net() self.policy_y_hat = get_output(self.policy) self.value_y_hat = get_output(self.value) self.policy_input_vars = get_input_vars(self.policy) self.value_input_vars = get_input_vars(self.value)