def __init__(self, config, network, loss, optimizer): self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") self.memory = ReplayMemory(config['REPLAY']) self.policy_net = network.to(self.device) self.target_net = network.to(self.device) self.target_net.load_state_dict(self.policy_net.state_dict()) self.target_net.eval() self.loss = loss self.optimizer = optimizer(self.policy_net.parameters(), config['lr']) self.steps_done = 0 self.config = config
def __init__(self, sess, logger, config, env): super(Agent, self).__init__(config, logger) self.sess = sess self.logger = logger self.config = config params = DeepSenseParams(config) self.env = env self.history = History(logger, config) self.replay_memory = ReplayMemory(logger, config) with tf.variable_scope(STEPS): self.step_op = tf.Variable(0, trainable=False, name=STEP) self.step_input = tf.placeholder('int32', None, name=STEP_INPUT) self.step_assign_op = self.step_op.assign(self.step_input) self.build_dqn(params)
def __init__(self, sess, logger, config, env): super(Agent, self).__init__(config, logger) self.sess = sess self.logger = logger self.config = config params = DeepSenseParams(config) self.env = env self.history = History(logger, config) self.replay_memory = ReplayMemory(logger, config) with tf.variable_scope(STEPS): self.step_op = tf.Variable(0, trainable=False, name=STEP) self.step_input = tf.placeholder('int32', None, name=STEP_INPUT) self.step_assign_op = self.step_op.assign(self.step_input) self.build_dqn(params)
class Agent(BaseAgent): '''Deep Trading Agent based on Deep Q Learning''' '''TODO: 1. add `play` function to run tests in the simulated environment ''' def __init__(self, sess, logger, config, env): super(Agent, self).__init__(config, logger) self.sess = sess self.logger = logger self.config = config params = DeepSenseParams(config) self.env = env self.history = History(logger, config) self.replay_memory = ReplayMemory(logger, config) with tf.variable_scope(STEPS): self.step_op = tf.Variable(0, trainable=False, name=STEP) self.step_input = tf.placeholder('int32', None, name=STEP_INPUT) self.step_assign_op = self.step_op.assign(self.step_input) self.build_dqn(params) @property def summary_writer(self): return self._summary_writer def train(self): start_step = self.sess.run(self.step_op) num_episodes, self.update_count, ep_reward = 0, 0, 0. total_reward, self.total_loss, self.total_q = 0., 0., 0. max_avg_ep_reward = 0 ep_rewards, actions = [], [] trade_rem = self.env.new_random_episode(self.history, self.replay_memory) for self.step in tqdm(range(start_step, self.max_step), ncols=70, initial=start_step): if self.step == self.learn_start: num_episodes, self.update_count, ep_reward = 0, 0, 0. total_reward, self.total_loss, self.total_q = 0., 0., 0. ep_rewards, actions = [], [] # 1. predict action = self.predict((self.history.history, trade_rem)) # 2. act screen, reward, terminal, trade_rem = self.env.act(action) # 3. observe self.observe(screen, reward, action, terminal, trade_rem) if terminal: self.env.new_random_episode(self.history, self.replay_memory) num_episodes += 1 ep_rewards.append(ep_reward) ep_reward = 0. else: ep_reward += reward actions.append(action) total_reward += reward if self.step >= self.learn_start: if self.step % self.test_step == self.test_step - 1: avg_reward = total_reward / self.test_step avg_loss = self.total_loss / self.update_count avg_q = self.total_q / self.update_count try: max_ep_reward = np.max(ep_rewards) min_ep_reward = np.min(ep_rewards) avg_ep_reward = np.mean(ep_rewards) except: max_ep_reward, min_ep_reward, avg_ep_reward = 0, 0, 0 message = 'avg_r: %.4f, avg_l: %.6f, avg_q: %3.6f, avg_ep_r: %.4f, max_ep_r: %.4f, min_ep_r: %.4f, # game: %d' \ % (avg_reward, avg_loss, avg_q, avg_ep_reward, max_ep_reward, min_ep_reward, num_episodes) self.logger.info(message) if max_avg_ep_reward * 0.9 <= avg_ep_reward: self.sess.run( fetches=self.step_assign_op, feed_dict={self.step_input: self.step + 1} ) self.save_model(self.step + 1) max_avg_ep_reward = max(max_avg_ep_reward, avg_ep_reward) if self.step > 180: self.inject_summary({ 'average.reward': avg_reward, 'average.loss': avg_loss, 'average.q': avg_q, 'episode.max reward': max_ep_reward, 'episode.min reward': min_ep_reward, 'episode.avg reward': avg_ep_reward, 'episode.num of episodes': num_episodes, 'episode.rewards': ep_rewards, 'episode.actions': actions, 'training.learning_rate': self.sess.run( fetches=self.learning_rate_op, feed_dict={self.learning_rate_step: self.step} ) }, self.step) num_episodes = 0 total_reward = 0. self.total_loss = 0. self.total_q = 0. self.update_count = 0 ep_reward = 0. ep_rewards = [] actions = [] def predict(self, state, test_ep=None): s_t = state[0] trade_rem_t = state[1] ep = test_ep or (self.ep_end + max(0., (self.ep_start - self.ep_end) \ * (self.ep_end_t - max(0., self.step - self.learn_start)) / self.ep_end_t)) if random.random() < ep: action = random.randrange(self.config[NUM_ACTIONS]) else: action = self.sess.run( fetches=self.q.action, feed_dict={ self.q.phase: 0, self.s_t: [s_t], self.trade_rem_t: [trade_rem_t], self.q_conv_keep_prob: 1.0, self.q_dense_keep_prob: 1.0, self.q_gru_keep_prob: 1.0 } )[0] return action def observe(self, screen, reward, action, terminal, trade_rem): #clip reward in the range min to max reward = max(self.min_reward, min(self.max_reward, reward)) self.history.add(screen) self.replay_memory.add(screen, reward, action, terminal, trade_rem) if self.step > self.learn_start: if self.step % self.train_frequency == 0: self.q_learning_mini_batch() if self.step % self.target_q_update_step == self.target_q_update_step - 1: self.update_target_network() def q_learning_mini_batch(self): if self.replay_memory.count >= self.replay_memory.history_length: state_t, action, reward, state_t_plus_1, terminal = self.replay_memory.sample s_t, trade_rem_t = state_t[0], state_t[1] s_t_plus_1, trade_rem_t_plus_1 = state_t_plus_1[0], state_t_plus_1[1] q_t_plus_1 = self.sess.run( fetches=self.t_q.values, feed_dict={ self.t_q.phase: 0, self.t_s_t: s_t_plus_1, self.t_trade_rem_t: trade_rem_t_plus_1 } ) max_q_t_plus_1 = np.max(q_t_plus_1, axis=1) terminal = np.array(terminal) + 0. target_q = reward + (1 - terminal) * max_q_t_plus_1 _, q_t, loss, avg_q_summary = self.sess.run([self.optimizer, self.q.values, self.loss, self.q.avg_q_summary], { self.q.phase: 1, self.target_q: target_q, self.action: action, self.s_t: s_t, self.trade_rem_t: trade_rem_t, self.q_conv_keep_prob: self.config[CONV_KEEP_PROB], self.q_dense_keep_prob: self.config[DENSE_KEEP_PROB], self.q_gru_keep_prob: self.config[GRU_KEEP_PROB], self.learning_rate_step: self.step }) self.summary_writer.add_summary(avg_q_summary, self.step) self.total_loss += loss self.total_q += q_t.mean() self.update_count += 1 def build_dqn(self, params): with tf.variable_scope(PREDICTION): self.s_t = tf.placeholder( dtype=tf.float32, shape=[None, self.replay_memory.history_length, self.replay_memory.num_channels], name=HISTORICAL_PRICES ) self.trade_rem_t = tf.placeholder( dtype=tf.float32, shape=[None,], name=TRADE_REM ) with tf.variable_scope(DROPOUT_KEEP_PROBS): self.q_conv_keep_prob = tf.placeholder(tf.float32) self.q_dense_keep_prob = tf.placeholder(tf.float32) self.q_gru_keep_prob = tf.placeholder(tf.float32) params.dropoutkeepprobs = DropoutKeepProbs( self.q_conv_keep_prob, self.q_dense_keep_prob, self.q_gru_keep_prob ) self.q = DeepSense(params, self.logger, self.sess, self.config, name=Q_NETWORK) self.q.build_model((self.s_t, self.trade_rem_t)) with tf.variable_scope(TARGET): self.t_s_t = tf.placeholder( dtype=tf.float32, shape=[None, self.replay_memory.history_length, self.replay_memory.num_channels], name=HISTORICAL_PRICES ) self.t_trade_rem_t = tf.placeholder( dtype=tf.float32, shape=[None,], name=TRADE_REM ) params.dropoutkeepprobs = DropoutKeepProbs() self.t_q = DeepSense(params, self.logger, self.sess, self.config, name=T_Q_NETWORK) self.t_q.build_model((self.t_s_t, self.t_trade_rem_t)) with tf.variable_scope(UPDATE_TARGET_NETWORK): self.q_weights_placeholders = {} self.t_weights_assign_ops = {} for name in self.q.weights.keys(): self.q_weights_placeholders[name] = tf.placeholder( tf.float32, self.q.weights[name].get_shape().as_list() ) for name in self.q.weights.keys(): self.t_weights_assign_ops[name] = self.t_q.weights[name].assign( self.q_weights_placeholders[name] ) with tf.variable_scope(TRAINING): self.target_q = tf.placeholder(tf.float32, [None], name=TARGET_Q) self.action = tf.placeholder(tf.int64, [None], name=ACTION) action_one_hot = tf.one_hot(self.action, self.config[NUM_ACTIONS], 1.0, 0.0, name=ACTION_ONE_HOT) q_acted = tf.reduce_sum(self.q.values * action_one_hot, reduction_indices=1, name=Q_ACTED) with tf.variable_scope(LOSS): self.delta = self.target_q - q_acted self.global_step = tf.Variable(0, trainable=False) self.loss = tf.reduce_mean(clipped_error(self.delta), name=LOSS) with tf.variable_scope(OPTIMIZER): self.learning_rate_step = tf.placeholder(tf.int64, None, name=LEARNING_RATE_STEP) self.learning_rate_op = tf.maximum(self.learning_rate_minimum, tf.train.exponential_decay( self.learning_rate, self.learning_rate_step, self.learning_rate_decay_step, self.learning_rate_decay, staircase=True)) self.optimizer = tf.train.RMSPropOptimizer( self.learning_rate_op, momentum=0.95, epsilon=0.01).minimize(self.loss) with tf.variable_scope(SUMMARY): scalar_summary_tags = ['average.reward', 'average.loss', 'average.q', \ 'episode.max reward', 'episode.min reward', 'episode.avg reward', \ 'episode.num of episodes', 'training.learning_rate'] self.summary_placeholders = {} self.summary_ops = {} for tag in scalar_summary_tags: self.summary_placeholders[tag] = \ tf.placeholder('float32', None, name=tag.replace(' ', '_')) self.summary_ops[tag] = \ tf.summary.scalar( name="{}-{}".format(self.env_name, tag.replace(' ', '_')), tensor=self.summary_placeholders[tag] ) histogram_summary_tags = ['episode.rewards', 'episode.actions'] for tag in histogram_summary_tags: self.summary_placeholders[tag] = \ tf.placeholder('float32', None, name=tag) self.summary_ops[tag] = \ tf.summary.histogram( tag, self.summary_placeholders[tag] ) self.sess.run(tf.local_variables_initializer()) self.sess.run(tf.global_variables_initializer()) self._saver = tf.train.Saver(self.q.weights.values() + [self.step_op], max_to_keep=30) self.load_model() self.update_target_network() self._summary_writer = tf.summary.FileWriter(self.config[TENSORBOARD_LOG_DIR]) self._summary_writer.add_graph(self.sess.graph) def update_target_network(self): for name in self.q.weights.keys(): self.sess.run( fetches=self.t_weights_assign_ops[name], feed_dict= {self.q_weights_placeholders[name]: self.sess.run( fetches=self.q.weights[name] )} ) def inject_summary(self, tag_dict, step): summary_str_lists = self.sess.run([self.summary_ops[tag] for tag in tag_dict.keys()], { self.summary_placeholders[tag]: value for tag, value in tag_dict.items() }) for summary_str in summary_str_lists: self.summary_writer.add_summary(summary_str, self.step)
class DQN: def __init__(self, config, network, loss, optimizer): self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") self.memory = ReplayMemory(config['REPLAY']) self.policy_net = network.to(self.device) self.target_net = network.to(self.device) self.target_net.load_state_dict(self.policy_net.state_dict()) self.target_net.eval() self.loss = loss self.optimizer = optimizer(self.policy_net.parameters(), config['lr']) self.steps_done = 0 self.config = config def update(self): self.target_net.load_state_dict(self.policy_net.state_dict()) def select_action(self, state): EPS_START, EPS_END, EPS_DECAY, n_actions = self.config[ 'EPS_START'], self.config['EPS_END'], self.config[ 'EPS_DECAY'], self.config['ACTION_SPACE'] sample = random.random() eps_threshold = EPS_END + (EPS_START - EPS_END) * \ math.exp(-1. * self.steps_done / EPS_DECAY) self.steps_done += 1 if sample > eps_threshold: with torch.no_grad(): # t.max(1) will return largest column value of each row. # second column on max result is index of where max element was # found, so we pick action with the larger expected reward. return self.policy_net(state).max(1)[1].view(1, 1) else: return torch.tensor([[random.randrange(n_actions)]], device=self.device, dtype=torch.long) def optimize_model(self): BATCH_SIZE = self.config['BATCH_SIZE'] if len(self.memory) < BATCH_SIZE: return transitions = self.memory.sample(BATCH_SIZE) # Transpose the batch (see https://stackoverflow.com/a/19343/3343043 for # detailed explanation). This converts batch-array of Transitions # to Transition of batch-arrays. batch = Transition(*zip(*transitions)) # Compute a mask of non-final states and concatenate the batch elements # (a final state would've been the one after which simulation ended) non_final_mask = torch.tensor(tuple( map(lambda s: s is not None, batch.next_state)), device=self.device, dtype=torch.bool) non_final_next_states = torch.cat( [s for s in batch.next_state if s is not None]) state_batch = torch.cat(batch.state) action_batch = torch.cat(batch.action) reward_batch = torch.cat(batch.reward) # Compute Q(s_t, a) - the model computes Q(s_t), then we select the # columns of actions taken. These are the actions which would've been taken # for each batch state according to policy_net state_action_values = self.policy_net(state_batch).gather( 1, action_batch) # Compute V(s_{t+1}) for all next states. # Expected values of actions for non_final_next_states are computed based # on the "older" target_net; selecting their best reward with max(1)[0]. # This is merged based on the mask, such that we'll have either the expected # state value or 0 in case the state was final. next_state_values = torch.zeros(BATCH_SIZE, device=self.device) next_state_values[non_final_mask] = self.target_net( non_final_next_states).max(1)[0].detach() # Compute the expected Q values GAMMA = self.config['GAMMA'] expected_state_action_values = (next_state_values * GAMMA) + reward_batch # Compute Huber loss loss = self.loss(state_action_values, expected_state_action_values.unsqueeze(1)) # Optimize the model self.optimizer.zero_grad() loss.backward() for param in self.policy_net.parameters(): param.grad.data.clamp_(-1, 1) self.optimizer.step()
class Agent(BaseAgent): '''Deep Trading Agent based on Deep Q Learning''' '''TODO: 1. play ''' def __init__(self, sess, logger, config, env): super(Agent, self).__init__(config, logger) self.sess = sess self.logger = logger self.config = config params = DeepSenseParams(config) self.env = env self.history = History(logger, config) self.replay_memory = ReplayMemory(logger, config) with tf.variable_scope(STEPS): self.step_op = tf.Variable(0, trainable=False, name=STEP) self.step_input = tf.placeholder('int32', None, name=STEP_INPUT) self.step_assign_op = self.step_op.assign(self.step_input) self.build_dqn(params) @property def summary_writer(self): return self._summary_writer def train(self): start_step = self.step_op.eval() num_episodes, self.update_count, ep_reward = 0, 0, 0. total_reward, self.total_loss, self.total_q = 0., 0., 0. max_avg_ep_reward = 0 ep_rewards, actions = [], [] self.env.new_random_episode(self.history) for self.step in tqdm(range(start_step, self.max_step), ncols=70, initial=start_step): if self.step == self.learn_start: num_episodes, self.update_count, ep_reward = 0, 0, 0. total_reward, self.total_loss, self.total_q = 0., 0., 0. ep_rewards, actions = [], [] # 1. predict action = self.predict(self.history.get()) # 2. act screen, reward, terminal = self.env.act(action) # 3. observe self.observe(screen, reward, action, terminal) if terminal: self.env.new_random_episode(self.history) num_episodes += 1 ep_rewards.append(ep_reward) ep_reward = 0. else: ep_reward += reward actions.append(action) total_reward += reward if self.step >= self.learn_start: if self.step % self.test_step == self.test_step - 1: avg_reward = total_reward / self.test_step avg_loss = self.total_loss / self.update_count avg_q = self.total_q / self.update_count try: max_ep_reward = np.max(ep_rewards) min_ep_reward = np.min(ep_rewards) avg_ep_reward = np.mean(ep_rewards) except: max_ep_reward, min_ep_reward, avg_ep_reward = 0, 0, 0 message = 'avg_r: %.4f, avg_l: %.6f, avg_q: %3.6f, avg_ep_r: %.4f, max_ep_r: %.4f, min_ep_r: %.4f, # game: %d' \ % (avg_reward, avg_loss, avg_q, avg_ep_reward, max_ep_reward, min_ep_reward, num_game) print_and_log_message(message, self.logger) if max_avg_ep_reward * 0.9 <= avg_ep_reward: self.step_assign_op.eval( {self.step_input: self.step + 1}) self.save_model(self.step + 1) max_avg_ep_reward = max(max_avg_ep_reward, avg_ep_reward) if self.step > 180: self.inject_summary( { 'average.reward': avg_reward, 'average.loss': avg_loss, 'average.q': avg_q, 'episode.max reward': max_ep_reward, 'episode.min reward': min_ep_reward, 'episode.avg reward': avg_ep_reward, 'episode.num of game': num_game, 'episode.rewards': ep_rewards, 'episode.actions': actions, 'training.learning_rate': self.learning_rate_op.eval( {self.learning_rate_step: self.step}), }, self.step) num_game = 0 total_reward = 0. self.total_loss = 0. self.total_q = 0. self.update_count = 0 ep_reward = 0. ep_rewards = [] actions = [] def predict(self, s_t, test_ep=None): ep = test_ep or (self.ep_end + max(0., (self.ep_start - self.ep_end) \ * (self.ep_end_t - max(0., self.step - self.learn_start)) / self.ep_end_t)) if random.random() < ep: action = random.randrange(self.env.action_size) else: action = self.q.action.eval({self.s_t: [s_t]})[0] return action def observe(self, screen, reward, action, terminal): #clip reward in the range min to max reward = max(self.min_reward, min(self.max_reward, reward)) self.history.add(screen) self.replay_memory.add(screen, reward, action, terminal) if self.step > self.learn_start: if self.step % self.train_frequency == 0: self.q_learning_mini_batch() if self.step % self.target_q_update_step == self.target_q_update_step - 1: self.update_target_network() def q_learning_mini_batch(self): if self.replay_memory.count >= self.replay_memory.history_length: s_t, action, reward, s_t_plus_1, terminal = self.replay_memory.sample( ) max_q_t_plus_1 = self.t_q.action.eval({self.t_s_t: s_t_plus_1}) terminal = np.array(terminal) + 0. target_q = reward + (1 - terminal) * max_q_t_plus_1 _, q_t, loss, avg_q_summary = self.sess.run( [ self.optimizer, self.q.values, self.loss, self.q.avg_q_summary ], { self.target_q: target_q, self.action: action, self.s_t: s_t, self.learning_rate_step: self.step, }) self.summary_writer.add_summary(avg_q_summary, self.step) self.total_loss += loss self.total_q += q_t.mean() self.update_count += 1 def build_dqn(self, params): with tf.variable_scope(PREDICTION): self.s_t = tf.placeholder(dtype=tf.float32, shape=[ None, self.replay_memory.history_length, self.replay_memory.num_channels ]) self.q = DeepSense(params, self.logger, self.sess, self.config, name=Q_NETWORK) self.q.build_model(self.s_t) with tf.variable_scope(TARGET): self.t_s_t = tf.placeholder(dtype=tf.float32, shape=[ None, self.replay_memory.history_length, self.replay_memory.num_channels ]) self.t_q = DeepSense(params, self.logger, self.sess, self.config, name=T_Q_NETWORK) self.t_q.build_model(self.t_s_t, train=False) with tf.variable_scope(UPDATE_TARGET_NETWORK): self.q_weights_placeholders = {} self.t_weights_assign_ops = {} for name in self.q.weights.keys(): self.q_weights_placeholders[name] = tf.placeholder( tf.float32, self.q.weights[name].get_shape().as_list()) for name in self.q.weights.keys(): self.t_weights_assign_ops[name] = self.t_q.weights[ name].assign(self.q_weights_placeholders[name]) with tf.variable_scope(TRAINING): self.target_q = tf.placeholder(tf.float32, [None], name=TARGET_Q) self.action = tf.placeholder(tf.int64, [None], name=ACTION) action_one_hot = tf.one_hot(self.action, self.env.action_size, 1.0, 0.0, name=ACTION_ONE_HOT) q_acted = tf.reduce_sum(self.q.values * action_one_hot, reduction_indices=1, name=Q_ACTED) with tf.variable_scope(LOSS): self.delta = self.target_q - q_acted self.global_step = tf.Variable(0, trainable=False) self.loss = tf.reduce_mean(clipped_error(self.delta), name=LOSS) with tf.variable_scope(OPTIMIZER): self.learning_rate_step = tf.placeholder( tf.int64, None, name=LEARNING_RATE_STEP) self.learning_rate_op = tf.maximum( self.learning_rate_minimum, tf.train.exponential_decay(self.learning_rate, self.learning_rate_step, self.learning_rate_decay_step, self.learning_rate_decay, staircase=True)) self.optimizer = tf.train.RMSPropOptimizer( self.learning_rate_op, momentum=0.95, epsilon=0.01).minimize(self.loss) with tf.variable_scope(SUMMARY): scalar_summary_tags = ['average.reward', 'average.loss', 'average.q', \ 'episode.max reward', 'episode.min reward', 'episode.avg reward', \ 'episode.num of game', 'training.learning_rate'] self.summary_placeholders = {} self.summary_ops = {} for tag in scalar_summary_tags: self.summary_placeholders[tag] = \ tf.placeholder('float32', None, name=tag.replace(' ', '_')) self.summary_ops[tag] = \ tf.summary.scalar( name="{}-{}".format(self.env_name, tag), tensor=self.summary_placeholders[tag] ) histogram_summary_tags = ['episode.rewards', 'episode.actions'] for tag in histogram_summary_tags: self.summary_placeholders[tag] = \ tf.placeholder('float32', None, name=tag.replace(' ', '_')) self.summary_ops[tag] = \ tf.summary.histogram( name=tag, self.summary_placeholders[tag] ) self._summary_writer = tf.summary.FileWriter( config[TENSORBOARD_LOG_DIR]) self._summary_writer.add_graph(sess.graph) tf.initialize_all_variables().run() self._saver = tf.train.Saver(self.q.weights.values + [self.step_op], max_to_keep=30) self.load_model() self.update_target_network() def update_target_network(self): for name in self.q.weights.keys(): self.t_weights_assign_ops[name].eval({ self.q_weights_placeholders[name]: self.q.weights[name].eval() }) def inject_summary(self, tag_dict, step): summary_str_lists = self.sess.run( [self.summary_ops[tag] for tag in tag_dict.keys()], { self.summary_placeholders[tag]: value for tag, value in tag_dict.items() }) for summary_str in summary_str_lists: self.writer.add_summary(summary_str, self.step)
def run(comm, env, policy, policy_path, action_bound, optimizer): actor, actor_target, critic, critic_target = policy actor_opt, critic_opt = optimizer # set OU noise noise_c = OUNoise(action_dim=ACT_SIZE, action_bound=action_bound) noise_c.reset() buff = [] global_update = 0 global_step = 0 replay_memory = ReplayMemory(REPLAY_SIZE, SEED) #world reset if env.index == 0: env.reset_world() for id in range(MAX_EPISODES): #reset env.reset_pose() terminal = False ep_reward = 0 step = 1 # generate goal env.generate_goal_point() # get_state obs = env.get_laser_observation() obs_stack = deque([obs, obs, obs]) goal = np.asarray(env.get_local_goal()) speed = np.asarray(env.get_self_speed()) state = [obs_stack, goal, speed] # episode 1 while not terminal and not rospy.is_shutdown(): state_list = comm.gather(state, root=0) ## get_action #------------------------------------------------------------------------- # generate actions at rank==0 mean, action = generate_action(env=env, state_list=state_list, actor=actor, action_bound=action_bound) ## exploration #-------------------------------------------------------------------------- a = noise_c.get_action(mean, step) #noise = np.random.normal(0, exploration_noise, size=2) #action size check #a = mean + noise #a = a.clip(action_bound[0], action_bound[1]) # execute actions #------------------------------------------------------------------------- real_action = comm.scatter(a, root=0) ## run action #------------------------------------------------------------------------- env.control_vel(real_action) # rate.sleep() rospy.sleep(0.001) ## get reward #------------------------------------------------------------------------- # get informtion r, terminal, result = env.get_reward_and_terminate(step) ep_reward += r global_step += 1 #------------------------------------------------------------------------- # get next state #------------------------------------------------------------------------- s_next = env.get_laser_observation() left = obs_stack.popleft() obs_stack.append(s_next) goal_next = np.asarray(env.get_local_goal()) speed_next = np.asarray(env.get_self_speed()) state_next = [obs_stack, goal_next, speed_next] r_list = comm.gather(r, root=0) terminal_list = comm.gather(terminal, root=0) state_next_list = comm.gather(state_next, root=0) #------------------------------------------------------------------------- ## training #------------------------------------------------------------------------- if env.index == 0: ## save data in memory #------------------------------------------------------------------------- replay_memory.push(state[0], state[1], state[2], a, r_list, state_next[0], state_next[1], state_next[2], terminal_list) policy_list = [actor, actor_target, critic, critic_target] optimizer_list = [actor_opt, critic_opt] if len(replay_memory) > BATCH_SIZE: ## Update step by step #------------------------------------------------------------------------- ddpg_update_stage(policy=policy_list, optimizer=optimizer_list, batch_size=BATCH_SIZE, memory=replay_memory, epoch=EPOCH, replay_size=REPLAY_SIZE, gamma=GAMMA, num_step=BATCH_SIZE, num_env=NUM_ENV, frames=LASER_HIST, obs_size=OBS_SIZE, act_size=ACT_SIZE, tau=TAU) global_update += 1 step += 1 state = state_next ## save policy and log #------------------------------------------------------------------------- if env.index == 0: if global_update != 0 and global_update % 3000 == 0: torch.save(actor.state_dict(), policy_path + '/actor_{}'.format(global_update)) torch.save(critic.state_dict(), policy_path + '/critic_{}'.format(global_update)) logger.info( '########################## model saved when update {} times#########' '################'.format(global_update)) distance = np.sqrt((env.goal_point[0] - env.init_pose[0])**2 + (env.goal_point[1] - env.init_pose[1])**2) logger.info('Env %02d, Goal (%05.1f, %05.1f), Episode %05d, setp %03d, Reward %-5.1f, Distance %05.1f, %s' % \ (env.index, env.goal_point[0], env.goal_point[1], id + 1, step, ep_reward, distance, result)) logger_cal.info(ep_reward)
def run(comm, env, policy, critic, critic_opt, critic_target, policy_path, action_bound, optimizer): buff = [] global_update = 0 global_step = 0 # world reset if env.index == 0: env.reset_world() memory_position = 0 update = 0 # replay_memory replay_memory = ReplayMemory(REPLAY_SIZE, SEED) for id in range(MAX_EPISODES): # reset env.reset_pose() terminal = False ep_reward = 0 step = 1 # generate goal env.generate_goal_point() # get_state obs = env.get_laser_observation() obs_stack = deque([obs, obs, obs]) goal = np.asarray(env.get_local_goal()) speed = np.asarray(env.get_self_speed()) state = [obs_stack, goal, speed] # episode 1 while not terminal and not rospy.is_shutdown(): state_list = comm.gather(state, root=0) ## get_action #------------------------------------------------------------------------- # generate actions at rank==0 a, logprob, scaled_action = generate_action( env=env, state_list=state_list, policy=policy, action_bound=action_bound) # execute actions #------------------------------------------------------------------------- real_action = comm.scatter(scaled_action, root=0) ## run action #------------------------------------------------------------------------- env.control_vel(real_action) rospy.sleep(0.001) ## get reward #------------------------------------------------------------------------- # get informtion r, terminal, result = env.get_reward_and_terminate(step) ep_reward += r global_step += 1 #------------------------------------------------------------------------- # get next state #------------------------------------------------------------------------- s_next = env.get_laser_observation() left = obs_stack.popleft() obs_stack.append(s_next) goal_next = np.asarray(env.get_local_goal()) speed_next = np.asarray(env.get_self_speed()) state_next = [obs_stack, goal_next, speed_next] #------------------------------------------------------------------------- r_list = comm.gather(r, root=0) terminal_list = comm.gather(terminal, root=0) state_next_list = comm.gather(state_next, root=0) ## training #------------------------------------------------------------------------- if env.index == 0: # add data in replay_memory #------------------------------------------------------------------------- replay_memory.push(state[0], state[1], state[2], a, logprob, r_list, state_next[0], state_next[1], state_next[2], terminal_list) if len(replay_memory) > BATCH_SIZE: ## update #------------------------------------------------------------------------- update = sac_update_stage(policy=policy, optimizer=optimizer, critic=critic, critic_opt=critic_opt, critic_target=critic_target, batch_size=BATCH_SIZE, memory=replay_memory, epoch=EPOCH, replay_size=REPLAY_SIZE, tau=TAU, alpha=ALPHA, gamma=GAMMA, updates=update, update_interval=UPDATE_INTERVAL, num_step=BATCH_SIZE, num_env=NUM_ENV, frames=LASER_HIST, obs_size=OBS_SIZE, act_size=ACT_SIZE) buff = [] global_update += 1 update = update step += 1 state = state_next ## save policy #-------------------------------------------------------------------------------------------------------------- if env.index == 0: if global_update != 0 and global_update % 1000 == 0: torch.save(policy.state_dict(), policy_path + '/policy_{}'.format(global_update)) torch.save(critic.state_dict(), policy_path + '/critic_{}'.format(global_update)) logger.info( '########################## model saved when update {} times#########' '################'.format(global_update)) distance = np.sqrt((env.goal_point[0] - env.init_pose[0])**2 + (env.goal_point[1] - env.init_pose[1])**2) logger.info('Env %02d, Goal (%05.1f, %05.1f), Episode %05d, setp %03d, Reward %-5.1f, Distance %05.1f, %s' % \ (env.index, env.goal_point[0], env.goal_point[1], id + 1, step, ep_reward, distance, result)) logger_cal.info(ep_reward)
def run(comm, env, agent, policy_path, args): # Training Loop test_interval = 10 save_interval = 100 total_numsteps = 0 updates = 0 # world reset if env.index == 0: # step env.reset_gazebo_simulation() #Tesnorboard writer = SummaryWriter('test_runs/' + policy_path) # replay_memory memory = ReplayMemory(args.replay_size, args.seed) for i_episode in range(args.num_steps): episode_reward = 0 episode_steps = 0 done = False # Env reset env.set_gazebo_pose() # generate goal env.generate_goal_point_gazebo() # Get initial state frame = env.get_laser_observation() frame_stack = deque([frame, frame, frame]) goal = np.asarray(env.get_local_goal()) speed = np.asarray(env.get_self_speed()) state = [frame_stack, goal, speed] # Episode start while not done and not rospy.is_shutdown(): state_list = comm.gather(state, root=0) if env.index == 0: action = agent.select_action(state_list) else: action = None if env.index == 0: if len(memory) > args.batch_size: # Number of updates per step in environment for i in range(args.updates_per_step): # Update parameters of all the networks critic_1_loss, critic_2_loss, policy_loss, ent_loss, alpha = agent.update_parameters(memory, args.batch_size, updates) writer.add_scalar('loss/critic_1', critic_1_loss, updates) writer.add_scalar('loss/critic_2', critic_2_loss, updates) writer.add_scalar('loss/policy', policy_loss, updates) writer.add_scalar('loss/entropy_loss', ent_loss, updates) writer.add_scalar('entropy_temprature/alpha', alpha, updates) updates += 1 # Execute actions #------------------------------------------------------------------------- action_clip_bound = [[0, -1], [1, 1]] #### Action maximum, minimum values cliped_action = np.clip(action, a_min=action_clip_bound[0], a_max=action_clip_bound[1]) real_action = comm.scatter(cliped_action, root=0) #if real_action[0] >= 0.02 and real_action[0] <= 0.2: # real_action[0] = real_action[0] / 0.6 if real_action[0] < 0.10 and real_action[0] > 0: real_action[0] = 0.1 if real_action[1] > 0 and real_action[1] < 0.15: real_action[1] = 0.0 elif real_action[1] < 0 and real_action[1] > -0.15: real_action[1] = 0.0 env.control_vel(real_action) #rospy.sleep(0.001) ## Get reward and terminal state reward, done, result = env.get_reward_and_terminate(episode_steps) #print("Action : [{}, {}], Distance : {}, Reward : {}".format(real_action[0], real_action[1], env.distance, reward)) episode_steps += 1 total_numsteps += 1 episode_reward += reward # Get next state next_frame = env.get_laser_observation() left = frame_stack.popleft() frame_stack.append(next_frame) next_goal = np.asarray(env.get_local_goal()) next_speed = np.asarray(env.get_self_speed()) next_state = [frame_stack, next_goal, next_speed] r_list = comm.gather(reward, root=0) done_list = comm.gather(done, root=0) next_state_list = comm.gather(next_state, root=0) if env.index == 0: #meomry.list_push(state_list, action, r_list, next_state_list, done_list) for i in range(np.asarray(state_list).shape[0]): memory.push(state_list[i][0], state_list[i][1], state_list[i][2], action[i], r_list[i], next_state_list[i][0], next_state_list[i][1], next_state_list[i][2], done_list[i]) # Append transition to memory state = next_state #if total_numsteps > args.num_steps: # break if env.index == 0: writer.add_scalar('reward/train', episode_reward, i_episode) if env.index == 0: #if global_update != 0 and global_update % 20 == 0: if i_episode != 0 and i_episode % save_interval == 0: torch.save(agent.policy.state_dict(), policy_path + '/policy_epi_{}'.format(i_episode)) print('########################## policy model saved when update {} times#########' '################'.format(i_episode)) torch.save(agent.critic_1.state_dict(), policy_path + '/critic_1_epi_{}'.format(i_episode)) print('########################## critic model saved when update {} times#########' '################'.format(i_episode)) torch.save(agent.critic_2.state_dict(), policy_path + '/critic_2_epi_{}'.format(i_episode)) print('########################## critic model saved when update {} times#########' '################'.format(i_episode)) distance = np.sqrt((env.goal_point[0] - env.init_pose[0])**2 + (env.goal_point[1]-env.init_pose[1])**2) print("Env: {}, memory_size: {}, Goal: ({} , {}), Episode: {}, steps: {}, Reward: {}, Distance: {}, {}".format(env.index, len(memory), round(env.goal_point[0],2), round(env.goal_point[1],2), i_episode+1, episode_steps, round(episode_reward, 2), round(distance, 2), result))
class Agent(BaseAgent): '''Deep Trading Agent based on Deep Q Learning''' '''TODO: 1. add summary ops 2. timing and logging 3. model saving 4. increment self.step ''' def __init__(self, sess, logger, config, env): super(Agent, self).__init__(config) self.sess = sess self.logger = logger self.config = config params = DeepSenseParams(config) self.env = env self.history = History(logger, config) self.replay_memory = ReplayMemory(logger, config) with tf.variable_scope(STEPS): self.step_op = tf.Variable(0, trainable=False, name=STEP) self.step_input = tf.placeholder('int32', None, name=STEP_INPUT) self.step_assign_op = self.step_op.assign(self.step_input) self.build_dqn(params) def train(self): start_step = self.step_op.eval() num_episodes, self.update_count, ep_reward = 0, 0, 0. total_reward, self.total_loss, self.total_q = 0., 0., 0. max_avg_ep_reward = 0 ep_rewards, actions = [], [] self.env.new_random_episode(self.history) for self.step in range(start_step, self.max_step): if self.step == self.learn_start: num_episodes, self.update_count, ep_reward = 0, 0, 0. total_reward, self.total_loss, self.total_q = 0., 0., 0. ep_rewards, actions = [], [] # 1. predict action = self.predict(self.history.get()) # 2. act screen, reward, terminal = self.env.act(action) # 3. observe self.observe(screen, reward, action, terminal) if terminal: self.env.new_random_episode(self.history) num_episodes += 1 ep_rewards.append(ep_reward) ep_reward = 0. else: ep_reward += reward actions.append(action) total_reward += reward def predict(self, s_t, test_ep=None): ep = test_ep or (self.ep_end + max(0., (self.ep_start - self.ep_end) \ * (self.ep_end_t - max(0., self.step - self.learn_start)) / self.ep_end_t)) if random.random() < ep: action = random.randrange(self.env.action_size) else: action = self.q.action.eval({self.s_t: [s_t]})[0] return action def observe(self, screen, reward, action, terminal): #clip reward in the range min to max reward = max(self.min_reward, min(self.max_reward, reward)) self.history.add(screen) self.replay_memory.add(screen, reward, action, terminal) if self.step > self.learn_start: if self.step % self.train_frequency == 0: self.q_learning_mini_batch() if self.step % self.target_q_update_step == self.target_q_update_step - 1: self.update_target_network() def q_learning_mini_batch(self): if self.replay_memory.count >= self.replay_memory.history_length: s_t, action, reward, s_t_plus_1, terminal = self.replay_memory.sample( ) max_q_t_plus_1 = self.t_q.action.eval({self.t_s_t: s_t_plus_1}) terminal = np.array(terminal) + 0. target_q = reward + (1 - terminal) * max_q_t_plus_1 _, q_t, loss = self.sess.run( [self.optimizer, self.q.values, self.loss], { self.target_q: target_q, self.action: action, self.s_t: s_t, self.learning_rate_step: self.step, }) self.total_loss += loss self.total_q += q_t.mean() self.update_count += 1 def build_dqn(self, params): with tf.variable_scope(PREDICTION): self.s_t = tf.placeholder(dtype=tf.float32, shape=[ None, self.replay_memory.history_length, self.replay_memory.num_channels ]) self.q = DeepSense(params, self.logger, self.sess, self.config, name=Q_NETWORK) self.q.build_model(self.s_t) with tf.variable_scope(TARGET): self.t_s_t = tf.placeholder(dtype=tf.float32, shape=[ None, self.replay_memory.history_length, self.replay_memory.num_channels ]) self.t_q = DeepSense(params, self.logger, self.sess, self.config, name=T_Q_NETWORK) self.t_q.build_model(self.t_s_t, train=False) with tf.variable_scope(UPDATE_TARGET_NETWORK): self.q_weights_placeholders = {} self.t_weights_assign_ops = {} for name in self.q.weights.keys(): self.q_weights_placeholders[name] = tf.placeholder( tf.float32, self.q.weights[name].get_shape().as_list()) for name in self.q.weights.keys(): self.t_weights_assign_ops[name] = self.t_q.weights[ name].assign(self.q_weights_placeholders[name]) with tf.variable_scope(TRAINING): self.target_q = tf.placeholder(tf.float32, [None], name=TARGET_Q) self.action = tf.placeholder(tf.int64, [None], name=ACTION) action_one_hot = tf.one_hot(self.action, self.env.action_size, 1.0, 0.0, name=ACTION_ONE_HOT) q_acted = tf.reduce_sum(self.q.values * action_one_hot, reduction_indices=1, name=Q_ACTED) with tf.variable_scope(LOSS): self.delta = self.target_q - q_acted self.global_step = tf.Variable(0, trainable=False) self.loss = tf.reduce_mean(clipped_error(self.delta), name=LOSS) with tf.variable_scope(OPTIMIZER): self.learning_rate_step = tf.placeholder( tf.int64, None, name=LEARNING_RATE_STEP) self.learning_rate_op = tf.maximum( self.learning_rate_minimum, tf.train.exponential_decay(self.learning_rate, self.learning_rate_step, self.learning_rate_decay_step, self.learning_rate_decay, staircase=True)) self.optimizer = tf.train.RMSPropOptimizer( self.learning_rate_op, momentum=0.95, epsilon=0.01).minimize(self.loss) # tf.initialize_all_variables().run() #initialize the q network and the target network with the same weights # self.update_target_network() def update_target_network(self): for name in self.q.weights.keys(): self.t_weights_assign_ops[name].eval({ self.q_weights_placeholders[name]: self.q.weights[name].eval() })