def get_vr_feeder(self, batch): """ Returns feed dictionary for `value replay` loss estimation subgraph. """ if not self.use_off_policy_aac: # use single pass of network on same off-policy batch feeder = feed_dict_from_nested(self.local_network.vr_state_in, batch['state']) feeder.update(feed_dict_rnn_context(self.local_network.vr_lstm_state_pl_flatten, batch['context'])) feeder.update( { self.local_network.vr_batch_size: batch['batch_size'], self.local_network.vr_time_length: batch['time_steps'], self.local_network.vr_a_r_in: batch['last_action_reward'], self.vr_target: batch['r'] } ) else: feeder = {self.vr_target: batch['r']} # redundant actually :) return feeder
def process(self, sess): """ Grabs a on_policy_rollout that's been produced by the thread runner. If data identified as 'train data' - samples off_policy rollout[s] from replay memory and updates the parameters; writes summaries if any. The update is then sent to the parameter server. If on_policy_rollout contains 'test data' - no policy update is performed and learn rate is set to zero; Meanwile test data are stored in replay memory. """ # Collect data from child thread runners: data = self.get_data() # Test or train: if at least one rollout from parallel runners is test rollout - # set learn rate to zero for entire minibatch. Doh. try: is_train = not np.asarray([ env['state']['metadata']['type'] for env in data['on_policy'] ]).any() except KeyError: is_train = True # Copy weights from local policy to local target policy: if self.use_target_policy and self.local_steps % self.pi_prime_update_period == 0: sess.run(self.sync_pi_prime) if is_train: # If there is no testing rollouts - copy weights from shared to local new_policy: sess.run(self.sync_pi) #self.log.debug('is_train: {}'.format(is_train)) # Process minibatch for on-policy train step: on_policy_rollouts = data['on_policy'] on_policy_batch = batch_stack([ r.process( gamma=self.model_gamma, gae_lambda=self.model_gae_lambda, size=self.rollout_length, time_flat=self.time_flat, ) for r in on_policy_rollouts ]) # Feeder for on-policy AAC loss estimation graph: feed_dict = feed_dict_from_nested(self.local_network.on_state_in, on_policy_batch['state']) feed_dict.update( feed_dict_rnn_context(self.local_network.on_lstm_state_pl_flatten, on_policy_batch['context'])) feed_dict.update({ self.local_network.on_a_r_in: on_policy_batch['last_action_reward'], self.local_network.on_batch_size: on_policy_batch['batch_size'], self.local_network.on_time_length: on_policy_batch['time_steps'], self.on_pi_act_target: on_policy_batch['action'], self.on_pi_adv_target: on_policy_batch['advantage'], self.on_pi_r_target: on_policy_batch['r'], self.local_network.train_phase: is_train, # Zeroes learn rate, [+ batch_norm] }) if self.use_target_policy: feed_dict.update( feed_dict_from_nested(self.local_network_prime.on_state_in, on_policy_batch['state'])) feed_dict.update( feed_dict_rnn_context( self.local_network_prime.on_lstm_state_pl_flatten, on_policy_batch['context'])) feed_dict.update({ self.local_network_prime.on_batch_size: on_policy_batch['batch_size'], self.local_network_prime.on_time_length: on_policy_batch['time_steps'], self.local_network_prime.on_a_r_in: on_policy_batch['last_action_reward'] }) if self.use_memory: # Process rollouts from replay memory: off_policy_rollouts = data['off_policy'] off_policy_batch = batch_stack([ r.process( gamma=self.model_gamma, gae_lambda=self.model_gae_lambda, size=self.replay_rollout_length, time_flat=self.time_flat, ) for r in off_policy_rollouts ]) # Feeder for off-policy AAC loss estimation graph: off_policy_feed_dict = feed_dict_from_nested( self.local_network.off_state_in, off_policy_batch['state']) off_policy_feed_dict.update( feed_dict_rnn_context( self.local_network.off_lstm_state_pl_flatten, off_policy_batch['context'])) off_policy_feed_dict.update({ self.local_network.off_a_r_in: off_policy_batch['last_action_reward'], self.local_network.off_batch_size: off_policy_batch['batch_size'], self.local_network.off_time_length: off_policy_batch['time_steps'], self.off_pi_act_target: off_policy_batch['action'], self.off_pi_adv_target: off_policy_batch['advantage'], self.off_pi_r_target: off_policy_batch['r'], }) if self.use_target_policy: off_policy_feed_dict.update( feed_dict_from_nested( self.local_network_prime.off_state_in, off_policy_batch['state'])) off_policy_feed_dict.update({ self.local_network_prime.off_batch_size: off_policy_batch['batch_size'], self.local_network_prime.off_time_length: off_policy_batch['time_steps'], self.local_network_prime.off_a_r_in: off_policy_batch['last_action_reward'] }) off_policy_feed_dict.update( feed_dict_rnn_context( self.local_network_prime.off_lstm_state_pl_flatten, off_policy_batch['context'])) feed_dict.update(off_policy_feed_dict) # Update with reward prediction subgraph: if self.use_reward_prediction: # Rebalanced 50/50 sample for RP: rp_rollouts = data['off_policy_rp'] rp_batch = batch_stack([ rp.process_rp(self.rp_reward_threshold) for rp in rp_rollouts ]) feed_dict.update(self.get_rp_feeder(rp_batch)) # Pixel control ... if self.use_pixel_control: feed_dict.update(self.get_pc_feeder(off_policy_batch)) # VR... if self.use_value_replay: feed_dict.update(self.get_vr_feeder(off_policy_batch)) # Every worker writes train episode and model summaries: ep_summary_feeder = {} # Look for train episode summaries from all env runners: for stat in data['ep_summary']: if stat is not None: for key in stat.keys(): if key in ep_summary_feeder.keys(): ep_summary_feeder[key] += [stat[key]] else: ep_summary_feeder[key] = [stat[key]] # Average values among thread_runners, if any, and write episode summary: if ep_summary_feeder != {}: ep_summary_feed_dict = { self.ep_summary[key]: np.average(list) for key, list in ep_summary_feeder.items() } if self.test_mode: # Atari: fetched_episode_stat = sess.run( self.ep_summary['atari_stat_op'], ep_summary_feed_dict) else: # BTGym fetched_episode_stat = sess.run( self.ep_summary['btgym_stat_op'], ep_summary_feed_dict) self.summary_writer.add_summary(fetched_episode_stat, sess.run(self.global_episode)) self.summary_writer.flush() # Every worker writes test episode summaries: test_ep_summary_feeder = {} # Look for test episode summaries: for stat in data['test_ep_summary']: if stat is not None: for key in stat.keys(): if key in test_ep_summary_feeder.keys(): test_ep_summary_feeder[key] += [stat[key]] else: test_ep_summary_feeder[key] = [stat[key]] # Average values among thread_runners, if any, and write episode summary: if test_ep_summary_feeder != {}: test_ep_summary_feed_dict = { self.ep_summary[key]: np.average(list) for key, list in test_ep_summary_feeder.items() } fetched_test_episode_stat = sess.run( self.ep_summary['test_btgym_stat_op'], test_ep_summary_feed_dict) self.summary_writer.add_summary(fetched_test_episode_stat, sess.run(self.global_episode)) self.summary_writer.flush() wirte_model_summary =\ self.local_steps % self.model_summary_freq == 0 # Look for renderings (chief worker only, always 0-numbered environment): if self.task == 0: if data['render_summary'][0] is not None: render_feed_dict = { self.ep_summary[key]: pic for key, pic in data['render_summary'][0].items() } renderings = sess.run(self.ep_summary['render_op'], render_feed_dict) #if False: # if self.test_mode: # renderings = sess.run(self.ep_summary['atari_render_op'], render_feed_dict) # # else: # renderings = sess.run(self.ep_summary['btgym_render_op'], render_feed_dict) self.summary_writer.add_summary(renderings, sess.run(self.global_episode)) self.summary_writer.flush() #fetches = [self.train_op, self.local_network.debug] # include policy debug shapes fetches = [self.train_op] if wirte_model_summary: fetches_last = fetches + [self.model_summary_op, self.inc_step] else: fetches_last = fetches + [self.inc_step] # Do a number of SGD train epochs: # When doing more than one epoch, we actually use only last summary: for i in range(self.num_epochs - 1): fetched = sess.run(fetches, feed_dict=feed_dict) fetched = sess.run(fetches_last, feed_dict=feed_dict) if wirte_model_summary: self.summary_writer.add_summary(tf.Summary.FromString(fetched[-2]), fetched[-1]) self.summary_writer.flush() self.local_steps += 1
def process(self, sess): """ Grabs a on_policy_rollout that's been produced by the thread runner, samples off_policy rollout[s] from replay memory and updates the parameters. The update is then sent to the parameter server. """ # Copy weights from local policy to local target policy: if self.use_target_policy and self.local_steps % self.pi_prime_update_period == 0: sess.run(self.sync_pi_prime) # Copy weights from shared to local new_policy: sess.run(self.sync_pi) # Collect data from child thread runners: data = self.get_data() # Process minibatch for on-policy train step: on_policy_rollouts = data['on_policy'] on_policy_batch = batch_stack([ r.process( gamma=self.model_gamma, gae_lambda=self.model_gae_lambda, size=self.rollout_length, time_flat=self.time_flat, ) for r in on_policy_rollouts ]) # Feeder for on-policy AAC loss estimation graph: feed_dict = feed_dict_from_nested(self.local_network.on_state_in, on_policy_batch['state']) feed_dict.update( feed_dict_rnn_context(self.local_network.on_lstm_state_pl_flatten, on_policy_batch['context'])) feed_dict.update({ self.local_network.on_a_r_in: on_policy_batch['last_action_reward'], self.local_network.on_batch_size: on_policy_batch['batch_size'], self.local_network.on_time_length: on_policy_batch['time_steps'], self.on_pi_act_target: on_policy_batch['action'], self.on_pi_adv_target: on_policy_batch['advantage'], self.on_pi_r_target: on_policy_batch['r'], self.local_network.train_phase: True, }) if self.use_target_policy: feed_dict.update( feed_dict_from_nested(self.local_network_prime.on_state_in, on_policy_batch['state'])) feed_dict.update( feed_dict_rnn_context( self.local_network_prime.on_lstm_state_pl_flatten, on_policy_batch['context'])) feed_dict.update({ self.local_network_prime.on_batch_size: on_policy_batch['batch_size'], self.local_network_prime.on_time_length: on_policy_batch['time_steps'], self.local_network_prime.on_a_r_in: on_policy_batch['last_action_reward'] }) if self.use_memory: # Process rollouts from replay memory: off_policy_rollouts = data['off_policy'] off_policy_batch = batch_stack([ r.process( gamma=self.model_gamma, gae_lambda=self.model_gae_lambda, size=self.replay_rollout_length, time_flat=self.time_flat, ) for r in off_policy_rollouts ]) # Feeder for off-policy AAC loss estimation graph: off_policy_feed_dict = feed_dict_from_nested( self.local_network.off_state_in, off_policy_batch['state']) off_policy_feed_dict.update( feed_dict_rnn_context( self.local_network.off_lstm_state_pl_flatten, off_policy_batch['context'])) off_policy_feed_dict.update({ self.local_network.off_a_r_in: off_policy_batch['last_action_reward'], self.local_network.off_batch_size: off_policy_batch['batch_size'], self.local_network.off_time_length: off_policy_batch['time_steps'], self.off_pi_act_target: off_policy_batch['action'], self.off_pi_adv_target: off_policy_batch['advantage'], self.off_pi_r_target: off_policy_batch['r'], }) if self.use_target_policy: off_policy_feed_dict.update( feed_dict_from_nested( self.local_network_prime.off_state_in, off_policy_batch['state'])) off_policy_feed_dict.update({ self.local_network_prime.off_batch_size: off_policy_batch['batch_size'], self.local_network_prime.off_time_length: off_policy_batch['time_steps'], self.local_network_prime.off_a_r_in: off_policy_batch['last_action_reward'] }) off_policy_feed_dict.update( feed_dict_rnn_context( self.local_network_prime.off_lstm_state_pl_flatten, off_policy_batch['context'])) feed_dict.update(off_policy_feed_dict) # Update with reward prediction subgraph: if self.use_reward_prediction: # Rebalanced 50/50 sample for RP: rp_rollouts = data['off_policy_rp'] rp_batch = batch_stack([ rp.process_rp(self.rp_reward_threshold) for rp in rp_rollouts ]) feed_dict.update(self.get_rp_feeder(rp_batch)) # Pixel control ... if self.use_pixel_control: feed_dict.update(self.get_pc_feeder(off_policy_batch)) # VR... if self.use_value_replay: feed_dict.update(self.get_vr_feeder(off_policy_batch)) # Every worker writes episode and model summaries: ep_summary_feeder = {} # Collect episode summaries from all env runners: for stat in data['ep_summary']: if stat is not None: for key in stat.keys(): if key in ep_summary_feeder.keys(): ep_summary_feeder[key] += [stat[key]] else: ep_summary_feeder[key] = [stat[key]] # Average values among thread_runners, if any, and write episode summary: if ep_summary_feeder != {}: ep_summary_feed_dict = { self.ep_summary[key]: np.average(list) for key, list in ep_summary_feeder.items() } if self.test_mode: # Atari: fetched_episode_stat = sess.run( self.ep_summary['test_stat_op'], ep_summary_feed_dict) else: # BTGym fetched_episode_stat = sess.run(self.ep_summary['stat_op'], ep_summary_feed_dict) self.summary_writer.add_summary(fetched_episode_stat, sess.run(self.global_episode)) self.summary_writer.flush() wirte_model_summary =\ self.local_steps % self.model_summary_freq == 0 # Look for renderings (chief worker only, always 0-numbered environment): if self.task == 0: if data['render_summary'][0] is not None: render_feed_dict = { self.ep_summary[key]: pic for key, pic in data['render_summary'][0].items() } if self.test_mode: renderings = sess.run(self.ep_summary['test_render_op'], render_feed_dict) else: renderings = sess.run(self.ep_summary['render_op'], render_feed_dict) self.summary_writer.add_summary(renderings, sess.run(self.global_episode)) self.summary_writer.flush() fetches = [self.train_op] if wirte_model_summary: fetches_last = fetches + [self.model_summary_op, self.inc_step] else: fetches_last = fetches + [self.inc_step] # Do a number of SGD train epochs: # When doing more than one epoch, we actually use only last summary: for i in range(self.num_epochs - 1): fetched = sess.run(fetches, feed_dict=feed_dict) fetched = sess.run(fetches_last, feed_dict=feed_dict) if wirte_model_summary: self.summary_writer.add_summary(tf.Summary.FromString(fetched[-2]), fetched[-1]) self.summary_writer.flush() self.local_steps += 1