def get_action_parameterization(self, core_output_size): if not self.cfg.adaptive_stddev and is_continuous_action_space(self.action_space): action_parameterization = ActionParameterizationContinuousNonAdaptiveStddev( self.cfg, core_output_size, self.action_space, ) else: action_parameterization = ActionParameterizationDefault(self.cfg, core_output_size, self.action_space) return action_parameterization
def __init__(self, cfg, core_out_size, action_space): super().__init__(cfg, action_space) assert not cfg.adaptive_stddev assert is_continuous_action_space(self.action_space), \ 'Non-adaptive stddev makes sense only for continuous action spaces' num_action_outputs = calc_num_logits(action_space) # calculate only action means using the policy neural network self.distribution_linear = nn.Linear(core_out_size, num_action_outputs // 2) # stddev is a single learned parameter initial_stddev = torch.empty([num_action_outputs // 2]) initial_stddev.fill_(math.log(self.cfg.initial_stddev)) self.learned_stddev = nn.Parameter(initial_stddev, requires_grad=True)
def __init__( self, worker_idx, policy_id, cfg, obs_space, action_space, report_queue, policy_worker_queues, shared_buffers, policy_lock, resume_experience_collection_cv, ): log.info('Initializing the learner %d for policy %d', worker_idx, policy_id) self.worker_idx = worker_idx self.policy_id = policy_id self.cfg = cfg # PBT-related stuff self.should_save_model = True # set to true if we need to save the model to disk on the next training iteration self.load_policy_id = None # non-None when we need to replace our parameters with another policy's parameters self.pbt_mutex = threading.Lock() self.new_cfg = None # non-None when we need to update the learning hyperparameters self.terminate = False self.num_batches_processed = 0 self.obs_space = obs_space self.action_space = action_space self.rollout_tensors = shared_buffers.tensor_trajectories self.traj_tensors_available = shared_buffers.is_traj_tensor_available self.policy_versions = shared_buffers.policy_versions self.stop_experience_collection = shared_buffers.stop_experience_collection self.stop_experience_collection_num_msgs = self.resume_experience_collection_num_msgs = 0 self.device = None self.dqn = None self.optimizer = None self.policy_lock = policy_lock self.resume_experience_collection_cv = resume_experience_collection_cv self.task_queue = MpQueue() self.report_queue = report_queue self.initialized_event = MultiprocessingEvent() self.initialized_event.clear() self.model_saved_event = MultiprocessingEvent() self.model_saved_event.clear() # queues corresponding to policy workers using the same policy # we send weight updates via these queues self.policy_worker_queues = policy_worker_queues self.experience_buffer_queue = Queue() self.tensor_batch_pool = ObjectPool() self.tensor_batcher = TensorBatcher(self.tensor_batch_pool) self.with_training = True # set to False for debugging no-training regime self.train_in_background = self.cfg.train_in_background_thread # set to False for debugging self.training_thread = Thread(target=self._train_loop) if self.train_in_background else None self.train_thread_initialized = threading.Event() self.is_training = False self.train_step = self.env_steps = 0 # decay rate at which summaries are collected # save summaries every 20 seconds in the beginning, but decay to every 4 minutes in the limit, because we # do not need frequent summaries for longer experiments self.summary_rate_decay_seconds = LinearDecay([(0, 20), (100000, 120), (1000000, 240)]) self.last_summary_time = 0 self.last_saved_time = self.last_milestone_time = 0 self.discarded_experience_over_time = deque([], maxlen=30) self.discarded_experience_timer = time.time() self.num_discarded_rollouts = 0 self.process = Process(target=self._run, daemon=True) if is_continuous_action_space(self.action_space) and self.cfg.exploration_loss == 'symmetric_kl': raise NotImplementedError('KL-divergence exploration loss is not supported with ' 'continuous action spaces. Use entropy exploration loss') if self.cfg.exploration_loss_coeff == 0.0: self.exploration_loss_func = lambda action_distr: 0.0 elif self.cfg.exploration_loss == 'entropy': self.exploration_loss_func = self.entropy_exploration_loss elif self.cfg.exploration_loss == 'symmetric_kl': self.exploration_loss_func = self.symmetric_kl_exploration_loss else: raise NotImplementedError(f'{self.cfg.exploration_loss} not supported!')