def __init__( self, capacity: int = 10000, storage_unit: str = "timesteps", num_shards: int = 1, learning_starts: int = 1000, replay_batch_size: int = 1, prioritized_replay_alpha: float = 0.6, prioritized_replay_beta: float = 0.4, prioritized_replay_eps: float = 1e-6, replay_mode: str = "independent", replay_sequence_length: int = 1, replay_burn_in: int = 0, replay_zero_init_states: bool = True, replay_ratio: float = 0.66, ): """Initializes MixInMultiAgentReplayBuffer instance. Args: capacity: Number of batches to store in total. storage_unit (str): Either 'sequences' or 'timesteps'. Specifies how experiences are stored. num_shards: The number of buffer shards that exist in total (including this one). learning_starts: Number of timesteps after which a call to `replay()` will yield samples (before that, `replay()` will return None). capacity: The capacity of the buffer. Note that when `replay_sequence_length` > 1, this is the number of sequences (not single timesteps) stored. replay_batch_size: The batch size to be sampled (in timesteps). Note that if `replay_sequence_length` > 1, `self.replay_batch_size` will be set to the number of sequences sampled (B). prioritized_replay_alpha: Alpha parameter for a prioritized replay buffer. Use 0.0 for no prioritization. prioritized_replay_beta: Beta parameter for a prioritized replay buffer. prioritized_replay_eps: Epsilon parameter for a prioritized replay buffer. replay_mode: One of "independent" or "lockstep". Determined, whether in the multiagent case, sampling is done across all agents/policies equally. replay_sequence_length: The sequence length (T) of a single sample. If > 1, we will sample B x T from this buffer. replay_burn_in: The burn-in length in case `replay_sequence_length` > 0. This is the number of timesteps each sequence overlaps with the previous one to generate a better internal state (=state after the burn-in), instead of starting from 0.0 each RNN rollout. replay_zero_init_states: Whether the initial states in the buffer (if replay_sequence_length > 0) are alwayas 0.0 or should be updated with the previous train_batch state outputs. replay_ratio: Ratio of replayed samples in the returned batches. E.g. a ratio of 0.0 means only return new samples (no replay), a ratio of 0.5 means always return newest sample plus one old one (1:1), a ratio of 0.66 means always return the newest sample plus 2 old (replayed) ones (1:2), etc... """ if not 0 < replay_ratio < 1: raise ValueError("Replay ratio must be within [0, 1]") MultiAgentReplayBuffer.__init__( self, capacity, storage_unit, num_shards, learning_starts, replay_batch_size, prioritized_replay_alpha, prioritized_replay_beta, prioritized_replay_eps, replay_mode, replay_sequence_length, replay_burn_in, replay_zero_init_states, ) self.replay_ratio = replay_ratio self.replay_proportion = None if self.replay_ratio != 1.0: self.replay_proportion = self.replay_ratio / (1.0 - self.replay_ratio) # Last added batch(es). self.last_added_batches = collections.defaultdict(list)
def __init__( self, capacity: int = 10000, storage_unit: str = "timesteps", num_shards: int = 1, replay_batch_size: int = 1, learning_starts: int = 1000, replay_mode: str = "independent", replay_sequence_length: int = 1, replay_burn_in: int = 0, replay_zero_init_states: bool = True, prioritized_replay_alpha: float = 0.6, prioritized_replay_beta: float = 0.4, prioritized_replay_eps: float = 1e-6, underlying_buffer_config: dict = None, **kwargs ): """Initializes a MultiAgentReplayBuffer instance. Args: num_shards: The number of buffer shards that exist in total (including this one). storage_unit: Either 'timesteps', 'sequences' or 'episodes'. Specifies how experiences are stored. If they are stored in episodes, replay_sequence_length is ignored. If they are stored in episodes, replay_sequence_length is ignored. learning_starts: Number of timesteps after which a call to `replay()` will yield samples (before that, `replay()` will return None). capacity: The capacity of the buffer. Note that when `replay_sequence_length` > 1, this is the number of sequences (not single timesteps) stored. replay_batch_size: The batch size to be sampled (in timesteps). Note that if `replay_sequence_length` > 1, `self.replay_batch_size` will be set to the number of sequences sampled (B). prioritized_replay_alpha: Alpha parameter for a prioritized replay buffer. Use 0.0 for no prioritization. prioritized_replay_beta: Beta parameter for a prioritized replay buffer. prioritized_replay_eps: Epsilon parameter for a prioritized replay buffer. replay_sequence_length: The sequence length (T) of a single sample. If > 1, we will sample B x T from this buffer. replay_burn_in: The burn-in length in case `replay_sequence_length` > 0. This is the number of timesteps each sequence overlaps with the previous one to generate a better internal state (=state after the burn-in), instead of starting from 0.0 each RNN rollout. replay_zero_init_states: Whether the initial states in the buffer (if replay_sequence_length > 0) are alwayas 0.0 or should be updated with the previous train_batch state outputs. underlying_buffer_config: A config that contains all necessary constructor arguments and arguments for methods to call on the underlying buffers. This replaces the standard behaviour of the underlying PrioritizedReplayBuffer. The config follows the conventions of the general replay_buffer_config. kwargs for subsequent calls of methods may also be included. Example: "replay_buffer_config": {"type": PrioritizedReplayBuffer, "capacity": 10, "storage_unit": "timesteps", prioritized_replay_alpha: 0.5, prioritized_replay_beta: 0.5, prioritized_replay_eps: 0.5} **kwargs: Forward compatibility kwargs. """ if "replay_mode" in kwargs and ( kwargs["replay_mode"] == "lockstep" or kwargs["replay_mode"] == ReplayMode.LOCKSTEP ): if log_once("lockstep_mode_not_supported"): logger.error( "Replay mode `lockstep` is not supported for " "MultiAgentPrioritizedReplayBuffer. " "This buffer will run in `independent` mode." ) kwargs["replay_mode"] = "independent" if underlying_buffer_config is not None: if log_once("underlying_buffer_config_not_supported"): logger.info( "PrioritizedMultiAgentReplayBuffer instantiated " "with underlying_buffer_config. This will " "overwrite the standard behaviour of the " "underlying PrioritizedReplayBuffer." ) prioritized_replay_buffer_config = underlying_buffer_config else: prioritized_replay_buffer_config = { "type": PrioritizedReplayBuffer, "alpha": prioritized_replay_alpha, "beta": prioritized_replay_beta, } shard_capacity = capacity // num_shards MultiAgentReplayBuffer.__init__( self, shard_capacity, storage_unit, **kwargs, underlying_buffer_config=prioritized_replay_buffer_config, replay_batch_size=replay_batch_size, learning_starts=learning_starts, replay_mode=replay_mode, replay_sequence_length=replay_sequence_length, replay_burn_in=replay_burn_in, replay_zero_init_states=replay_zero_init_states, ) self.prioritized_replay_eps = prioritized_replay_eps self.update_priorities_timer = TimerStat()