def __init__(self, py_client: types.ReverbClient, table_name: Union[Text, Sequence[Text]], sequence_length: int, stride_length: int = 1, priority: Union[float, int] = 1, pad_end_of_episodes: bool = False): """Creates an instance of the ReverbAddTrajectoryObserver. If multiple table_names and sequence lengths are provided data will only be stored once but be available for sampling with multiple sequence lengths from the respective reverb tables. **Note**: This observer is designed to work with py_drivers only, and does not support batches. Args: py_client: Python client for the reverb replay server. table_name: The table name(s) where samples will be written to. sequence_length: The sequence_length used to write to the given table. stride_length: The integer stride for the sliding window for overlapping sequences. The default value of `1` creates an item for every window. Using `L = sequence_length` this means items are created for times `{0, 1, .., L-1}, {1, 2, .., L}, ...`. In contrast, `stride_length = L` will create an item only for disjoint windows `{0, 1, ..., L-1}, {L, ..., 2 * L - 1}, ...`. priority: Initial priority for new samples in the RB. pad_end_of_episodes: At the end of an episode, the cache is dropped by default. When `pad_end_of_episodes = True`, the cache gets padded with boundary steps (last->first) with `0` values everywhere and padded items of `sequence_length` are written to Reverb. The last padded item starts with a boundary step from the episode. This ensures that the last few steps are not less likely to get sampled compared to middle steps, this is most useful for environments that have useful rewards at the end of episodes. Note: because we do not pad at the beginning of an episode, for `sequence_length = N > 1` scenarios, the first `N-1` steps in an episode are sampled less frequently than all other steps. This generally does not impact training performance. However, if you have an environment where the only meaningful rewards are at the beginning of the episodes, you may consider filing a feature request to support padding in the front as well. """ if isinstance(table_name, Text): self._table_names = [table_name] else: self._table_names = table_name self._sequence_length = sequence_length self._stride_length = stride_length self._priority = priority self._pad_end_of_episodes = pad_end_of_episodes self._py_client = py_client # TODO(b/153700282): Use a single writer with max_sequence_length=max(...) # once Reverb Dataset with emit_timesteps=True returns properly shaped # sequences. self._writer = py_client.trajectory_writer( num_keep_alive_refs=sequence_length + 1) self._cached_steps = 0 self._last_trajectory = None
def __init__(self, py_client: types.ReverbClient, table_name: Union[Text, Sequence[Text]], sequence_length: int, stride_length: int = 1, priority: Union[float, int] = 1, pad_end_of_episodes: bool = False, tile_end_of_episodes: bool = False): """Creates an instance of the ReverbAddTrajectoryObserver. If multiple table_names and sequence lengths are provided data will only be stored once but be available for sampling with multiple sequence lengths from the respective reverb tables. **Note**: This observer is designed to work with py_drivers only, and does not support batches. Args: py_client: Python client for the reverb replay server. table_name: The table name(s) where samples will be written to. sequence_length: The sequence_length used to write to the given table. stride_length: The integer stride for the sliding window for overlapping sequences. The default value of `1` creates an item for every window. Using `L = sequence_length` this means items are created for times `{0, 1, .., L-1}, {1, 2, .., L}, ...`. In contrast, `stride_length = L` will create an item only for disjoint windows `{0, 1, ..., L-1}, {L, ..., 2 * L - 1}, ...`. priority: Initial priority for new samples in the RB. pad_end_of_episodes: At the end of an episode, the cache is dropped by default. When `pad_end_of_episodes = True`, the cache gets padded with boundary steps (last->first) with `0` values everywhere and padded items of `sequence_length` are written to Reverb. tile_end_of_episodes: If `pad_end_of_episodes` is True then, the last padded item starts with a boundary step from the episode. When this option is True the following items will be generated: F, M, L, P M, L, P, P L, P, P, P If False, only a single one will be generated: F, M, L, P For training recurrent models on environments where required information is only available at the start of the episode it is useful to set `tile_end_of_episodes=False` and the `sequence_length` to be the length of the longest episode. Raises: ValueError: If `tile_end_of_episodes` is set without `pad_end_of_episodes`. """ if isinstance(table_name, Text): self._table_names = [table_name] else: self._table_names = table_name self._sequence_length = sequence_length self._stride_length = stride_length self._priority = priority self._pad_end_of_episodes = pad_end_of_episodes self._tile_end_of_episodes = tile_end_of_episodes if tile_end_of_episodes and not pad_end_of_episodes: raise ValueError("Must set `pad_end_of_episodes=True` when using " "`tile_end_of_episodes`") self._py_client = py_client # TODO(b/153700282): Use a single writer with max_sequence_length=max(...) # once Reverb Dataset with emit_timesteps=True returns properly shaped # sequences. self._writer = py_client.trajectory_writer( num_keep_alive_refs=sequence_length + 1, get_signature_timeout_ms=None) self._cached_steps = 0 self._last_trajectory = None
def __init__(self, py_client: types.ReverbClient, table_name: Union[Text, Sequence[Text]], max_sequence_length: int, priority: Union[float, int] = 1, bypass_partial_episodes: bool = False): """Creates an instance of the ReverbAddEpisodeObserver. **Note**: This observer is designed to work with py_drivers only, and does not support batches. TODO(b/158865335): Optionally truncate long episodes and add to buffer. Args: py_client: Python client for the reverb replay server. table_name: The table name(s) where samples will be written to. max_sequence_length: An integer. `max_sequence_length` used to write to the replay buffer tables. This defines the size of the internal buffer controlling the `upper` limit of the number of timesteps which can be referenced in a single prioritized item. Note that this is the maximum number of trajectories across all the cached episodes that you are writing into the replay buffer (e.g. `number_of_episodes`). `max_sequence_length` is not a limit of how many timesteps or items that can be inserted into the replay buffer. Note that, since `max_sequence_length` controls the size of internal buffer, it is suggested not to set this value to a very large number. If the number of steps in an episode is more than `max_sequence_length`, only items up to `max_sequence_length` is written into the table. priority: Initial priority for the table. bypass_partial_episodes: If `False` (default) and an episode length is greater than `max_sequence_length`, a `ValueError` is raised. If set to `True`, the episodes with length more than `max_sequence_length` do not cause a `ValueError`. These episodes are bypassed (will NOT be written into the replay buffer) and an error message is shown to the user. Note that in this case (`bypass_partial_episodes=True`), the steps for episodes with length more than `max_sequence_length` are wasted and thrown away. This decision is made to guarantee that the replay buffer always has FULL episodes. Note that, `max_sequence_length` is just an upper bound. Raises: ValueError: If `priority` is not numeric. ValueError: If max_sequence_length is not positive. """ if max_sequence_length <= 0: raise ValueError( "`max_sequence_length` must be an integer greater equal one.") if isinstance(table_name, Text): self._table_names = [table_name] else: self._table_names = table_name self._max_sequence_length = max_sequence_length self._priority = priority self._py_client = py_client self._writer = py_client.trajectory_writer( num_keep_alive_refs=self._max_sequence_length + 1) self._cached_steps = 0 self._bypass_partial_episodes = bypass_partial_episodes self._overflow_episode = False