def __init__(self, env, normalize_observations=True, normalize_returns=True,
clip_observations=10., clip_rewards=10., gamma=0.99, epsilon=1e-8):
super().__init__(env)
self.ob_rms = RunningMeanStd(shape=self.observation_space.shape) if normalize_observations else None
self.ret_rms = RunningMeanStd(shape=()) if normalize_returns else None
self.clipob = clip_observations
self.cliprew = clip_rewards
self.ret = 0.0
self.gamma = gamma
self.epsilon = epsilon
def __init__(self, environment, device, batch_size, buffer_capacity, buffer_initial_size, noise_std_dev,
normalize_observations=False):
self.device = device
self.batch_size = batch_size
self.buffer_capacity = buffer_capacity
self.buffer_initial_size = buffer_initial_size
self.normalize_observations = normalize_observations
self.device = device
self._environment = environment
self.backend = DequeBufferBackend(
buffer_capacity=self.buffer_capacity,
observation_space=environment.observation_space,
action_space=environment.action_space
)
self.last_observation = self.environment.reset()
len_action_space = self.environment.action_space.shape[-1]
self.noise_process = OrnsteinUhlenbeckNoiseProcess(
np.zeros(len_action_space), float(noise_std_dev) * np.ones(len_action_space)
)
self.ob_rms = RunningMeanStd(shape=self.environment.observation_space.shape) if normalize_observations else None
self.clip_obs = 10.0
def __init__(self, environment, device, replay_buffer: ReplayBuffer, discount_factor: typing.Optional[float]=None,
normalize_returns: bool=False, forward_steps: int=1, action_noise: typing.Optional[nn.Module]=None):
self._environment = environment
self.device = device
self.replay_buffer = replay_buffer
self.normalize_returns = normalize_returns
self.forward_steps = forward_steps
self.discount_factor = discount_factor
self.action_noise = action_noise.to(self.device) if action_noise is not None else None
if self.normalize_returns:
assert self.discount_factor is not None, \
"TransitionReplayEnvRoller must have a discount factor defined if normalize_returns is turned on"
if self.forward_steps > 1:
assert self.discount_factor is not None, \
"TransitionReplayEnvRoller must have a discount factor defined if forward_steps is larger than one"
self.ret_rms = RunningMeanStd(shape=()) if normalize_returns else None
# Initial observation
self.last_observation_cpu = torch.from_numpy(self.environment.reset()).clone()
self.last_observation = self.last_observation_cpu.to(self.device)
# Return normalization
self.clip_obs = 5.0
self.accumulated_returns = np.zeros(environment.num_envs, dtype=np.float32)
def __init__(self,
venv,
ob=True,
ret=True,
clipob=10.,
cliprew=10.,
gamma=0.99,
epsilon=1e-8):
VecEnvWrapper.__init__(self, venv)
self.ob_rms = RunningMeanStd(
shape=self.observation_space.shape) if ob else None
self.ret_rms = RunningMeanStd(shape=()) if ret else None
self.clipob = clipob
self.cliprew = cliprew
self.ret = np.zeros(self.num_envs)
self.gamma = gamma
self.epsilon = epsilon