def _initialize(self):
"""Initialize non-common things."""
# load demo replay memory
with open(self.args.demo_path, "rb") as f:
demo = list(pickle.load(f))
# HER
if self.hyper_params["USE_HER"]:
if self.hyper_params["DESIRED_STATES_FROM_DEMO"]:
self.her.fetch_desired_states_from_demo(demo)
self.transitions_epi: list = list()
self.desired_state = np.zeros((1,))
demo = self.her.generate_demo_transitions(demo)
if not self.args.test:
# Replay buffers
demo_batch_size = self.hyper_params["DEMO_BATCH_SIZE"]
self.demo_memory = ReplayBuffer(len(demo), demo_batch_size)
self.demo_memory.extend(demo)
self.memory = ReplayBuffer(
self.hyper_params["BUFFER_SIZE"], self.hyper_params["BATCH_SIZE"]
)
# set hyper parameters
self.lambda1 = self.hyper_params["LAMBDA1"]
self.lambda2 = self.hyper_params["LAMBDA2"] / demo_batch_size
def _initialize(self):
"""Initialize non-common things."""
# load demo replay memory
with open(self.args.demo_path, "rb") as f:
demo = list(pickle.load(f))
# HER
if self.hyper_params["USE_HER"]:
self.her = HER(self.args.demo_path)
self.transitions_epi: list = list()
self.desired_state = np.zeros((1,))
self.hook_transition = True
demo = self.her.generate_demo_transitions(demo)
if not self.args.test:
# Replay buffers
self.demo_memory = ReplayBuffer(
len(demo), self.hyper_params["DEMO_BATCH_SIZE"]
)
self.demo_memory.extend(demo)
self.memory = ReplayBuffer(
self.hyper_params["BUFFER_SIZE"], self.hyper_params["BATCH_SIZE"]
)
# set hyper parameters
self.lambda1 = self.hyper_params["LAMBDA1"]
self.lambda2 = (
self.hyper_params["LAMBDA2"] / self.hyper_params["DEMO_BATCH_SIZE"]
)
def _initialize(self):
"""Initialize non-common things."""
if not self.args.test:
# replay memory
self.memory = ReplayBuffer(
self.hyper_params["BUFFER_SIZE"], self.hyper_params["BATCH_SIZE"]
)
def _initialize(self):
"""Initialize non-common things."""
if not self.args.test:
# load demo replay memory
demos = self._load_demos()
if self.use_n_step:
demos, demos_n_step = common_utils.get_n_step_info_from_demo(
demos, self.hyper_params["N_STEP"],
self.hyper_params["GAMMA"])
self.memory_n = ReplayBuffer(
buffer_size=self.hyper_params["BUFFER_SIZE"],
n_step=self.hyper_params["N_STEP"],
gamma=self.hyper_params["GAMMA"],
demo=demos_n_step,
)
# replay memory
self.beta = self.hyper_params["PER_BETA"]
self.memory = PrioritizedReplayBuffer(
self.hyper_params["BUFFER_SIZE"],
self.hyper_params["BATCH_SIZE"],
demo=demos,
alpha=self.hyper_params["PER_ALPHA"],
epsilon_d=self.hyper_params["PER_EPS_DEMO"],
)
def _initialize(self):
"""Initialize non-common things."""
self.use_n_step = self.hyper_params["N_STEP"] > 1
if not self.args.test:
# load demo replay memory
with open(self.args.demo_path, "rb") as f:
demos = pickle.load(f)
if self.use_n_step:
demos, demos_n_step = common_utils.get_n_step_info_from_demo(
demos, self.hyper_params["N_STEP"], self.hyper_params["GAMMA"]
)
# replay memory for multi-steps
self.memory_n = ReplayBuffer(
buffer_size=self.hyper_params["BUFFER_SIZE"],
n_step=self.hyper_params["N_STEP"],
gamma=self.hyper_params["GAMMA"],
demo=demos_n_step,
)
# replay memory for a single step
self.beta = self.hyper_params["PER_BETA"]
self.memory = PrioritizedReplayBuffer(
self.hyper_params["BUFFER_SIZE"],
self.hyper_params["BATCH_SIZE"],
demo=demos,
alpha=self.hyper_params["PER_ALPHA"],
epsilon_d=self.hyper_params["PER_EPS_DEMO"],
)
def __init__(
self,
env: gym.Env,
args: argparse.Namespace,
hyper_params: dict,
models: tuple,
optims: tuple,
noise: OUNoise,
):
"""Initialization.
Args:
env (gym.Env): openAI Gym environment with discrete action space
args (argparse.Namespace): arguments including hyperparameters and training settings
hyper_params (dict): hyper-parameters
models (tuple): models including actor and critic
optims (tuple): optimizers for actor and critic
noise (OUNoise): random noise for exploration
"""
AbstractAgent.__init__(self, env, args)
self.actor, self.actor_target, self.critic, self.critic_target = models
self.actor_optimizer, self.critic_optimizer = optims
self.hyper_params = hyper_params
self.curr_state = np.zeros((1, ))
self.noise = noise
# load the optimizer and model parameters
if args.load_from is not None and os.path.exists(args.load_from):
self.load_params(args.load_from)
# replay memory
self.memory = ReplayBuffer(hyper_params["BUFFER_SIZE"],
hyper_params["BATCH_SIZE"], self.args.seed)
def _initialize(self):
"""Initialize non-common things."""
if not self.args.test:
# replay memory
self.memory = ReplayBuffer(self.hyper_params["BUFFER_SIZE"],
self.hyper_params["BATCH_SIZE"])
# HER
if self.hyper_params["USE_HER"]:
# load demo replay memory
with open(self.args.demo_path, "rb") as f:
demo = pickle.load(f)
if self.hyper_params["DESIRED_STATES_FROM_DEMO"]:
self.her.fetch_desired_states_from_demo(demo)
self.transitions_epi = list()
self.desired_state = np.zeros((1, ))
demo = self.her.generate_demo_transitions(demo)
if not self.args.test:
# Replay buffers
self.memory = ReplayBuffer(self.hyper_params["BUFFER_SIZE"],
self.hyper_params["BATCH_SIZE"])
def __init__(
self,
env: gym.Env,
args: argparse.Namespace,
hyper_params: dict,
models: tuple,
optims: tuple,
exploration_noise: GaussianNoise,
target_policy_noise: GaussianNoise,
):
"""Initialization.
Args:
env (gym.Env): openAI Gym environment
args (argparse.Namespace): arguments including hyperparameters and training settings
hyper_params (dict): hyper-parameters
models (tuple): models including actor and critic
optims (tuple): optimizers for actor and critic
exploration_noise (GaussianNoise): random noise for exploration
target_policy_noise (GaussianNoise): random noise for target values
"""
Agent.__init__(self, env, args)
self.actor, self.actor_target = models[0:2]
self.critic1, self.critic2 = models[2:4]
self.critic_target1, self.critic_target2 = models[4:6]
self.actor_optim = optims[0]
self.critic_optim = optims[1]
self.hyper_params = hyper_params
self.curr_state = np.zeros((1,))
self.exploration_noise = exploration_noise
self.target_policy_noise = target_policy_noise
self.total_step = 0
self.episode_step = 0
self.update_step = 0
self.i_episode = 0
# load the optimizer and model parameters
if args.load_from is not None and os.path.exists(args.load_from):
self.load_params(args.load_from)
if not self.args.test:
# replay memory
self.memory = ReplayBuffer(
hyper_params["BUFFER_SIZE"], hyper_params["BATCH_SIZE"]
)
def _initialize(self):
"""Initialize non-common things."""
if not self.args.test:
# replay memory for a single step
self.beta = self.hyper_params["PER_BETA"]
self.memory = PrioritizedReplayBuffer(
self.hyper_params["BUFFER_SIZE"],
self.hyper_params["BATCH_SIZE"],
alpha=self.hyper_params["PER_ALPHA"],
)
# replay memory for multi-steps
if self.use_n_step:
self.memory_n = ReplayBuffer(
self.hyper_params["BUFFER_SIZE"],
n_step=self.hyper_params["N_STEP"],
gamma=self.hyper_params["GAMMA"],
)