def _initialize(self):
"""Initialize non-common things."""
if not self.is_test:
# replay memory for a single step
self.memory = ReplayBuffer(
self.hyper_params.buffer_size,
self.hyper_params.batch_size,
)
self.memory = PrioritizedBufferWrapper(
self.memory, alpha=self.hyper_params.per_alpha)
# replay memory for multi-steps
if self.use_n_step:
self.memory_n = ReplayBuffer(
self.hyper_params.buffer_size,
self.hyper_params.batch_size,
n_step=self.hyper_params.n_step,
gamma=self.hyper_params.gamma,
)
build_args = dict(
hyper_params=self.hyper_params,
log_cfg=self.log_cfg,
env_name=self.env_info.name,
state_size=self.env_info.observation_space.shape,
output_size=self.env_info.action_space.n,
is_test=self.is_test,
load_from=self.load_from,
)
self.learner = build_learner(self.learner_cfg, build_args)
def _initialize(self):
"""Initialize non-common things."""
self.per_beta = self.hyper_params.per_beta
self.use_n_step = self.hyper_params.n_step > 1
if not self.args.test:
# load demo replay memory
with open(self.args.demo_path, "rb") as f:
demos = pickle.load(f)
if self.use_n_step:
demos, demos_n_step = common_utils.get_n_step_info_from_demo(
demos, self.hyper_params.n_step, self.hyper_params.gamma)
# replay memory for multi-steps
self.memory_n = ReplayBuffer(
max_len=self.hyper_params.buffer_size,
batch_size=self.hyper_params.batch_size,
n_step=self.hyper_params.n_step,
gamma=self.hyper_params.gamma,
demo=demos_n_step,
)
# replay memory for a single step
self.memory = ReplayBuffer(
self.hyper_params.buffer_size,
self.hyper_params.batch_size,
)
self.memory = PrioritizedBufferWrapper(
self.memory, alpha=self.hyper_params.per_alpha)
self.learner_cfg.type = "DDPGfDLearner"
self.learner = build_learner(self.learner_cfg)
def _spawn(self):
"""Intialize distributed worker, learner and centralized replay buffer."""
replay_buffer = ReplayBuffer(
self.hyper_params.buffer_size,
self.hyper_params.batch_size,
)
per_buffer = PrioritizedBufferWrapper(
replay_buffer, alpha=self.hyper_params.per_alpha)
self.global_buffer = ApeXBufferWrapper.remote(per_buffer, self.args,
self.hyper_params,
self.comm_cfg)
learner = build_learner(self.learner_cfg)
self.learner = ApeXLearnerWrapper.remote(learner, self.comm_cfg)
state_dict = learner.get_state_dict()
worker_build_args = dict(args=self.args, state_dict=state_dict)
self.workers = []
self.num_workers = self.hyper_params.num_workers
for rank in range(self.num_workers):
worker_build_args["rank"] = rank
worker = build_worker(self.worker_cfg,
build_args=worker_build_args)
apex_worker = ApeXWorkerWrapper.remote(worker, self.args,
self.comm_cfg)
self.workers.append(apex_worker)
self.logger = build_logger(self.logger_cfg)
self.processes = self.workers + [
self.learner, self.global_buffer, self.logger
]
def _initialize(self):
"""Initialize non-common things."""
if not self.args.test:
# load demo replay memory
demos = self._load_demos()
if self.use_n_step:
demos, demos_n_step = common_utils.get_n_step_info_from_demo(
demos, self.hyper_params.n_step, self.hyper_params.gamma)
self.memory_n = ReplayBuffer(
max_len=self.hyper_params.buffer_size,
batch_size=self.hyper_params.batch_size,
n_step=self.hyper_params.n_step,
gamma=self.hyper_params.gamma,
demo=demos_n_step,
)
# replay memory
self.memory = ReplayBuffer(
self.hyper_params.buffer_size,
self.hyper_params.batch_size,
demo=demos,
)
self.memory = PrioritizedBufferWrapper(
self.memory,
alpha=self.hyper_params.per_alpha,
epsilon_d=self.hyper_params.per_eps_demo,
)
self.learner_cfg.type = "DQfDLearner"
self.learner = build_learner(self.learner_cfg)
def _initialize(self):
"""Initialize non-common things."""
if not self.args.test:
self.memory = RecurrentReplayBuffer(
self.hyper_params.buffer_size,
self.hyper_params.batch_size,
self.hyper_params.sequence_size,
self.hyper_params.overlap_size,
n_step=self.hyper_params.n_step,
gamma=self.hyper_params.gamma,
)
self.memory = PrioritizedBufferWrapper(
self.memory, alpha=self.hyper_params.per_alpha)
# replay memory for multi-steps
if self.use_n_step:
self.memory_n = RecurrentReplayBuffer(
self.hyper_params.buffer_size,
self.hyper_params.batch_size,
self.hyper_params.sequence_size,
self.hyper_params.overlap_size,
n_step=self.hyper_params.n_step,
gamma=self.hyper_params.gamma,
)
self.learner = build_learner(self.learner_cfg)
def generate_prioritized_buffer(
buffer_length: int, batch_size: int, idx_lst=None, prior_lst=None
) -> Tuple[PrioritizedBufferWrapper, List]:
"""Generate Prioritized Replay Buffer with random Prior."""
buffer = ReplayBuffer(max_len=buffer_length, batch_size=batch_size)
prioritized_buffer = PrioritizedBufferWrapper(buffer)
priority = np.random.randint(10, size=buffer_length)
for i, j in enumerate(priority):
prioritized_buffer.sum_tree[i] = j
if idx_lst:
for i, j in list(zip(idx_lst, prior_lst)):
priority[i] = j
prioritized_buffer.sum_tree[i] = j
prop_lst = [i / sum(priority) for i in priority]
return prioritized_buffer, prop_lst
def _initialize(self):
"""Initialize non-common things."""
self.per_beta = self.hyper_params.per_beta
self.use_n_step = self.hyper_params.n_step > 1
if not self.is_test:
# load demo replay memory
with open(self.hyper_params.demo_path, "rb") as f:
demos = pickle.load(f)
if self.use_n_step:
demos, demos_n_step = common_utils.get_n_step_info_from_demo(
demos, self.hyper_params.n_step, self.hyper_params.gamma)
# replay memory for multi-steps
self.memory_n = ReplayBuffer(
max_len=self.hyper_params.buffer_size,
batch_size=self.hyper_params.batch_size,
n_step=self.hyper_params.n_step,
gamma=self.hyper_params.gamma,
demo=demos_n_step,
)
# replay memory for a single step
self.memory = ReplayBuffer(
self.hyper_params.buffer_size,
self.hyper_params.batch_size,
demo=demos,
)
self.memory = PrioritizedBufferWrapper(
self.memory,
alpha=self.hyper_params.per_alpha,
epsilon_d=self.hyper_params.per_eps_demo,
)
build_args = dict(
hyper_params=self.hyper_params,
log_cfg=self.log_cfg,
noise_cfg=self.noise_cfg,
env_name=self.env_info.name,
state_size=self.env_info.observation_space.shape,
output_size=self.env_info.action_space.shape[0],
is_test=self.is_test,
load_from=self.load_from,
)
self.learner = build_learner(self.learner_cfg, build_args)
def _initialize(self):
"""Initialize non-common things."""
if not self.is_test:
# load demo replay memory
demos = self._load_demos()
if self.use_n_step:
demos, demos_n_step = common_utils.get_n_step_info_from_demo(
demos, self.hyper_params.n_step, self.hyper_params.gamma)
self.memory_n = ReplayBuffer(
max_len=self.hyper_params.buffer_size,
batch_size=self.hyper_params.batch_size,
n_step=self.hyper_params.n_step,
gamma=self.hyper_params.gamma,
demo=demos_n_step,
)
# replay memory
self.memory = ReplayBuffer(
self.hyper_params.buffer_size,
self.hyper_params.batch_size,
demo=demos,
)
self.memory = PrioritizedBufferWrapper(
self.memory,
alpha=self.hyper_params.per_alpha,
epsilon_d=self.hyper_params.per_eps_demo,
)
build_args = dict(
hyper_params=self.hyper_params,
log_cfg=self.log_cfg,
env_name=self.env_info.name,
state_size=self.env_info.observation_space.shape,
output_size=self.env_info.action_space.n,
is_test=self.is_test,
load_from=self.load_from,
)
self.learner_cfg.type = "DQfDLearner"
self.learner = build_learner(self.learner_cfg, build_args)
def _spawn(self):
"""Intialize distributed worker, learner and centralized replay buffer."""
replay_buffer = ReplayBuffer(
self.hyper_params.buffer_size,
self.hyper_params.batch_size,
)
per_buffer = PrioritizedBufferWrapper(
replay_buffer, alpha=self.hyper_params.per_alpha)
self.global_buffer = ApeXBufferWrapper.remote(per_buffer,
self.hyper_params,
self.comm_cfg)
# Build learner
learner_build_args = dict(
hyper_params=self.hyper_params,
log_cfg=self.log_cfg,
env_name=self.env_info.name,
state_size=self.env_info.observation_space.shape,
output_size=self.env_info.action_space.n,
is_test=self.is_test,
load_from=self.load_from,
)
learner = build_learner(self.learner_cfg, learner_build_args)
self.learner = ApeXLearnerWrapper.remote(learner, self.comm_cfg)
# Build workers
state_dict = learner.get_state_dict()
worker_build_args = dict(
hyper_params=self.hyper_params,
backbone=self.learner_cfg.backbone,
head=self.learner_cfg.head,
loss_type=self.learner_cfg.loss_type,
state_dict=state_dict,
env_name=self.env_info.name,
state_size=self.env_info.observation_space.shape,
output_size=self.env_info.action_space.n,
is_atari=self.env_info.is_atari,
max_episode_steps=self.max_episode_steps,
)
self.workers = []
self.num_workers = self.hyper_params.num_workers
for rank in range(self.num_workers):
worker_build_args["rank"] = rank
worker = build_worker(self.worker_cfg,
build_args=worker_build_args)
apex_worker = ApeXWorkerWrapper.remote(worker, self.comm_cfg)
self.workers.append(apex_worker)
# Build logger
logger_build_args = dict(
log_cfg=self.log_cfg,
comm_cfg=self.comm_cfg,
backbone=self.learner_cfg.backbone,
head=self.learner_cfg.head,
env_name=self.env_info.name,
is_atari=self.env_info.is_atari,
state_size=self.env_info.observation_space.shape,
output_size=self.env_info.action_space.n,
max_update_step=self.hyper_params.max_update_step,
episode_num=self.episode_num,
max_episode_steps=self.max_episode_steps,
is_log=self.is_log,
is_render=self.is_render,
interim_test_num=self.interim_test_num,
)
self.logger = build_logger(self.logger_cfg, logger_build_args)
self.processes = self.workers + [
self.learner, self.global_buffer, self.logger
]