def train_batch(self, batch_info: BatchInfo) -> None:
"""
Batch - the most atomic unit of learning.
For this reinforforcer, that involves:
1. Roll out the environmnent using current policy
2. Use that rollout to train the policy
"""
# Calculate environment rollout on the evaluation version of the model
self.model.train()
rollout = self.env_roller.rollout(batch_info, self.model,
self.settings.number_of_steps)
# Process rollout by the 'algo' (e.g. perform the advantage estimation)
rollout = self.algo.process_rollout(batch_info, rollout)
# Perform the training step
# Algo will aggregate data into this list:
batch_info['sub_batch_data'] = []
if self.settings.shuffle_transitions:
rollout = rollout.to_transitions()
if self.settings.stochastic_experience_replay:
# Always play experience at least once
experience_replay_count = max(
np.random.poisson(self.settings.experience_replay), 1)
else:
experience_replay_count = self.settings.experience_replay
# Repeat the experience N times
for i in range(experience_replay_count):
# We may potentially need to split rollout into multiple batches
if self.settings.batch_size >= rollout.frames():
batch_result = self.algo.optimizer_step(
batch_info=batch_info,
device=self.device,
model=self.model,
rollout=rollout.to_device(self.device))
batch_info['sub_batch_data'].append(batch_result)
else:
# Rollout too big, need to split in batches
for batch_rollout in rollout.shuffled_batches(
self.settings.batch_size):
batch_result = self.algo.optimizer_step(
batch_info=batch_info,
device=self.device,
model=self.model,
rollout=batch_rollout.to_device(self.device))
batch_info['sub_batch_data'].append(batch_result)
batch_info['frames'] = rollout.frames()
batch_info['episode_infos'] = rollout.episode_information()
# Even with all the experience replay, we count the single rollout as a single batch
batch_info.aggregate_key('sub_batch_data')
def train_batch(self, batch_info: BatchInfo) -> None:
"""
Batch - the most atomic unit of learning.
For this reinforforcer, that involves:
1. Roll out environment and store out experience in the buffer
2. Sample the buffer and train the algo on sample batch
"""
# Each DQN batch is
# 1. Roll out environment and store out experience in the buffer
self.model.eval()
# Helper variables for rollouts
episode_information = []
frames = 0
with torch.no_grad():
if not self.env_roller.is_ready_for_sampling():
while not self.env_roller.is_ready_for_sampling():
rollout = self.env_roller.rollout(batch_info, self.model)
episode_information.extend(rollout.episode_information())
frames += rollout.frames()
else:
for i in range(self.settings.batch_rollout_rounds):
rollout = self.env_roller.rollout(batch_info, self.model)
episode_information.extend(rollout.episode_information())
frames += rollout.frames()
batch_info['frames'] = frames
batch_info['episode_infos'] = episode_information
# 2. Sample the buffer and train the algo on sample batch
self.model.train()
# Algo will aggregate data into this list:
batch_info['sub_batch_data'] = []
for i in range(self.settings.batch_training_rounds):
sampled_rollout = self.env_roller.sample(batch_info, self.model)
batch_result = self.algo.optimizer_step(
batch_info=batch_info,
device=self.device,
model=self.model,
rollout=sampled_rollout
)
self.env_roller.update(rollout=sampled_rollout, batch_info=batch_result)
batch_info['sub_batch_data'].append(batch_result)
batch_info.aggregate_key('sub_batch_data')
def train_batch(self, batch_info: BatchInfo):
""" Single, most atomic 'step' of learning this reinforcer can perform """
batch_info['sub_batch_data'] = []
self.on_policy_train_batch(batch_info)
if self.settings.experience_replay > 0 and self.env_roller.is_ready_for_sampling(
):
if self.settings.stochastic_experience_replay:
experience_replay_count = np.random.poisson(
self.settings.experience_replay)
else:
experience_replay_count = self.settings.experience_replay
for i in range(experience_replay_count):
self.off_policy_train_batch(batch_info)
# Even with all the experience replay, we count the single rollout as a single batch
batch_info.aggregate_key('sub_batch_data')
def train_batch(self, batch_info: BatchInfo) -> None:
"""
Batch - the most atomic unit of learning.
For this reinforforcer, that involves:
1. Roll out environment and store out experience in the buffer
2. Sample the buffer and train the algo on sample batch
"""
# Each DQN batch is
# 1. Roll out environment and store out experience in the buffer
self.model.eval()
# Helper variables for rollouts
episode_information = []
rollout_actions = []
rollout_values = []
frames = 0
with torch.no_grad():
if not self.env_roller.is_ready_for_sampling():
while not self.env_roller.is_ready_for_sampling():
rollout = self.env_roller.rollout(batch_info, self.model)
maybe_episode_info = rollout['episode_information']
if maybe_episode_info is not None:
episode_information.append(maybe_episode_info)
frames += 1
rollout_actions.append(rollout['action'].detach().cpu().numpy())
rollout_values.append(rollout['value'].detach().cpu().numpy())
else:
for i in range(self.settings.batch_rollout_rounds):
rollout = self.env_roller.rollout(batch_info, self.model)
maybe_episode_info = rollout['episode_information']
if maybe_episode_info is not None:
episode_information.append(maybe_episode_info)
frames += 1
rollout_actions.append(rollout['action'].detach().cpu().numpy())
rollout_values.append(rollout['value'].detach().cpu().numpy())
batch_info['rollout_action_mean'] = np.mean(rollout_actions)
batch_info['rollout_action_std'] = np.std(rollout_actions)
batch_info['rollout_value_mean'] = np.std(rollout_values)
batch_info['frames'] = frames
batch_info['episode_infos'] = episode_information
# 2. Sample the buffer and train the algo on sample batch
self.model.train()
# Algo will aggregate data into this list:
batch_info['sub_batch_data'] = []
for i in range(self.settings.batch_training_rounds):
batch_sample = self.env_roller.sample(batch_info, self.model)
batch_result = self.algo.optimizer_step(
batch_info=batch_info,
device=self.device,
model=self.model,
rollout=batch_sample
)
self.env_roller.update(sample=batch_sample, batch_info=batch_result)
batch_info['sub_batch_data'].append(batch_result)
batch_info.aggregate_key('sub_batch_data')
