def test_store_and_get(buffer: SingleEpisodeBuffer):
# store single non terminal transition
transition = Transition(state={"observation": np.array([1, 2, 3])},
action=1,
reward=1,
game_over=False)
buffer.store(transition)
assert buffer.length() == 1
assert buffer.num_complete_episodes() == 0
assert buffer.num_transitions_in_complete_episodes() == 0
assert buffer.num_transitions() == 1
# get the single stored transition
episode = buffer.get(0)
assert episode.length() == 1
assert episode.get_first_transition(
) is transition # check addresses are the same
assert episode.get_last_transition(
) is transition # check addresses are the same
# store single terminal transition
transition = Transition(state={"observation": np.array([1, 2, 3])},
action=1,
reward=1,
game_over=True)
buffer.store(transition)
assert buffer.length() == 1
assert buffer.num_complete_episodes() == 1
assert buffer.num_transitions_in_complete_episodes() == 2
# check that the episode is valid
episode = buffer.get(0)
assert episode.length() == 2
assert episode.get_transition(0).total_return == 1 + 0.99
assert episode.get_transition(1).total_return == 1
assert buffer.mean_reward() == 1
# only one episode in the replay buffer
episode = buffer.get(1)
assert episode is None
# adding transitions after the first episode was closed
transition = Transition(state={"observation": np.array([1, 2, 3])},
action=1,
reward=0,
game_over=False)
buffer.store(transition)
assert buffer.length() == 1
assert buffer.num_complete_episodes() == 0
assert buffer.num_transitions_in_complete_episodes() == 0
# still only one episode
assert buffer.get(1) is None
assert buffer.mean_reward() == 0
def test_update_episode(her):
episode = Episode()
for i in range(10):
episode.insert(
Transition(
state={
'observation': np.array([i]),
'desired_goal': np.array([i + 1]),
'achieved_goal': np.array([i + 1])
},
action=i,
game_over=i == 9,
reward=0 if i == 9 else -1,
))
her.store_episode(episode)
# print('her._num_transitions', her._num_transitions)
# 10 original transitions, and 9 transitions * 4 hindsight episodes
assert her.num_transitions() == 10 + (4 * 9)
# make sure that the goal state was never sampled from the past
for transition in her.transitions:
assert transition.state['desired_goal'] > transition.state[
'observation']
assert transition.next_state['desired_goal'] >= transition.next_state[
'observation']
if transition.reward == 0:
assert transition.game_over
else:
assert not transition.game_over
def load_csv(self, csv_dataset: CsvDataset) -> None:
"""
Restore the replay buffer contents from a csv file.
The csv file is assumed to include a list of transitions.
:param csv_dataset: A construct which holds the dataset parameters
"""
self.assert_not_frozen()
df = pd.read_csv(csv_dataset.filepath)
if len(df) > self.max_size[1]:
screen.warning(
"Warning! The number of transitions to load into the replay buffer ({}) is "
"bigger than the max size of the replay buffer ({}). The excessive transitions will "
"not be stored.".format(len(df), self.max_size[1]))
episode_ids = df['episode_id'].unique()
progress_bar = ProgressBar(len(episode_ids))
state_columns = [
col for col in df.columns if col.startswith('state_feature')
]
for e_id in episode_ids:
progress_bar.update(e_id)
df_episode_transitions = df[df['episode_id'] == e_id]
episode = Episode()
for (_, current_transition), (_, next_transition) in zip(
df_episode_transitions[:-1].iterrows(),
df_episode_transitions[1:].iterrows()):
state = np.array(
[current_transition[col] for col in state_columns])
next_state = np.array(
[next_transition[col] for col in state_columns])
episode.insert(
Transition(
state={'observation': state},
action=current_transition['action'],
reward=current_transition['reward'],
next_state={'observation': next_state},
game_over=False,
info={
'all_action_probabilities':
ast.literal_eval(
current_transition['all_action_probabilities'])
}))
# Set the last transition to end the episode
if csv_dataset.is_episodic:
episode.get_last_transition().game_over = True
self.store_episode(episode)
# close the progress bar
progress_bar.update(len(episode_ids))
progress_bar.close()
self.shuffle_episodes()
def update_transition_before_adding_to_replay_buffer(self, transition: Transition) -> Transition:
"""
Allows agents to update the transition just before adding it to the replay buffer.
Can be useful for agents that want to tweak the reward, termination signal, etc.
:param transition: the transition to update
:return: the updated transition
"""
transition.game_over = False if self.current_episode_steps_counter ==\
self.parent_level_manager.environment.env._max_episode_steps\
else transition.game_over
return transition
def episode():
episode = []
for i in range(10):
episode.append(
Transition(
state={
'observation': np.array([i]),
'desired_goal': np.array([i]),
'achieved_goal': np.array([i])
},
action=i,
))
return episode
def test_clean(buffer: SingleEpisodeBuffer):
# add several transitions and then clean the buffer
transition = Transition(state={"observation": np.array([1, 2, 3])},
action=1,
reward=1,
game_over=False)
for i in range(10):
buffer.store(transition)
assert buffer.num_transitions() == 10
buffer.clean()
assert buffer.num_transitions() == 0
# add more transitions after the clean and make sure they were really cleaned
transition = Transition(state={"observation": np.array([1, 2, 3])},
action=1,
reward=1,
game_over=True)
buffer.store(transition)
assert buffer.num_transitions() == 1
assert buffer.num_transitions_in_complete_episodes() == 1
assert buffer.num_complete_episodes() == 1
for i in range(10):
assert buffer.sample(1)[0] is transition
def create_dataset(dataset_root, output_path):
maybe_download(dataset_root)
dataset_root = os.path.join(dataset_root, 'AgentHuman')
train_set_root = os.path.join(dataset_root, 'SeqTrain')
validation_set_root = os.path.join(dataset_root, 'SeqVal')
# training set extraction
memory = ExperienceReplay(max_size=(MemoryGranularity.Transitions,
sys.maxsize))
train_set_files = sorted(os.listdir(train_set_root))
print("found {} files".format(len(train_set_files)))
progress_bar = ProgressBar(len(train_set_files))
for file_idx, file in enumerate(train_set_files[:3000]):
progress_bar.update(file_idx, "extracting file {}".format(file))
train_set = h5py.File(os.path.join(train_set_root, file), 'r')
observations = train_set['rgb'][:] # forward camera
measurements = np.expand_dims(train_set['targets'][:, 10],
-1) # forward speed
actions = train_set['targets'][:, :3] # steer, gas, brake
# actions[:, :2] = actions[:, 1:3]
# actions[:, 2] = train_set['targets'][:, 0] # gas, brake, steer
# actions[:, 1] -= actions[:, 2]
# actions = actions[:, :2][:, ::-1]
high_level_commands = train_set['targets'][:, 24].astype(
'int') - 2 # follow lane, left, right, straight
file_length = train_set['rgb'].len()
assert train_set['rgb'].len() == train_set['targets'].len()
for transition_idx in range(file_length):
transition = Transition(state={
'CameraRGB':
observations[transition_idx],
'measurements':
measurements[transition_idx],
'high_level_command':
high_level_commands[transition_idx]
},
action=actions[transition_idx],
reward=0)
memory.store(transition)
progress_bar.close()
print("Saving pickle file to {}".format(output_path))
memory.save(output_path)
def observe(self, env_response: EnvResponse) -> bool:
"""
Given a response from the environment, distill the observation from it and store it for later use.
The response should be a dictionary containing the performed action, the new observation and measurements,
the reward, a game over flag and any additional information necessary.
:param env_response: result of call from environment.step(action)
:return:
"""
# filter the env_response
filtered_env_response = self.input_filter.filter(env_response)[0]
# inject agent collected statistics, if required
if self.ap.algorithm.use_accumulated_reward_as_measurement:
if 'measurements' in filtered_env_response.next_state:
filtered_env_response.next_state['measurements'] = np.append(
filtered_env_response.next_state['measurements'],
self.total_shaped_reward_in_current_episode)
else:
filtered_env_response.next_state['measurements'] = np.array(
[self.total_shaped_reward_in_current_episode])
# if we are in the first step in the episode, then we don't have a a next state and a reward and thus no
# transition yet, and therefore we don't need to store anything in the memory.
# also we did not reach the goal yet.
if self.current_episode_steps_counter == 0:
# initialize the current state
self.curr_state = filtered_env_response.next_state
return env_response.game_over
else:
transition = Transition(
state=copy.copy(self.curr_state),
action=self.last_action_info.action,
reward=filtered_env_response.reward,
next_state=filtered_env_response.next_state,
game_over=filtered_env_response.game_over,
info=filtered_env_response.info)
# now that we have formed a basic transition - the next state progresses to be the current state
self.curr_state = filtered_env_response.next_state
# make agent specific changes to the transition if needed
transition = self.update_transition_before_adding_to_replay_buffer(
transition)
# merge the intrinsic reward in
if self.ap.algorithm.scale_external_reward_by_intrinsic_reward_value:
transition.reward = transition.reward * (
1 + self.last_action_info.action_intrinsic_reward)
else:
transition.reward = transition.reward + self.last_action_info.action_intrinsic_reward
# sum up the total shaped reward
self.total_shaped_reward_in_current_episode += transition.reward
self.total_reward_in_current_episode += env_response.reward
self.shaped_reward.add_sample(transition.reward)
self.reward.add_sample(env_response.reward)
# add action info to transition
if type(self.parent).__name__ == 'CompositeAgent':
transition.add_info(self.parent.last_action_info.__dict__)
else:
transition.add_info(self.last_action_info.__dict__)
# create and store the transition
if self.phase in [RunPhase.TRAIN, RunPhase.HEATUP]:
# for episodic memories we keep the transitions in a local buffer until the episode is ended.
# for regular memories we insert the transitions directly to the memory
if isinstance(self.memory, EpisodicExperienceReplay):
self.current_episode_buffer.insert(transition)
else:
self.call_memory('store', transition)
if self.ap.visualization.dump_in_episode_signals:
self.update_step_in_episode_log()
return transition.game_over
def load_csv(self, csv_dataset: CsvDataset,
input_filter: InputFilter) -> None:
"""
Restore the replay buffer contents from a csv file.
The csv file is assumed to include a list of transitions.
:param csv_dataset: A construct which holds the dataset parameters
:param input_filter: A filter used to filter the CSV data before feeding it to the memory.
"""
self.assert_not_frozen()
df = pd.read_csv(csv_dataset.filepath)
if len(df) > self.max_size[1]:
screen.warning(
"Warning! The number of transitions to load into the replay buffer ({}) is "
"bigger than the max size of the replay buffer ({}). The excessive transitions will "
"not be stored.".format(len(df), self.max_size[1]))
episode_ids = df["episode_id"].unique()
progress_bar = ProgressBar(len(episode_ids))
state_columns = [
col for col in df.columns if col.startswith("state_feature")
]
for e_id in episode_ids:
progress_bar.update(e_id)
df_episode_transitions = df[df["episode_id"] == e_id]
input_filter.reset()
if len(df_episode_transitions) < 2:
# we have to have at least 2 rows in each episode for creating a transition
continue
episode = Episode()
transitions = []
for (_, current_transition), (_, next_transition) in zip(
df_episode_transitions[:-1].iterrows(),
df_episode_transitions[1:].iterrows()):
state = np.array(
[current_transition[col] for col in state_columns])
next_state = np.array(
[next_transition[col] for col in state_columns])
transitions.append(
Transition(
state={"observation": state},
action=int(current_transition["action"]),
reward=current_transition["reward"],
next_state={"observation": next_state},
game_over=False,
info={
"all_action_probabilities":
ast.literal_eval(
current_transition["all_action_probabilities"])
},
), )
transitions = input_filter.filter(transitions, deep_copy=False)
for t in transitions:
episode.insert(t)
# Set the last transition to end the episode
if csv_dataset.is_episodic:
episode.get_last_transition().game_over = True
self.store_episode(episode)
# close the progress bar
progress_bar.update(len(episode_ids))
progress_bar.close()
progress_bar.update(file_idx, "extracting file {}".format(file))
train_set = h5py.File(os.path.join(train_set_root, file), 'r')
observations = train_set['rgb'][:] # forward camera
measurements = np.expand_dims(train_set['targets'][:, 10],
-1) # forward speed
actions = train_set['targets'][:, :3] # steer, gas, brake
actions[:, 1] -= actions[:, 2]
actions = actions[:, :2][:, ::-1]
high_level_commands = train_set['targets'][:, 24].astype(
'int') - 2 # follow lane, left, right, straight
file_length = train_set['rgb'].len()
assert train_set['rgb'].len() == train_set['targets'].len()
for transition_idx in range(file_length):
transition = Transition(state={
'forward_camera':
observations[transition_idx],
'measurements':
measurements[transition_idx],
'high_level_command':
high_level_commands[transition_idx]
},
action=actions[transition_idx],
reward=0)
memory.store(transition)
progress_bar.close()
print("Saving pickle file.")
memory.save('carla_train_set_replay_buffer.p')