def create_reward_vector(agent, locations, move_actions):
"""
Creates the default linear reward vector.
:param Agent agent: the PsychSim agent capable of retrieving the features' values given a state.
:param list[str] locations: the list of possible world locations.
:param list[ActionSet] move_actions: the list of the agent's move_actions.
:rtype: LinearRewardVector
:return: the linear reward vector.
"""
world = agent.world
features = []
features.append(ValueComparisonLinearRewardFeature(
'Before Middle', world, get_mission_seconds_key(),
MISSION_PHASE_END_TIMES[MISSION_PHASES.index(MIDDLE_STR)], '<'))
features.append(ValueComparisonLinearRewardFeature(
'After Middle', world, get_mission_seconds_key(),
MISSION_PHASE_END_TIMES[MISSION_PHASES.index(MIDDLE_STR)] - 1, '>'))
# features.append(LocationFrequencyReward('Location Frequency', agent, locations, False))
features.append(LocationVisitedReward('Location Visited', agent, locations))
features.append(NumericLinearRewardFeature('Triaged Green', get_triaged_key(agent, 'Green')))
features.append(NumericLinearRewardFeature('Triaged Gold', get_triaged_key(agent, 'Gold')))
features.append(ValueComparisonLinearRewardFeature('See White', world, get_fov_key(agent), 'White', '=='))
features.append(ValueComparisonLinearRewardFeature('See Red', world, get_fov_key(agent), 'Red', '=='))
features.extend([ActionLinearRewardFeature(
'Move ' + next(iter(action))['object'], agent, action) for action in move_actions])
return LinearRewardVector(features)
def collect_sub_trajectories(self, coordinates, locations, neighbors,
trajectory):
# only consider first half of mission
idx = 0
for idx in range(len(trajectory)):
secs = trajectory[idx][0].getFeature(get_mission_seconds_key(),
unique=True)
if secs >= 5 * 60:
break
# collect sub-trajectories from player's trajectory
trajectories = sample_spread_sub_trajectories(trajectory[:idx],
self.num_trajectories,
self.length)
logging.info(
'Collected {} trajectories of length {} from original trajectory (length {}).'
.format(self.num_trajectories, self.length, idx + 1))
plot_trajectories(self.triage_agent,
trajectories,
locations,
neighbors,
os.path.join(
self._output_dir,
'sub-trajectories.{}'.format(self.img_format)),
coordinates,
title='Training Sub-Trajectories')
return trajectories
def __init__(self, name, agent, all_locations, inverse=True, max_frequency=1):
"""
Creates a new reward feature.
:param Agent agent: the PsychSim agent capable of retrieving the feature's value given a state.
:param str name: the label for this reward feature.
:param list[str] all_locations: all the world's locations.
:param bool inverse: whether to take the inverse frequency, i.e., `time - freq`.
:param int max_frequency: the maximum frequency that the agent can achieve (either for any or all locations).
"""
super().__init__(name, 1. / max_frequency) # use max frequency as normalization factor to keep var in [0,1]
self.agent = agent
self.world = agent.world
self.inverse = inverse
self.all_locations = all_locations
self.location_feat = get_location_key(agent)
self.time_feat = get_mission_seconds_key()
def get_actions_durations(trajectories, agents):
"""
Gets the average durations of actions according to the given trajectories.
:param list[list[(World, Distribution)]] trajectories: the set of trajectories, containing sequences of state-action pairs.
:param Agent or list[Agent] agents: a list with the agent for each trajectory set whose actions durations we want to retrieve.
:rtype: dict[str,(float,float)]
:return: the mean and std error action duration.
"""
if isinstance(agents, Agent):
agents = [agents] * len(trajectories)
assert len(trajectories) == len(
agents), 'One agent per set of trajectories has to be provided'
data = {}
clock_key = get_mission_seconds_key()
for i in range(len(trajectories)):
trajectory = trajectories[i]
for t in range(len(trajectory) - 1):
# compute clock diff
duration = trajectory[t + 1][0].getFeature(clock_key, unique=True) - \
trajectory[t][0].getFeature(clock_key, unique=True)
# get action and register duration
a_dist = trajectory[t][1]
for a, p in a_dist.items():
a = str(a).replace('{}-'.format(agents[i].name),
'').replace('_',
' ') # get clean action name
if a not in data:
data[a] = []
data[a].append(p * duration)
return OrderedDict({
a: [np.mean(data[a]), np.std(data[a]) / len(data[a])]
for a in sorted(data)
})
def process_players_data(analyzer, output_dir, clear=False, verbosity=1):
"""
Collects statistics regarding the players' behavior, mean location and action frequencies from the collected trajectories.
:param RewardModelAnalyzer analyzer: the reward model analyzer containing the necessary data.
:param str output_dir: the directory in which to save the results.
:param bool clear: whether to clear the directory before processing.
:param int verbosity: the verbosity level of the log file.
:return:
"""
create_clear_dir(output_dir, clear)
change_log_handler(os.path.join(output_dir, 'post-process.log'), verbosity)
file_names = list(analyzer.trajectories)
logging.info('\n=================================')
logging.info('Analyzing mean player behavior for {} results...'.format(
len(file_names)))
# separates stats by map name
map_files = {}
for file_name in file_names:
map_table = analyzer.map_tables[file_name]
map_name = map_table.name.lower()
if map_name not in map_files:
map_files[map_name] = []
map_files[map_name].append(file_name)
for map_name, files in map_files.items():
map_table = analyzer.map_tables[files[0]]
locations = map_table.rooms_list
trajectories = [analyzer.trajectories[filename] for filename in files]
agents = [
trajectories[i][-1][0].agents[analyzer.agent_names[files[i]]]
for i in range(len(files))
]
# saves mean location frequencies
location_data = get_locations_frequencies(trajectories, agents,
locations)
plot_bar(location_data,
'Mean Location Visitation Frequencies',
os.path.join(
output_dir,
'{}-loc-frequencies.{}'.format(map_name,
analyzer.img_format)),
y_label='Frequency')
# saves mean action frequencies
act_data = get_actions_frequencies(trajectories, agents)
plot_bar(act_data,
'Mean Action Execution Frequencies',
os.path.join(
output_dir,
'{}-action-frequencies.{}'.format(map_name,
analyzer.img_format)),
y_label='Frequency')
# saves mean action durations
act_data = get_actions_durations(trajectories, agents)
plot_bar(act_data,
'Mean Action Durations',
os.path.join(
output_dir,
'{}-action-durations.{}'.format(map_name,
analyzer.img_format)),
y_label='Duration (secs)')
# saves all player trajectories
plot_trajectories(agents,
trajectories,
locations,
map_table.adjacency,
os.path.join(
output_dir, '{}-trajectories.{}'.format(
map_name, analyzer.img_format)),
map_table.coordinates,
title='Player Trajectories')
# saves trajectory length
traj_len_data = OrderedDict({
analyzer.get_player_name(file_name):
len(analyzer.trajectories[file_name])
for file_name in file_names
})
traj_len_data = {
name: traj_len_data[name]
for name in sorted(traj_len_data)
}
plot_bar(
traj_len_data, 'Player Trajectory Length',
os.path.join(output_dir,
'trajectory-length.{}'.format(analyzer.img_format)))
# saves game mission times
mission_time_data = {}
for file_name in file_names:
mission_time_feat = get_mission_seconds_key()
world = analyzer.trajectories[file_name][-1][0]
state = copy.deepcopy(world.state)
state.select(True)
mission_time_data[analyzer.get_player_name(
file_name)] = world.getFeature(mission_time_feat, state, True)
mission_time_data = {
name: mission_time_data[name]
for name in sorted(mission_time_data)
}
plot_bar(
mission_time_data, 'Player Mission Time (secs)',
os.path.join(output_dir,
'mission-time.{}'.format(analyzer.img_format)))