def __init__(self, eval_keyword: str, args):
self.eval_keyword = eval_keyword
GameEnvironment.__init__(self, args=args, agent_type='evaluation')
self.eval_name = args.eval_name
# load params and evaluators
self.control_param, self.control_evaluator = \
load_param_and_evaluator(eval_keyword=eval_keyword, args=args, model_type='control')
self.stop_param, self.stop_evaluator = \
load_param_and_evaluator(eval_keyword=eval_keyword, args=args, model_type='stop')
self.high_level_param, self.high_level_evaluator = \
load_param_and_evaluator(eval_keyword=eval_keyword, args=args, model_type='high')
# set image type
self.image_type = self.high_level_param.image_type
if 'd' in self.image_type:
from model import DeepLabModel, prepare_deeplab_model
self.deeplab_model: DeepLabModel = prepare_deeplab_model()
self.final_images = []
self.eval_dataset, self.eval_sentences = load_evaluation_dataset(
self.high_level_param)
self.eval_transforms = list(
map(lambda x: x[0].state.transform, self.eval_dataset))
self.high_level_sentences = self.eval_sentences
logger.info('fetched {} sentences from {}'.format(
len(self.high_level_sentences),
self.high_level_param.eval_keyword.lower()))
self.softmax = torch.nn.Softmax(dim=1)
EvaluationDirectory.__init__(self, *self.eval_info)
self.high_level_data_dict = dict()
def load_model_single_trajectory(eval_dir: EvaluationDirectory,
traj_index: int) -> EvaluationTrajectory:
try:
with open(str(eval_dir.state_path(traj_index)), 'r') as file:
data = json.load(file)
except:
raise FileNotFoundError('failed to load {}'.format(
eval_dir.state_path(traj_index)))
collided = False if 'collided' not in data else parse_bool(
data['collided'])
data_frames = [
DriveDataFrame.load_from_str(f) for f in data['data_frames']
]
info = EvaluationUnitInfo(eval_dir.data_keyword, eval_dir.exp_index,
eval_dir.exp_name, eval_dir.exp_step, traj_index)
return EvaluationTrajectory(info, data_frames, collided)
def load_model_single_trajectory_group(
eval_dir: EvaluationDirectory) -> EvaluationTrajectoryGroup:
group_info = EvaluationUnitInfo(eval_dir.data_keyword, eval_dir.exp_index,
eval_dir.exp_name, eval_dir.exp_step, -1)
trajectories = []
for traj_index in eval_dir.traj_indices_from_state_dir():
trajectories.append(load_model_single_trajectory(eval_dir, traj_index))
return EvaluationTrajectoryGroup(group_info, trajectories)
def evaluation_directory_from_path(eval_path: Path) -> EvaluationDirectory:
pattern = 'exp([\d]+)\/([\w-]+)\/step([\d]+)/([\w,]+)'
res = re.findall(pattern, str(eval_path))
if not res or len(res[0]) < 4:
return None
else:
exp_index = int(res[0][0])
exp_name = res[0][1]
exp_step = int(res[0][2])
data_keyword = res[0][3]
return EvaluationDirectory(exp_index, exp_name, exp_step, data_keyword)
def __init__(self, expert_group: EvaluationTrajectoryGroup,
model_group: EvaluationTrajectoryGroup, overwrite: bool):
assert len(expert_group.trajectory_indices) == len(
expert_group.trajectories)
assert len(model_group.trajectory_indices) == len(
model_group.trajectories)
EvaluationInfoBase.__init__(self, model_group.info)
self.eval_dir = EvaluationDirectory(self.exp_index, self.exp_name,
self.exp_step, self.data_keyword)
if self.has_summary:
if overwrite:
self.summary_path.unlink()
else:
logger.info('skipped the summarization: {}'.format(
self.summary_path))
return
original_indices = expert_group.trajectory_indices
common_indices = sorted(
list(
set(expert_group.trajectory_indices).intersection(
set(model_group.trajectory_indices))))
if len(common_indices) < len(original_indices):
logger.info(
'not all of the evaluation trajectories were evaluated at {}'.
format(self.info))
if not common_indices:
logger.error(
'no valid trajectories were found in the model trajectory {}'.
format(model_group.info))
return
self.indices = common_indices
for index in self.indices:
if index >= len(expert_group.trajectories) or index >= len(
model_group.trajectories):
logger.error('invalid trajectory {}, {}'.format(
len(expert_group.trajectories),
len(model_group.trajectories)))
return
self.expert_trajectories = [
expert_group.trajectories[i] for i in self.indices
]
self.model_trajectories = [
model_group.trajectories[i] for i in self.indices
]
self.metrics = [
TrajectoryComparator(e, m)
for e, m in zip(self.expert_trajectories, self.model_trajectories)
]
self.endpoint_dist_avg = sum(
map(attrgetter('endpoint_dist'), self.metrics)) / len(self)
self.endpoint_success_rate = sum(
map(attrgetter('endpoint_success'), self.metrics)) / len(self)
self.trajectory_dist_avg_avg = sum(
map(attrgetter('trajectory_dist_avg'), self.metrics)) / len(self)
self.trajectory_dist_rate_avg = sum(
map(attrgetter('trajectory_rate'), self.metrics)) / len(self)
self.trajectory_success_rate = sum(
map(attrgetter('trajectory_success'), self.metrics)) / len(self)
self.collision_success_rate = sum(
map(attrgetter('collision_success'), self.metrics)) / len(self)
self.all_success_rate = sum(
map(attrgetter('all_success'), self.metrics)) / len(self)
logger.info(self)
# self.send_files()
self.write_summary()
def __init__(self, args, model_type: str):
GameEnvironment.__init__(self, args=args, agent_type='evaluation')
self.eval_param, self.evaluator = load_param_and_evaluator(
args=args, model_type=model_type)
self.eval_transforms = self.world.get_map().get_spawn_points()
EvaluationDirectory.__init__(self, *self.eval_info)