def __init__(self, eval_keyword: str, args): self.eval_keyword = eval_keyword GameEnvironment.__init__(self, args=args, agent_type='evaluation') self.eval_name = args.eval_name # load params and evaluators self.control_param, self.control_evaluator = \ load_param_and_evaluator(eval_keyword=eval_keyword, args=args, model_type='control') self.stop_param, self.stop_evaluator = \ load_param_and_evaluator(eval_keyword=eval_keyword, args=args, model_type='stop') self.high_level_param, self.high_level_evaluator = \ load_param_and_evaluator(eval_keyword=eval_keyword, args=args, model_type='high') # set image type self.image_type = self.high_level_param.image_type if 'd' in self.image_type: from model import DeepLabModel, prepare_deeplab_model self.deeplab_model: DeepLabModel = prepare_deeplab_model() self.final_images = [] self.eval_dataset, self.eval_sentences = load_evaluation_dataset( self.high_level_param) self.eval_transforms = list( map(lambda x: x[0].state.transform, self.eval_dataset)) self.high_level_sentences = self.eval_sentences logger.info('fetched {} sentences from {}'.format( len(self.high_level_sentences), self.high_level_param.eval_keyword.lower())) self.softmax = torch.nn.Softmax(dim=1) EvaluationDirectory.__init__(self, *self.eval_info) self.high_level_data_dict = dict()
def load_model_single_trajectory(eval_dir: EvaluationDirectory, traj_index: int) -> EvaluationTrajectory: try: with open(str(eval_dir.state_path(traj_index)), 'r') as file: data = json.load(file) except: raise FileNotFoundError('failed to load {}'.format( eval_dir.state_path(traj_index))) collided = False if 'collided' not in data else parse_bool( data['collided']) data_frames = [ DriveDataFrame.load_from_str(f) for f in data['data_frames'] ] info = EvaluationUnitInfo(eval_dir.data_keyword, eval_dir.exp_index, eval_dir.exp_name, eval_dir.exp_step, traj_index) return EvaluationTrajectory(info, data_frames, collided)
def load_model_single_trajectory_group( eval_dir: EvaluationDirectory) -> EvaluationTrajectoryGroup: group_info = EvaluationUnitInfo(eval_dir.data_keyword, eval_dir.exp_index, eval_dir.exp_name, eval_dir.exp_step, -1) trajectories = [] for traj_index in eval_dir.traj_indices_from_state_dir(): trajectories.append(load_model_single_trajectory(eval_dir, traj_index)) return EvaluationTrajectoryGroup(group_info, trajectories)
def evaluation_directory_from_path(eval_path: Path) -> EvaluationDirectory: pattern = 'exp([\d]+)\/([\w-]+)\/step([\d]+)/([\w,]+)' res = re.findall(pattern, str(eval_path)) if not res or len(res[0]) < 4: return None else: exp_index = int(res[0][0]) exp_name = res[0][1] exp_step = int(res[0][2]) data_keyword = res[0][3] return EvaluationDirectory(exp_index, exp_name, exp_step, data_keyword)
def __init__(self, expert_group: EvaluationTrajectoryGroup, model_group: EvaluationTrajectoryGroup, overwrite: bool): assert len(expert_group.trajectory_indices) == len( expert_group.trajectories) assert len(model_group.trajectory_indices) == len( model_group.trajectories) EvaluationInfoBase.__init__(self, model_group.info) self.eval_dir = EvaluationDirectory(self.exp_index, self.exp_name, self.exp_step, self.data_keyword) if self.has_summary: if overwrite: self.summary_path.unlink() else: logger.info('skipped the summarization: {}'.format( self.summary_path)) return original_indices = expert_group.trajectory_indices common_indices = sorted( list( set(expert_group.trajectory_indices).intersection( set(model_group.trajectory_indices)))) if len(common_indices) < len(original_indices): logger.info( 'not all of the evaluation trajectories were evaluated at {}'. format(self.info)) if not common_indices: logger.error( 'no valid trajectories were found in the model trajectory {}'. format(model_group.info)) return self.indices = common_indices for index in self.indices: if index >= len(expert_group.trajectories) or index >= len( model_group.trajectories): logger.error('invalid trajectory {}, {}'.format( len(expert_group.trajectories), len(model_group.trajectories))) return self.expert_trajectories = [ expert_group.trajectories[i] for i in self.indices ] self.model_trajectories = [ model_group.trajectories[i] for i in self.indices ] self.metrics = [ TrajectoryComparator(e, m) for e, m in zip(self.expert_trajectories, self.model_trajectories) ] self.endpoint_dist_avg = sum( map(attrgetter('endpoint_dist'), self.metrics)) / len(self) self.endpoint_success_rate = sum( map(attrgetter('endpoint_success'), self.metrics)) / len(self) self.trajectory_dist_avg_avg = sum( map(attrgetter('trajectory_dist_avg'), self.metrics)) / len(self) self.trajectory_dist_rate_avg = sum( map(attrgetter('trajectory_rate'), self.metrics)) / len(self) self.trajectory_success_rate = sum( map(attrgetter('trajectory_success'), self.metrics)) / len(self) self.collision_success_rate = sum( map(attrgetter('collision_success'), self.metrics)) / len(self) self.all_success_rate = sum( map(attrgetter('all_success'), self.metrics)) / len(self) logger.info(self) # self.send_files() self.write_summary()
def __init__(self, args, model_type: str): GameEnvironment.__init__(self, args=args, agent_type='evaluation') self.eval_param, self.evaluator = load_param_and_evaluator( args=args, model_type=model_type) self.eval_transforms = self.world.get_map().get_spawn_points() EvaluationDirectory.__init__(self, *self.eval_info)