def result_output(self): if os.path.isfile(self.result_file): os.remove(self.result_file) self.origin_df.set_index(["agent"], inplace=True) write_csv_file(self.result_file, "ranking without diversity:") self.rank_result_to_csv(self.weight_ritio, self.val_list[:-1]) write_csv_file(self.result_file, "ranking with diversity:") self.rank_result_to_csv(self.weight_ritios, self.val_list)
def run_evaluation(self): result_file = os.path.join(self.csv_file_result_path, "diversity_evaluation_result.csv") for actor_name, agent_list in self.agent_groups.items(): result_dict = self.diversity_data_analyze(agent_list) # pytype: disable=attribute-error scenario_name_list = list(result_dict.keys()) df = pd.DataFrame({ "scenario": scenario_name_list, }) df.set_index(["scenario"], inplace=True) for agent in agent_list: for scenario_name, result_data in result_dict.items(): # pytype: enable=attribute-error df.loc[scenario_name, "%s-time_score" % agent] = result_data[agent][0] df.loc[scenario_name, "%s-dis_score" % agent] = result_data[agent][1] df.loc[""] = "" write_csv_file(result_file, actor_name) df.to_csv(result_file, mode="a") for actor_name, agent_list in self.agent_groups.items(): write_csv_file(result_file, actor_name) for agent in agent_list: write_csv_file(result_file, agent)
def rank_result_to_csv(self, weight_ritio, val_list): agents_score = {} for agent in self.agents_list: agent_score = 0 for value in list(weight_ritio.keys()): agent_score += (float( (self.origin_df.loc[agent, value])[:-1]) * weight_ritio[value] * 0.01) agents_score[agent] = agent_score agents_score = sorted(agents_score.items(), key=lambda x: x[1], reverse=True) weight_score_ranking, agent_ranking = [], [] for i in agents_score: agent_ranking.append(i[0]) weight_score_ranking.append(i[1]) blank_list = [""] * len(self.agents_list) weight_score_ranking = [ str(round(i * 100, 2)) + "%" for i in weight_score_ranking ] if "diversity" in val_list: rank_df = pd.DataFrame({ "ranking": range(1, len(self.agents_list) + 1), "group": blank_list, "agent": agent_ranking, "summary_result": weight_score_ranking, "collision": blank_list, "offroad": blank_list, "kinematics": blank_list, "diversity": blank_list, }) else: rank_df = pd.DataFrame({ "ranking": range(1, len(self.agents_list) + 1), "group": blank_list, "agent": agent_ranking, "summary_result": weight_score_ranking, "collision": blank_list, "offroad": blank_list, "kinematics": blank_list, }) rank_df.set_index(["agent"], inplace=True) for value in val_list: for agent in agent_ranking: rank_df.loc[agent, value] = self.origin_df.loc[agent, value] for index, agent in enumerate(agent_ranking): group_name = self.group_list[self.agents_list.index(agent)] rank_df.loc[agent, "group"] = group_name if (index and rank_df.loc[agent, "summary_result"] == rank_df.loc[agent_ranking[index - 1], "summary_result"]): rank_df.loc[agent, "ranking"] = rank_df.loc[agent_ranking[index - 1], "ranking"] rank_df.loc[""] = "" rank_df.to_csv(self.result_file, mode="a") write_csv_file(self.result_file, "weight ritio:" + str(weight_ritio).replace(",", "")) write_csv_file(self.result_file, "")
def result_output(self): result_file = os.path.join(self.csv_file_result_path, "report.csv") if os.path.isfile(result_file): os.remove(result_file) empty_list = [""] * len(self.scenarios_list) val_list = ["collision", "offroad", "kinematics", "diversity"] data_dict = {} for agent in self.agents_list: temp_dict = {agent: {}} for val in val_list: temp_dict[agent].update({val: 0}) data_dict.update(temp_dict) diversity_result = self.read_diversity_result() for agent in self.agents_list: df = pd.DataFrame( { "scenario": self.scenarios_list, "collision": empty_list, "offroad": empty_list, "kinematics": empty_list, "diversity": empty_list, } ) df.set_index(["scenario"], inplace=True) current_group = self.group_list[self.agents_list.index(agent)] df.loc[:, "diversity"] = diversity_result[current_group] for scenario in self.scenarios_list: scenario_path = os.path.join(self.csv_file_result_path, scenario) scenario_json_file = os.path.join( scenario_path, "evaluation_results.json" ) with open(scenario_json_file, "r") as f: json_result = json.load(f) pass_agents = json_result["pass"] fail_agents = list(json_result["fail"].keys()) if agent in pass_agents: df.loc[scenario, val_list[:-1]] = "pass" elif agent in fail_agents: for value in val_list[:-1]: if value in list((json_result["fail"][agent]).keys()): df.loc[scenario, value] = "fail" else: df.loc[scenario, value] = "pass" df.loc[""] = "" group_agent_name = self.group_agents_list[self.agents_list.index(agent)] write_csv_file(result_file, group_agent_name) df.to_csv(result_file, mode="a") for scenario in self.scenarios_list: for val in val_list: if df.loc[scenario, val] == "pass": data_dict[agent][val] += 1 for agent in self.agents_list: for val in val_list: data_dict[agent][val] /= len(self.scenarios_list) data_dict[agent][val] = str(round(data_dict[agent][val] * 100, 2)) + "%" blank_list = [""] * len(self.agents_list) df_result = pd.DataFrame( { "agent": self.agents_list, "collision": blank_list, "offroad": blank_list, "kinematics": blank_list, "diversity": blank_list, } ) df_result.set_index(["agent"], inplace=True) for agent in self.agents_list: for val in val_list: df_result.loc[agent, val] = data_dict[agent][val] df_result.insert(0, "group", self.group_list) df_temp = pd.read_csv(result_file) df_result.to_csv(result_file) write_csv_file(result_file, "") write_csv_file( result_file, "The result represents the scenario pass rate of each agent under different evaluation items", ) write_csv_file(result_file, "") df_temp.to_csv(result_file, mode="a", index_label=False)
def run_evaluation(self): result_file = os.path.join(self.csv_file_result_path, "diversity_evaluation_result.csv") for actor_name, agent_list in self.agent_groups.items(): result_dict = self.diversity_result(agent_list) scenario_name_list = list(result_dict.keys()) empty_list = [""] * len(scenario_name_list) df = pd.DataFrame({ "scenario": scenario_name_list, "agent_score": empty_list, "agent_result": empty_list, }) df.set_index(["scenario"], inplace=True) for scenario_name, result_data in result_dict.items(): df.loc[scenario_name, "agent_score"] = result_data[0] if result_data[0] >= 0.8: df.loc[scenario_name, "agent_result"] = "pass" else: df.loc[scenario_name, "agent_result"] = "fail" df.loc[""] = "" write_csv_file(result_file, actor_name) df.to_csv(result_file, mode="a") self.criterion_curve_plan_plot(actor_name, agent_list) for actor_name, agent_list in self.agent_groups.items(): write_csv_file(result_file, actor_name) for agent in agent_list: write_csv_file(result_file, agent) write_csv_file(result_file, "") write_csv_file( result_file, "Agent above 0.8 is good, while agent below 0.8 is bad") write_csv_file(result_file, "NPC below 0.1 is good, while NPC above 0.1 is bad")