def result_output(self):
if os.path.isfile(self.result_file):
os.remove(self.result_file)
self.origin_df.set_index(["agent"], inplace=True)
write_csv_file(self.result_file, "ranking without diversity:")
self.rank_result_to_csv(self.weight_ritio, self.val_list[:-1])
write_csv_file(self.result_file, "ranking with diversity:")
self.rank_result_to_csv(self.weight_ritios, self.val_list)
def run_evaluation(self):
result_file = os.path.join(self.csv_file_result_path,
"diversity_evaluation_result.csv")
for actor_name, agent_list in self.agent_groups.items():
result_dict = self.diversity_data_analyze(agent_list)
# pytype: disable=attribute-error
scenario_name_list = list(result_dict.keys())
df = pd.DataFrame({
"scenario": scenario_name_list,
})
df.set_index(["scenario"], inplace=True)
for agent in agent_list:
for scenario_name, result_data in result_dict.items():
# pytype: enable=attribute-error
df.loc[scenario_name,
"%s-time_score" % agent] = result_data[agent][0]
df.loc[scenario_name,
"%s-dis_score" % agent] = result_data[agent][1]
df.loc[""] = ""
write_csv_file(result_file, actor_name)
df.to_csv(result_file, mode="a")
for actor_name, agent_list in self.agent_groups.items():
write_csv_file(result_file, actor_name)
for agent in agent_list:
write_csv_file(result_file, agent)
def rank_result_to_csv(self, weight_ritio, val_list):
agents_score = {}
for agent in self.agents_list:
agent_score = 0
for value in list(weight_ritio.keys()):
agent_score += (float(
(self.origin_df.loc[agent, value])[:-1]) *
weight_ritio[value] * 0.01)
agents_score[agent] = agent_score
agents_score = sorted(agents_score.items(),
key=lambda x: x[1],
reverse=True)
weight_score_ranking, agent_ranking = [], []
for i in agents_score:
agent_ranking.append(i[0])
weight_score_ranking.append(i[1])
blank_list = [""] * len(self.agents_list)
weight_score_ranking = [
str(round(i * 100, 2)) + "%" for i in weight_score_ranking
]
if "diversity" in val_list:
rank_df = pd.DataFrame({
"ranking":
range(1,
len(self.agents_list) + 1),
"group":
blank_list,
"agent":
agent_ranking,
"summary_result":
weight_score_ranking,
"collision":
blank_list,
"offroad":
blank_list,
"kinematics":
blank_list,
"diversity":
blank_list,
})
else:
rank_df = pd.DataFrame({
"ranking":
range(1,
len(self.agents_list) + 1),
"group":
blank_list,
"agent":
agent_ranking,
"summary_result":
weight_score_ranking,
"collision":
blank_list,
"offroad":
blank_list,
"kinematics":
blank_list,
})
rank_df.set_index(["agent"], inplace=True)
for value in val_list:
for agent in agent_ranking:
rank_df.loc[agent, value] = self.origin_df.loc[agent, value]
for index, agent in enumerate(agent_ranking):
group_name = self.group_list[self.agents_list.index(agent)]
rank_df.loc[agent, "group"] = group_name
if (index and rank_df.loc[agent, "summary_result"]
== rank_df.loc[agent_ranking[index - 1],
"summary_result"]):
rank_df.loc[agent,
"ranking"] = rank_df.loc[agent_ranking[index - 1],
"ranking"]
rank_df.loc[""] = ""
rank_df.to_csv(self.result_file, mode="a")
write_csv_file(self.result_file,
"weight ritio:" + str(weight_ritio).replace(",", ""))
write_csv_file(self.result_file, "")
def result_output(self):
result_file = os.path.join(self.csv_file_result_path, "report.csv")
if os.path.isfile(result_file):
os.remove(result_file)
empty_list = [""] * len(self.scenarios_list)
val_list = ["collision", "offroad", "kinematics", "diversity"]
data_dict = {}
for agent in self.agents_list:
temp_dict = {agent: {}}
for val in val_list:
temp_dict[agent].update({val: 0})
data_dict.update(temp_dict)
diversity_result = self.read_diversity_result()
for agent in self.agents_list:
df = pd.DataFrame(
{
"scenario": self.scenarios_list,
"collision": empty_list,
"offroad": empty_list,
"kinematics": empty_list,
"diversity": empty_list,
}
)
df.set_index(["scenario"], inplace=True)
current_group = self.group_list[self.agents_list.index(agent)]
df.loc[:, "diversity"] = diversity_result[current_group]
for scenario in self.scenarios_list:
scenario_path = os.path.join(self.csv_file_result_path, scenario)
scenario_json_file = os.path.join(
scenario_path, "evaluation_results.json"
)
with open(scenario_json_file, "r") as f:
json_result = json.load(f)
pass_agents = json_result["pass"]
fail_agents = list(json_result["fail"].keys())
if agent in pass_agents:
df.loc[scenario, val_list[:-1]] = "pass"
elif agent in fail_agents:
for value in val_list[:-1]:
if value in list((json_result["fail"][agent]).keys()):
df.loc[scenario, value] = "fail"
else:
df.loc[scenario, value] = "pass"
df.loc[""] = ""
group_agent_name = self.group_agents_list[self.agents_list.index(agent)]
write_csv_file(result_file, group_agent_name)
df.to_csv(result_file, mode="a")
for scenario in self.scenarios_list:
for val in val_list:
if df.loc[scenario, val] == "pass":
data_dict[agent][val] += 1
for agent in self.agents_list:
for val in val_list:
data_dict[agent][val] /= len(self.scenarios_list)
data_dict[agent][val] = str(round(data_dict[agent][val] * 100, 2)) + "%"
blank_list = [""] * len(self.agents_list)
df_result = pd.DataFrame(
{
"agent": self.agents_list,
"collision": blank_list,
"offroad": blank_list,
"kinematics": blank_list,
"diversity": blank_list,
}
)
df_result.set_index(["agent"], inplace=True)
for agent in self.agents_list:
for val in val_list:
df_result.loc[agent, val] = data_dict[agent][val]
df_result.insert(0, "group", self.group_list)
df_temp = pd.read_csv(result_file)
df_result.to_csv(result_file)
write_csv_file(result_file, "")
write_csv_file(
result_file,
"The result represents the scenario pass rate of each agent under different evaluation items",
)
write_csv_file(result_file, "")
df_temp.to_csv(result_file, mode="a", index_label=False)
def run_evaluation(self):
result_file = os.path.join(self.csv_file_result_path,
"diversity_evaluation_result.csv")
for actor_name, agent_list in self.agent_groups.items():
result_dict = self.diversity_result(agent_list)
scenario_name_list = list(result_dict.keys())
empty_list = [""] * len(scenario_name_list)
df = pd.DataFrame({
"scenario": scenario_name_list,
"agent_score": empty_list,
"agent_result": empty_list,
})
df.set_index(["scenario"], inplace=True)
for scenario_name, result_data in result_dict.items():
df.loc[scenario_name, "agent_score"] = result_data[0]
if result_data[0] >= 0.8:
df.loc[scenario_name, "agent_result"] = "pass"
else:
df.loc[scenario_name, "agent_result"] = "fail"
df.loc[""] = ""
write_csv_file(result_file, actor_name)
df.to_csv(result_file, mode="a")
self.criterion_curve_plan_plot(actor_name, agent_list)
for actor_name, agent_list in self.agent_groups.items():
write_csv_file(result_file, actor_name)
for agent in agent_list:
write_csv_file(result_file, agent)
write_csv_file(result_file, "")
write_csv_file(
result_file,
"Agent above 0.8 is good, while agent below 0.8 is bad")
write_csv_file(result_file,
"NPC below 0.1 is good, while NPC above 0.1 is bad")