if not args.all:
sys_nlu, sys_dst, sys_policy, sys_nlg = set_system(
args.sys_policy, args.sys_path)
sys_agent = PipelineAgent(sys_nlu,
sys_dst,
sys_policy,
sys_nlg,
name='sys')
analyzer.comprehensive_analyze_2(sys_agent=sys_agent,
model_name='sys_agent',
total_dialog=100)
#analyzer.compare_models([sys_agent, ], ['ppo_attraction_only', ], total_dialog=args.num)
# if sys_nlu!=None, set use_nlu=True to collect more information
else:
agent_list = []
agent_name = []
for i in range(99, 100, 5):
sys_nlu, sys_dst, sys_policy, sys_nlg = set_system(
args.sys_policy, args.sys_path + str(i))
sys_agent = PipelineAgent(sys_nlu,
sys_dst,
sys_policy,
sys_nlg,
name='sys')
agent_list.append(sys_agent)
agent_name.append(str(i))
analyzer.compare_models(agent_list, agent_name, total_dialog=args.num)
def build_sys_agent_svmnlu():
sys_nlu = SVMNLU()
sys_dst = RuleDST()
sys_policy = RulePolicy(character='sys')
sys_nlg = TemplateNLG(is_user=False)
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')
return sys_agent
if __name__ == "__main__":
# user agent for simulator
user_agent = build_user_agent_bertnlu()
# build your own sys agent, modify the func to change the settings
sys_agent_svm = build_sys_agent_svmnlu()
sys_agent_bert = build_sys_agent_bertnlu()
# build analyzer, temporarily only for multiwoz
analyzer = Analyzer(user_agent=user_agent, dataset='multiwoz')
#sample dialog
analyzer.sample_dialog(sys_agent_bert)
#analyze and generate test report
analyzer.comprehensive_analyze(sys_agent=sys_agent_svm, model_name='svmnlu', total_dialog=10)
# analyzer.comprehensive_analyze(sys_agent=sys_agent_bert, model_name='bertnlu', total_dialog=100)
#compare multiple model
analyzer.compare_models(agent_list=[sys_agent_svm, sys_agent_bert], model_name=['svmnlu', 'bertnlu'], total_dialog=10)