if not args.all: sys_nlu, sys_dst, sys_policy, sys_nlg = set_system( args.sys_policy, args.sys_path) sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, name='sys') analyzer.comprehensive_analyze_2(sys_agent=sys_agent, model_name='sys_agent', total_dialog=100) #analyzer.compare_models([sys_agent, ], ['ppo_attraction_only', ], total_dialog=args.num) # if sys_nlu!=None, set use_nlu=True to collect more information else: agent_list = [] agent_name = [] for i in range(99, 100, 5): sys_nlu, sys_dst, sys_policy, sys_nlg = set_system( args.sys_policy, args.sys_path + str(i)) sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, name='sys') agent_list.append(sys_agent) agent_name.append(str(i)) analyzer.compare_models(agent_list, agent_name, total_dialog=args.num)
def build_sys_agent_svmnlu(): sys_nlu = SVMNLU() sys_dst = RuleDST() sys_policy = RulePolicy(character='sys') sys_nlg = TemplateNLG(is_user=False) sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys') return sys_agent if __name__ == "__main__": # user agent for simulator user_agent = build_user_agent_bertnlu() # build your own sys agent, modify the func to change the settings sys_agent_svm = build_sys_agent_svmnlu() sys_agent_bert = build_sys_agent_bertnlu() # build analyzer, temporarily only for multiwoz analyzer = Analyzer(user_agent=user_agent, dataset='multiwoz') #sample dialog analyzer.sample_dialog(sys_agent_bert) #analyze and generate test report analyzer.comprehensive_analyze(sys_agent=sys_agent_svm, model_name='svmnlu', total_dialog=10) # analyzer.comprehensive_analyze(sys_agent=sys_agent_bert, model_name='bertnlu', total_dialog=100) #compare multiple model analyzer.compare_models(agent_list=[sys_agent_svm, sys_agent_bert], model_name=['svmnlu', 'bertnlu'], total_dialog=10)