def test_end2end(): # go to README.md of each model for more information # BERT nlu sys_nlu = None # BERTNLU() # simple rule DST sys_dst = RuleDST() # rule policy sys_policy = PPOPolicy() # template NLG sys_nlg = None #TemplateNLG(is_user=False) # assemble sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, name='sys') # BERT nlu trained on sys utterance user_nlu = None # BERTNLU(mode='sys', config_file='multiwoz_sys_context.json', # model_file='https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_sys_context.zip') # not use dst user_dst = None #RuleDST() # None # rule policy user_policy = RulePolicy(character='usr') # template NLG user_nlg = None # TemplateNLG(is_user=True) # assemble user_agent = PipelineAgent(user_nlu, user_dst, user_policy, user_nlg, name='user') analyzer = Analyzer(user_agent=user_agent, dataset='multiwoz') set_seed(20200202) analyzer.comprehensive_analyze(sys_agent=sys_agent, model_name='BERTNLU-RuleDST-PPOPolicy-TemplateNLG', total_dialog=1000)
def test_end2end(): sys_dst = RuleDST() sys_policy = DQNPolicy() sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, name='sys') user_policy = RulePolicy(character='usr') user_agent = PipelineAgent(None, None, user_policy, None, name='user') analyzer = Analyzer(user_agent=user_agent, dataset='multiwoz') set_seed(20200202) analyzer.comprehensive_analyze(sys_agent=sys_agent, model_name='RuleDST-DQNPolicy', total_dialog=1000)
sys_response, user_response, session_over, reward = sess.next_turn( sys_response) print('user:'******'sys:', sys_response) print() if session_over is True: break print('task success:', sess.evaluator.task_success()) print('book rate:', sess.evaluator.book_rate()) print('inform precision/recall/f1:', sess.evaluator.inform_F1()) print('-' * 50) print('final goal:') pprint(sess.evaluator.goal) print('=' * 100) analyzer = Analyzer(user_agent=user_agent, dataset='multiwoz') if not args.all: sys_nlu, sys_dst, sys_policy, sys_nlg = set_system( args.sys_policy, args.sys_path) sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, name='sys') analyzer.comprehensive_analyze_2(sys_agent=sys_agent, model_name='sys_agent', total_dialog=100) #analyzer.compare_models([sys_agent, ], ['ppo_attraction_only', ], total_dialog=args.num) # if sys_nlu!=None, set use_nlu=True to collect more information
def build_sys_agent_svmnlu(): sys_nlu = SVMNLU() sys_dst = RuleDST() sys_policy = RulePolicy(character='sys') sys_nlg = TemplateNLG(is_user=False) sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys') return sys_agent if __name__ == "__main__": # user agent for simulator user_agent = build_user_agent_bertnlu() # build your own sys agent, modify the func to change the settings sys_agent_svm = build_sys_agent_svmnlu() sys_agent_bert = build_sys_agent_bertnlu() # build analyzer, temporarily only for multiwoz analyzer = Analyzer(user_agent=user_agent, dataset='multiwoz') #sample dialog analyzer.sample_dialog(sys_agent_bert) #analyze and generate test report analyzer.comprehensive_analyze(sys_agent=sys_agent_svm, model_name='svmnlu', total_dialog=10) # analyzer.comprehensive_analyze(sys_agent=sys_agent_bert, model_name='bertnlu', total_dialog=100) #compare multiple model analyzer.compare_models(agent_list=[sys_agent_svm, sys_agent_bert], model_name=['svmnlu', 'bertnlu'], total_dialog=10)