def test_end2end(): # go to README.md of each model for more information # BERT nlu sys_nlu = None # BERTNLU() # simple rule DST sys_dst = RuleDST() # rule policy sys_policy = PPOPolicy() # template NLG sys_nlg = None #TemplateNLG(is_user=False) # assemble sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, name='sys') # BERT nlu trained on sys utterance user_nlu = None # BERTNLU(mode='sys', config_file='multiwoz_sys_context.json', # model_file='https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_sys_context.zip') # not use dst user_dst = None #RuleDST() # None # rule policy user_policy = RulePolicy(character='usr') # template NLG user_nlg = None # TemplateNLG(is_user=True) # assemble user_agent = PipelineAgent(user_nlu, user_dst, user_policy, user_nlg, name='user') analyzer = Analyzer(user_agent=user_agent, dataset='multiwoz') set_seed(20200202) analyzer.comprehensive_analyze(sys_agent=sys_agent, model_name='BERTNLU-RuleDST-PPOPolicy-TemplateNLG', total_dialog=1000)
def build_sys_agent_bertnlu_context(use_nlu=True): sys_nlu = BERTNLU(mode='all', config_file='multiwoz_all_context.json', model_file='https://tatk-data.s3-ap-northeast-1.amazonaws.com/bert_multiwoz_all_context.zip') sys_dst = RuleDST() sys_policy = RulePolicy(character='sys') sys_nlg = TemplateNLG(is_user=False) if use_nlu: sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys') else: sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, 'sys') return sys_agent
def build_sys_agent_svmnlu(use_nlu=True): sys_nlu = SVMNLU(mode='all') sys_dst = RuleDST() sys_policy = RulePolicy(character='sys') sys_nlg = TemplateNLG(is_user=False) if use_nlu: sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys') else: sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, 'sys') return sys_agent
def test_end2end(): sys_dst = RuleDST() sys_policy = DQNPolicy() sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, name='sys') user_policy = RulePolicy(character='usr') user_agent = PipelineAgent(None, None, user_policy, None, name='user') analyzer = Analyzer(user_agent=user_agent, dataset='multiwoz') set_seed(20200202) analyzer.comprehensive_analyze(sys_agent=sys_agent, model_name='RuleDST-DQNPolicy', total_dialog=1000)
def build_user_agent_svmnlu(use_nlu=True): user_nlu = SVMNLU(mode='all') user_dst = None user_policy = RulePolicy(character='usr') user_nlg = TemplateNLG(is_user=True) if use_nlu: user_agent = PipelineAgent(user_nlu, None, user_policy, user_nlg, 'user') else: user_agent = PipelineAgent(None, None, user_policy, None, 'user') return user_agent
def build_user_agent_bertnlu(use_nlu=True): user_nlu = BERTNLU(mode='all', config_file='multiwoz_all.json', model_file='https://tatk-data.s3-ap-northeast-1.amazonaws.com/bert_multiwoz_all.zip') user_dst = None user_policy = RulePolicy(character='usr') user_nlg = TemplateNLG(is_user=True) if use_nlu: user_agent = PipelineAgent(user_nlu, None, user_policy, user_nlg, 'user') else: user_agent = PipelineAgent(None, None, user_policy, None, 'user') return user_agent
def build_user_agent_bertnlu(): user_nlu = BERTNLU() user_dst = None user_policy = RulePolicy(character='usr') user_nlg = TemplateNLG(is_user=True) user_agent = PipelineAgent(user_nlu, user_dst, user_policy, user_nlg, 'user') return user_agent
def build_sys_agent_svmnlu(): sys_nlu = SVMNLU() sys_dst = RuleDST() sys_policy = RulePolicy(character='sys') sys_nlg = TemplateNLG(is_user=False) sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys') return sys_agent
def __init__(self, dataset='multiwoz-test'): user_nlu = BERTNLU( mode='sys', config_file='multiwoz_sys_context.json', model_file= 'https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_sys_context.zip' ) user_dst = None user_policy = RulePolicy(character='usr') user_nlg = TemplateNLG(is_user=True) user_agent = PipelineAgent(user_nlu, user_dst, user_policy, user_nlg, name='user') dataset, _ = data.split_name(dataset) super().__init__(user_agent, dataset)
def end2end_evaluate_simulation(policy): nlu = BERTNLU('all', 'crosswoz_all_context.json', None) nlg_usr = TemplateNLG(is_user=True, mode='auto_manual') nlg_sys = TemplateNLG(is_user=False, mode='auto_manual') # nlg_usr = SCLSTM(is_user=True, use_cuda=False) # nlg_sys = SCLSTM(is_user=False, use_cuda=False) usr_policy = Simulator() usr_agent = PipelineAgent(nlu, None, usr_policy, nlg_usr, name='usr') sys_policy = policy sys_dst = RuleDST() sys_agent = PipelineAgent(nlu, sys_dst, sys_policy, nlg_sys, name='sys') sess = BiSession(sys_agent=sys_agent, user_agent=usr_agent) task_success = {'All': list(), '单领域': list(), '独立多领域': list(), '独立多领域+交通': list(), '不独立多领域': list(), '不独立多领域+交通': list()} simulate_sess_num = 100 repeat = 10 random_seed = 2019 random.seed(random_seed) np.random.seed(random_seed) torch.manual_seed(random_seed) random_seeds = [random.randint(1, 2**32-1) for _ in range(simulate_sess_num * repeat * 10000)] while True: sys_response = '' random_seed = random_seeds[0] random.seed(random_seed) np.random.seed(random_seed) torch.manual_seed(random_seed) random_seeds.pop(0) sess.init_session() # print(usr_policy.goal_type) if len(task_success[usr_policy.goal_type]) == simulate_sess_num*repeat: continue for i in range(15): sys_response, user_response, session_over, reward = sess.next_turn(sys_response) # print('user:'******'sys:', sys_response) # print(session_over, reward) # print() if session_over is True: task_success['All'].append(1) task_success[usr_policy.goal_type].append(1) break else: task_success['All'].append(0) task_success[usr_policy.goal_type].append(0) print([len(x) for x in task_success.values()]) # print(min([len(x) for x in task_success.values()])) if len(task_success['All']) % 100 == 0: for k, v in task_success.items(): print(k) all_samples = [] for i in range(repeat): samples = v[i * simulate_sess_num:(i + 1) * simulate_sess_num] all_samples += samples print(sum(samples), len(samples), (sum(samples) / len(samples)) if len(samples) else 0) print('avg', (sum(all_samples) / len(all_samples)) if len(all_samples) else 0) if min([len(x) for x in task_success.values()]) == simulate_sess_num*repeat: break # pprint(usr_policy.original_goal) # pprint(task_success) print('task_success') for k, v in task_success.items(): print(k) all_samples = [] for i in range(repeat): samples = v[i * simulate_sess_num:(i + 1) * simulate_sess_num] all_samples += samples print(sum(samples), len(samples), (sum(samples) / len(samples)) if len(samples) else 0) print('avg', (sum(all_samples) / len(all_samples)) if len(all_samples) else 0)
def da_evaluate_simulation(policy): usr_policy = Simulator() usr_agent = PipelineAgent(None, None, usr_policy, None, name='usr') sys_policy = policy sys_dst = RuleDST() sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, name='sys') sess = BiSession(sys_agent=sys_agent, user_agent=usr_agent) task_success = {'All': list(), '单领域': list(), '独立多领域': list(), '独立多领域+交通': list(), '不独立多领域': list(), '不独立多领域+交通': list()} simulate_sess_num = 100 repeat = 10 random_seed = 2019 random.seed(random_seed) np.random.seed(random_seed) torch.manual_seed(random_seed) random_seeds = [random.randint(1, 2**32-1) for _ in range(simulate_sess_num * repeat * 10000)] while True: sys_response = [] random_seed = random_seeds[0] random.seed(random_seed) np.random.seed(random_seed) torch.manual_seed(random_seed) random_seeds.pop(0) sess.init_session() # print(usr_policy.goal_type) if len(task_success[usr_policy.goal_type]) == simulate_sess_num*repeat: continue for i in range(15): sys_response, user_response, session_over, reward = sess.next_turn(sys_response) # print('user:'******'sys:', sys_response) # print(session_over, reward) # print() if session_over is True: # pprint(sys_agent.tracker.state) task_success['All'].append(1) task_success[usr_policy.goal_type].append(1) break else: task_success['All'].append(0) task_success[usr_policy.goal_type].append(0) print([len(x) for x in task_success.values()]) # print(min([len(x) for x in task_success.values()])) if len(task_success['All']) % 100 == 0: for k, v in task_success.items(): print(k) all_samples = [] for i in range(repeat): samples = v[i * simulate_sess_num:(i + 1) * simulate_sess_num] all_samples += samples print(sum(samples), len(samples), (sum(samples) / len(samples)) if len(samples) else 0) print('avg', (sum(all_samples) / len(all_samples)) if len(all_samples) else 0) if min([len(x) for x in task_success.values()]) == simulate_sess_num*repeat: break # pprint(usr_policy.original_goal) # pprint(task_success) print('task_success') for k, v in task_success.items(): print(k) all_samples = [] for i in range(repeat): samples = v[i * simulate_sess_num:(i + 1) * simulate_sess_num] all_samples += samples print(sum(samples), len(samples), (sum(samples) / len(samples)) if len(samples) else 0) print('avg', (sum(all_samples) / len(all_samples)) if len(all_samples) else 0)
from convlab2.nlg.template.multiwoz.nlg import TemplateNLG from convlab2.dst.rule.multiwoz.dst import RuleDST from convlab2.nlu.jointBERT.multiwoz.nlu import BERTNLU seed = 50 np.random.seed(seed) random.seed(seed) torch.manual_seed(seed) sys_nlu = BERTNLU() sys_dst = RuleDST() sys_policy = RulePolicy() sys_nlg = TemplateNLG(is_user=False) sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, name='sys') user_nlu = BERTNLU( mode='sys', config_file='multiwoz_sys_context.json', model_file= 'https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_sys_context.zip' ) user_dst = None user_policy = RulePolicy(character='usr') user_nlg = TemplateNLG(is_user=True) user_agent = PipelineAgent(user_nlu, user_dst, user_policy,
import random import numpy as np from pprint import pprint rgi_queue = PriorityQueue(maxsize=0) rgo_queue = PriorityQueue(maxsize=0) app = Flask(__name__) # sys_nlu = BERTNLU() sys_nlu = MILU() sys_dst = RuleDST() sys_policy = RulePolicy(character='sys') sys_nlg = TemplateNLG(is_user=False) agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys') print(agent.response('I am looking for a hotel')) @app.route('/', methods=['GET', 'POST']) def process(): try: in_request = request.json print(in_request) except: return "invalid input: {}".format(in_request) rgi_queue.put(in_request) rgi_queue.join() output = rgo_queue.get() print(output['response'])
# BERT nlu sys_nlu = BERTNLU() # simple rule DST sys_dst = RuleDST() # rule policy sys_policy = RulePolicy() from convlab2.policy.mgail.multiwoz import MGAIL policy_sys = MGAIL() policy_sys.load( '/home/nightop/ConvLab-2/convlab2/policy/mgail/multiwoz/save/all/99') # template NLG sys_nlg = TemplateNLG(is_user=False) # assemble sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, name='sys') # MILU user_nlu = BERTNLU() # not use dst user_dst = None # rule policy user_policy = RulePolicy(character='usr') #user_policy.policy.goal_generator = GoalGenerator_7() #user_policy.policy.goal_generator = GoalGenerator_restaurant() # template NLG user_nlg = TemplateNLG(is_user=True) # assemble user_agent = PipelineAgent(user_nlu,
plt.savefig('results/compare_results.jpg') plt.close() if __name__ == "__main__": sys_nlu = SVMNLU(mode='all') sys_dst = RuleDST() # rule policy sys_policy = RulePolicy(character='sys') # template NLG sys_nlg = TemplateNLG(is_user=False) # assemble if self.use_nlu: sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys') else: sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, 'sys') user_nlu = SVMNLU(mode='all') # not use dst user_dst = None # rule policy user_policy = RulePolicy(character='usr') # template NLG user_nlg = TemplateNLG(is_user=True) # assemble if self.use_nlu: user_agent = PipelineAgent(user_nlu, None, user_policy, user_nlg, 'user')