def test_end2end():
# go to README.md of each model for more information
# BERT nlu
sys_nlu = None # BERTNLU()
# simple rule DST
sys_dst = RuleDST()
# rule policy
sys_policy = PPOPolicy()
# template NLG
sys_nlg = None #TemplateNLG(is_user=False)
# assemble
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, name='sys')
# BERT nlu trained on sys utterance
user_nlu = None # BERTNLU(mode='sys', config_file='multiwoz_sys_context.json',
# model_file='https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_sys_context.zip')
# not use dst
user_dst = None #RuleDST() # None
# rule policy
user_policy = RulePolicy(character='usr')
# template NLG
user_nlg = None # TemplateNLG(is_user=True)
# assemble
user_agent = PipelineAgent(user_nlu, user_dst, user_policy, user_nlg, name='user')
analyzer = Analyzer(user_agent=user_agent, dataset='multiwoz')
set_seed(20200202)
analyzer.comprehensive_analyze(sys_agent=sys_agent, model_name='BERTNLU-RuleDST-PPOPolicy-TemplateNLG', total_dialog=1000)
def build_sys_agent_bertnlu_context(use_nlu=True):
sys_nlu = BERTNLU(mode='all', config_file='multiwoz_all_context.json',
model_file='https://tatk-data.s3-ap-northeast-1.amazonaws.com/bert_multiwoz_all_context.zip')
sys_dst = RuleDST()
sys_policy = RulePolicy(character='sys')
sys_nlg = TemplateNLG(is_user=False)
if use_nlu:
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')
else:
sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, 'sys')
return sys_agent
def build_sys_agent_svmnlu(use_nlu=True):
sys_nlu = SVMNLU(mode='all')
sys_dst = RuleDST()
sys_policy = RulePolicy(character='sys')
sys_nlg = TemplateNLG(is_user=False)
if use_nlu:
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')
else:
sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, 'sys')
return sys_agent
def test_end2end():
sys_dst = RuleDST()
sys_policy = DQNPolicy()
sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, name='sys')
user_policy = RulePolicy(character='usr')
user_agent = PipelineAgent(None, None, user_policy, None, name='user')
analyzer = Analyzer(user_agent=user_agent, dataset='multiwoz')
set_seed(20200202)
analyzer.comprehensive_analyze(sys_agent=sys_agent,
model_name='RuleDST-DQNPolicy',
total_dialog=1000)
def build_user_agent_svmnlu(use_nlu=True):
user_nlu = SVMNLU(mode='all')
user_dst = None
user_policy = RulePolicy(character='usr')
user_nlg = TemplateNLG(is_user=True)
if use_nlu:
user_agent = PipelineAgent(user_nlu, None, user_policy, user_nlg, 'user')
else:
user_agent = PipelineAgent(None, None, user_policy, None, 'user')
return user_agent
def build_user_agent_bertnlu(use_nlu=True):
user_nlu = BERTNLU(mode='all', config_file='multiwoz_all.json',
model_file='https://tatk-data.s3-ap-northeast-1.amazonaws.com/bert_multiwoz_all.zip')
user_dst = None
user_policy = RulePolicy(character='usr')
user_nlg = TemplateNLG(is_user=True)
if use_nlu:
user_agent = PipelineAgent(user_nlu, None, user_policy, user_nlg, 'user')
else:
user_agent = PipelineAgent(None, None, user_policy, None, 'user')
return user_agent
def build_user_agent_bertnlu():
user_nlu = BERTNLU()
user_dst = None
user_policy = RulePolicy(character='usr')
user_nlg = TemplateNLG(is_user=True)
user_agent = PipelineAgent(user_nlu, user_dst, user_policy, user_nlg, 'user')
return user_agent
def build_sys_agent_svmnlu():
sys_nlu = SVMNLU()
sys_dst = RuleDST()
sys_policy = RulePolicy(character='sys')
sys_nlg = TemplateNLG(is_user=False)
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')
return sys_agent
def __init__(self, dataset='multiwoz-test'):
user_nlu = BERTNLU(
mode='sys',
config_file='multiwoz_sys_context.json',
model_file=
'https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_sys_context.zip'
)
user_dst = None
user_policy = RulePolicy(character='usr')
user_nlg = TemplateNLG(is_user=True)
user_agent = PipelineAgent(user_nlu,
user_dst,
user_policy,
user_nlg,
name='user')
dataset, _ = data.split_name(dataset)
super().__init__(user_agent, dataset)
def end2end_evaluate_simulation(policy):
nlu = BERTNLU('all', 'crosswoz_all_context.json', None)
nlg_usr = TemplateNLG(is_user=True, mode='auto_manual')
nlg_sys = TemplateNLG(is_user=False, mode='auto_manual')
# nlg_usr = SCLSTM(is_user=True, use_cuda=False)
# nlg_sys = SCLSTM(is_user=False, use_cuda=False)
usr_policy = Simulator()
usr_agent = PipelineAgent(nlu, None, usr_policy, nlg_usr, name='usr')
sys_policy = policy
sys_dst = RuleDST()
sys_agent = PipelineAgent(nlu, sys_dst, sys_policy, nlg_sys, name='sys')
sess = BiSession(sys_agent=sys_agent, user_agent=usr_agent)
task_success = {'All': list(), '单领域': list(), '独立多领域': list(), '独立多领域+交通': list(), '不独立多领域': list(),
'不独立多领域+交通': list()}
simulate_sess_num = 100
repeat = 10
random_seed = 2019
random.seed(random_seed)
np.random.seed(random_seed)
torch.manual_seed(random_seed)
random_seeds = [random.randint(1, 2**32-1) for _ in range(simulate_sess_num * repeat * 10000)]
while True:
sys_response = ''
random_seed = random_seeds[0]
random.seed(random_seed)
np.random.seed(random_seed)
torch.manual_seed(random_seed)
random_seeds.pop(0)
sess.init_session()
# print(usr_policy.goal_type)
if len(task_success[usr_policy.goal_type]) == simulate_sess_num*repeat:
continue
for i in range(15):
sys_response, user_response, session_over, reward = sess.next_turn(sys_response)
# print('user:'******'sys:', sys_response)
# print(session_over, reward)
# print()
if session_over is True:
task_success['All'].append(1)
task_success[usr_policy.goal_type].append(1)
break
else:
task_success['All'].append(0)
task_success[usr_policy.goal_type].append(0)
print([len(x) for x in task_success.values()])
# print(min([len(x) for x in task_success.values()]))
if len(task_success['All']) % 100 == 0:
for k, v in task_success.items():
print(k)
all_samples = []
for i in range(repeat):
samples = v[i * simulate_sess_num:(i + 1) * simulate_sess_num]
all_samples += samples
print(sum(samples), len(samples), (sum(samples) / len(samples)) if len(samples) else 0)
print('avg', (sum(all_samples) / len(all_samples)) if len(all_samples) else 0)
if min([len(x) for x in task_success.values()]) == simulate_sess_num*repeat:
break
# pprint(usr_policy.original_goal)
# pprint(task_success)
print('task_success')
for k, v in task_success.items():
print(k)
all_samples = []
for i in range(repeat):
samples = v[i * simulate_sess_num:(i + 1) * simulate_sess_num]
all_samples += samples
print(sum(samples), len(samples), (sum(samples) / len(samples)) if len(samples) else 0)
print('avg', (sum(all_samples) / len(all_samples)) if len(all_samples) else 0)
def da_evaluate_simulation(policy):
usr_policy = Simulator()
usr_agent = PipelineAgent(None, None, usr_policy, None, name='usr')
sys_policy = policy
sys_dst = RuleDST()
sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, name='sys')
sess = BiSession(sys_agent=sys_agent, user_agent=usr_agent)
task_success = {'All': list(), '单领域': list(), '独立多领域': list(), '独立多领域+交通': list(), '不独立多领域': list(),
'不独立多领域+交通': list()}
simulate_sess_num = 100
repeat = 10
random_seed = 2019
random.seed(random_seed)
np.random.seed(random_seed)
torch.manual_seed(random_seed)
random_seeds = [random.randint(1, 2**32-1) for _ in range(simulate_sess_num * repeat * 10000)]
while True:
sys_response = []
random_seed = random_seeds[0]
random.seed(random_seed)
np.random.seed(random_seed)
torch.manual_seed(random_seed)
random_seeds.pop(0)
sess.init_session()
# print(usr_policy.goal_type)
if len(task_success[usr_policy.goal_type]) == simulate_sess_num*repeat:
continue
for i in range(15):
sys_response, user_response, session_over, reward = sess.next_turn(sys_response)
# print('user:'******'sys:', sys_response)
# print(session_over, reward)
# print()
if session_over is True:
# pprint(sys_agent.tracker.state)
task_success['All'].append(1)
task_success[usr_policy.goal_type].append(1)
break
else:
task_success['All'].append(0)
task_success[usr_policy.goal_type].append(0)
print([len(x) for x in task_success.values()])
# print(min([len(x) for x in task_success.values()]))
if len(task_success['All']) % 100 == 0:
for k, v in task_success.items():
print(k)
all_samples = []
for i in range(repeat):
samples = v[i * simulate_sess_num:(i + 1) * simulate_sess_num]
all_samples += samples
print(sum(samples), len(samples), (sum(samples) / len(samples)) if len(samples) else 0)
print('avg', (sum(all_samples) / len(all_samples)) if len(all_samples) else 0)
if min([len(x) for x in task_success.values()]) == simulate_sess_num*repeat:
break
# pprint(usr_policy.original_goal)
# pprint(task_success)
print('task_success')
for k, v in task_success.items():
print(k)
all_samples = []
for i in range(repeat):
samples = v[i * simulate_sess_num:(i + 1) * simulate_sess_num]
all_samples += samples
print(sum(samples), len(samples), (sum(samples) / len(samples)) if len(samples) else 0)
print('avg', (sum(all_samples) / len(all_samples)) if len(all_samples) else 0)
from convlab2.nlg.template.multiwoz.nlg import TemplateNLG
from convlab2.dst.rule.multiwoz.dst import RuleDST
from convlab2.nlu.jointBERT.multiwoz.nlu import BERTNLU
seed = 50
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
sys_nlu = BERTNLU()
sys_dst = RuleDST()
sys_policy = RulePolicy()
sys_nlg = TemplateNLG(is_user=False)
sys_agent = PipelineAgent(sys_nlu,
sys_dst,
sys_policy,
sys_nlg,
name='sys')
user_nlu = BERTNLU(
mode='sys',
config_file='multiwoz_sys_context.json',
model_file=
'https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_sys_context.zip'
)
user_dst = None
user_policy = RulePolicy(character='usr')
user_nlg = TemplateNLG(is_user=True)
user_agent = PipelineAgent(user_nlu,
user_dst,
user_policy,
import random
import numpy as np
from pprint import pprint
rgi_queue = PriorityQueue(maxsize=0)
rgo_queue = PriorityQueue(maxsize=0)
app = Flask(__name__)
# sys_nlu = BERTNLU()
sys_nlu = MILU()
sys_dst = RuleDST()
sys_policy = RulePolicy(character='sys')
sys_nlg = TemplateNLG(is_user=False)
agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')
print(agent.response('I am looking for a hotel'))
@app.route('/', methods=['GET', 'POST'])
def process():
try:
in_request = request.json
print(in_request)
except:
return "invalid input: {}".format(in_request)
rgi_queue.put(in_request)
rgi_queue.join()
output = rgo_queue.get()
print(output['response'])
plt.savefig('results/compare_results.jpg')
plt.close()
if __name__ == "__main__":
sys_nlu = SVMNLU(mode='all')
sys_dst = RuleDST()
# rule policy
sys_policy = RulePolicy(character='sys')
# template NLG
sys_nlg = TemplateNLG(is_user=False)
# assemble
if self.use_nlu:
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')
else:
sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, 'sys')
user_nlu = SVMNLU(mode='all')
# not use dst
user_dst = None
# rule policy
user_policy = RulePolicy(character='usr')
# template NLG
user_nlg = TemplateNLG(is_user=True)
# assemble
if self.use_nlu:
user_agent = PipelineAgent(user_nlu, None, user_policy, user_nlg,
'user')