def build_sys_agent_svmnlu():
sys_nlu = SVMNLU()
sys_dst = RuleDST()
sys_policy = RulePolicy(character='sys')
sys_nlg = TemplateNLG(is_user=False)
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')
return sys_agent
def build_sys_agent_svmnlu(use_nlu=True):
sys_nlu = SVMNLU(mode='all')
sys_dst = RuleDST()
sys_policy = RulePolicy(character='sys')
sys_nlg = TemplateNLG(is_user=False)
if use_nlu:
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')
else:
sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, 'sys')
return sys_agent
def build_user_agent_svmnlu(use_nlu=True):
user_nlu = SVMNLU(mode='all')
user_dst = None
user_policy = RulePolicy(character='usr')
user_nlg = TemplateNLG(is_user=True)
if use_nlu:
user_agent = PipelineAgent(user_nlu, None, user_policy, user_nlg, 'user')
else:
user_agent = PipelineAgent(None, None, user_policy, None, 'user')
return user_agent
def test_end2end():
# go to README.md of each model for more information
# BERT nlu
sys_nlu = SVMNLU()
# simple rule DST
sys_dst = RuleDST()
# rule policy
sys_policy = RulePolicy()
# template NLG
sys_nlg = TemplateNLG(is_user=False)
# assemble
sys_agent = PipelineAgent(sys_nlu,
sys_dst,
sys_policy,
sys_nlg,
name='sys')
# BERT nlu trained on sys utterance
user_nlu = BERTNLU(
mode='sys',
config_file='multiwoz_sys_context.json',
model_file=
'https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_sys_context.zip'
)
# not use dst
user_dst = None
# rule policy
user_policy = RulePolicy(character='usr')
# template NLG
user_nlg = TemplateNLG(is_user=True)
# assemble
user_agent = PipelineAgent(user_nlu,
user_dst,
user_policy,
user_nlg,
name='user')
analyzer = Analyzer(user_agent=user_agent, dataset='multiwoz')
set_seed(20200202)
analyzer.comprehensive_analyze(
sys_agent=sys_agent,
model_name='SVMNLU-RuleDST-RulePolicy-TemplateNLG',
total_dialog=1000)
triples = []
for intent, svs in dialog_act.items():
for slot, value in svs:
triples.append([intent, slot, value])
return triples
if __name__ == '__main__':
if len(sys.argv) != 2:
print("usage:")
print("\t python evaluate.py mode")
print("\t mode=usr|sys|all")
sys.exit()
mode = sys.argv[1]
if mode== 'usr':
model = SVMNLU(mode='usr')
elif mode== 'sys':
model = SVMNLU(mode='sys')
elif mode== 'all':
model = SVMNLU(mode='all')
else:
raise Exception("Invalid mode")
archive = zipfile.ZipFile('../../../../data/multiwoz/test.json.zip', 'r')
test_data = json.load(archive.open('test.json'))
TP, FP, FN = 0, 0, 0
sen_num = 0
sess_num = 0
for no, session in test_data.items():
sess_num += 1
if sess_num%10==0:
width=0.1,
align='center',
tick_label=model_name,
label='Precision')
plt.bar(x3, y3, width=0.1, align='center', label='Recall')
plt.bar(x4, y4, width=0.1, align='center', label='Inform F1')
plt.legend(loc=2, prop=font1)
if not os.path.exists('results/'):
os.mkdir('results')
plt.savefig('results/compare_results.jpg')
plt.close()
if __name__ == "__main__":
sys_nlu = SVMNLU(mode='all')
sys_dst = RuleDST()
# rule policy
sys_policy = RulePolicy(character='sys')
# template NLG
sys_nlg = TemplateNLG(is_user=False)
# assemble
if self.use_nlu:
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')
else:
sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, 'sys')
user_nlu = SVMNLU(mode='all')
# not use dst