def test_end2end():
# go to README.md of each model for more information
# BERT nlu
sys_nlu = None # BERTNLU()
# simple rule DST
sys_dst = RuleDST()
# rule policy
sys_policy = PPOPolicy()
# template NLG
sys_nlg = None #TemplateNLG(is_user=False)
# assemble
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, name='sys')
# BERT nlu trained on sys utterance
user_nlu = None # BERTNLU(mode='sys', config_file='multiwoz_sys_context.json',
# model_file='https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_sys_context.zip')
# not use dst
user_dst = None #RuleDST() # None
# rule policy
user_policy = RulePolicy(character='usr')
# template NLG
user_nlg = None # TemplateNLG(is_user=True)
# assemble
user_agent = PipelineAgent(user_nlu, user_dst, user_policy, user_nlg, name='user')
analyzer = Analyzer(user_agent=user_agent, dataset='multiwoz')
set_seed(20200202)
analyzer.comprehensive_analyze(sys_agent=sys_agent, model_name='BERTNLU-RuleDST-PPOPolicy-TemplateNLG', total_dialog=1000)
def build_sys_agent_svmnlu():
sys_nlu = SVMNLU()
sys_dst = RuleDST()
sys_policy = RulePolicy(character='sys')
sys_nlg = TemplateNLG(is_user=False)
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')
return sys_agent
def set_system(sys_policy, sys_path):
# BERT nlu
sys_nlu = BERTNLU()
# simple rule DST
sys_dst = RuleDST()
# rule policy
sys_policy = get_policy(sys_policy, sys_path)
# template NLG
sys_nlg = TemplateNLG(is_user=False)
return sys_nlu, sys_dst, sys_policy, sys_nlg
def build_sys_agent_bertnlu_context(use_nlu=True):
sys_nlu = BERTNLU(mode='all', config_file='multiwoz_all_context.json',
model_file='https://tatk-data.s3-ap-northeast-1.amazonaws.com/bert_multiwoz_all_context.zip')
sys_dst = RuleDST()
sys_policy = RulePolicy(character='sys')
sys_nlg = TemplateNLG(is_user=False)
if use_nlu:
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')
else:
sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, 'sys')
return sys_agent
def build_sys_agent_svmnlu(use_nlu=True):
sys_nlu = SVMNLU(mode='all')
sys_dst = RuleDST()
sys_policy = RulePolicy(character='sys')
sys_nlg = TemplateNLG(is_user=False)
if use_nlu:
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')
else:
sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, 'sys')
return sys_agent
def test_end2end():
sys_dst = RuleDST()
sys_policy = DQNPolicy()
sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, name='sys')
user_policy = RulePolicy(character='usr')
user_agent = PipelineAgent(None, None, user_policy, None, name='user')
analyzer = Analyzer(user_agent=user_agent, dataset='multiwoz')
set_seed(20200202)
analyzer.comprehensive_analyze(sys_agent=sys_agent,
model_name='RuleDST-DQNPolicy',
total_dialog=1000)
def evaluate(dataset_name, model_name, load_path, calculate_reward=True):
seed = 20190827
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if dataset_name == 'MultiWOZ':
dst_sys = RuleDST()
if model_name == "PPO":
from convlab2.policy.ppo import PPO
if load_path:
policy_sys = PPO(False)
policy_sys.load(load_path)
else:
policy_sys = PPO.from_pretrained()
elif model_name == "PG":
from convlab2.policy.pg import PG
if load_path:
policy_sys = PG(False)
policy_sys.load(load_path)
else:
policy_sys = PG.from_pretrained()
elif model_name == "MLE":
from convlab2.policy.mle.multiwoz import MLE
if load_path:
policy_sys = MLE()
policy_sys.load(load_path)
else:
policy_sys = MLE.from_pretrained()
elif model_name == "GDPL":
from convlab2.policy.gdpl import GDPL
if load_path:
policy_sys = GDPL(False)
policy_sys.load(load_path)
else:
policy_sys = GDPL.from_pretrained()
elif model_name == "GAIL":
from convlab2.policy.gail import GAIL
if load_path:
policy_sys = GAIL(False)
policy_sys.load(load_path)
else:
policy_sys = GAIL.from_pretrained()
dst_usr = None
policy_usr = RulePolicy(character='usr')
simulator = PipelineAgent(None, None, policy_usr, None, 'user')
env = Environment(None, simulator, None, dst_sys)
agent_sys = PipelineAgent(None, dst_sys, policy_sys, None, 'sys')
evaluator = MultiWozEvaluator()
sess = BiSession(agent_sys, simulator, None, evaluator)
task_success = {'All': []}
for seed in range(100):
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
sess.init_session()
sys_response = []
logging.info('-'*50)
logging.info(f'seed {seed}')
for i in range(40):
sys_response, user_response, session_over, reward = sess.next_turn(sys_response)
if session_over is True:
task_succ = sess.evaluator.task_success()
logging.info(f'task success: {task_succ}')
logging.info(f'book rate: {sess.evaluator.book_rate()}')
logging.info(f'inform precision/recall/f1: {sess.evaluator.inform_F1()}')
logging.info('-'*50)
break
else:
task_succ = 0
for key in sess.evaluator.goal:
if key not in task_success:
task_success[key] = []
else:
task_success[key].append(task_succ)
task_success['All'].append(task_succ)
for key in task_success:
logging.info(f'{key} {len(task_success[key])} {np.average(task_success[key]) if len(task_success[key]) > 0 else 0}')
if calculate_reward:
reward_tot = []
for seed in range(100):
s = env.reset()
reward = []
value = []
mask = []
for t in range(40):
s_vec = torch.Tensor(policy_sys.vector.state_vectorize(s))
a = policy_sys.predict(s)
# interact with env
next_s, r, done = env.step(a)
logging.info(r)
reward.append(r)
if done: # one due to counting from 0, the one for the last turn
break
logging.info(f'{seed} reward: {np.mean(reward)}')
reward_tot.append(np.mean(reward))
logging.info(f'total avg reward: {np.mean(reward_tot)}')
else:
raise Exception("currently supported dataset: MultiWOZ")
from convlab2.dst.rule.multiwoz import RuleDST
from convlab2.policy.rule.multiwoz import RulePolicy
from convlab2.nlg.template.multiwoz import TemplateNLG
from convlab2.evaluator.multiwoz_eval import MultiWozEvaluator
import random
import numpy as np
from pprint import pprint
rgi_queue = PriorityQueue(maxsize=0)
rgo_queue = PriorityQueue(maxsize=0)
app = Flask(__name__)
# sys_nlu = BERTNLU()
sys_nlu = MILU()
sys_dst = RuleDST()
sys_policy = RulePolicy(character='sys')
sys_nlg = TemplateNLG(is_user=False)
agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')
print(agent.response('I am looking for a hotel'))
@app.route('/', methods=['GET', 'POST'])
def process():
try:
in_request = request.json
print(in_request)
except:
return "invalid input: {}".format(in_request)
# batch.action: ([1, a_dim], [1, a_dim]...)
# batch.reward/ batch.mask: ([1], [1]...)
s = torch.from_numpy(np.stack(batch.state)).to(device=DEVICE)
a = torch.from_numpy(np.stack(batch.action)).to(device=DEVICE)
r = torch.from_numpy(np.stack(batch.reward)).to(device=DEVICE)
mask = torch.Tensor(np.stack(batch.mask)).to(device=DEVICE)
batchsz_real = s.size(0)
policy.update(epoch, batchsz_real, s, a, r, mask)
if __name__ == '__main__':
# svm nlu trained on usr sentence of multiwoz
# nlu_sys = SVMNLU('usr')
# simple rule DST
dst_sys = RuleDST()
# rule policy
policy_sys = PPO(True)
# template NLG
# nlg_sys = TemplateNLG(is_user=False)
# svm nlu trained on sys sentence of multiwoz
# nlu_usr = SVMNLU('sys')
# not use dst
dst_usr = None
# rule policy
policy_usr = RulePolicy(character='usr')
# template NLG
# nlg_usr = TemplateNLG(is_user=True)
# assemble
simulator = PipelineAgent(None, None, policy_usr, None, 'simulator')
def evaluate(dataset_name, model_name, load_path):
seed = 20200722
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if dataset_name == 'MultiWOZ':
dst_sys = RuleDST()
if model_name == "PPO":
from convlab2.policy.ppo import PPO
if load_path:
policy_sys = PPO(False)
policy_sys.load(load_path)
else:
policy_sys = PPO.from_pretrained()
elif model_name == "DQN":
from convlab2.policy.dqn.DQN.DQN import DQN
if load_path:
policy_sys = DQN(False)
policy_sys.load(load_path)
else:
print('Please add load path.')
elif model_name == "DQfD_RE":
from convlab2.policy.dqn.RE.DQfD import DQfD
if load_path:
policy_sys = DQfD(False)
policy_sys.load(load_path)
else:
print('Please add load path.')
elif model_name == "DQfD_NLE":
from convlab2.policy.dqn.NLE.DQfD import DQfD
if load_path:
policy_sys = DQfD(False)
policy_sys.load(load_path)
else:
print('Please add load path.')
elif model_name == "MLE":
from convlab2.policy.mle.multiwoz import MLE
if load_path:
policy_sys = MLE()
policy_sys.load(load_path)
else:
policy_sys = MLE.from_pretrained()
policy_usr = RulePolicy(character='usr')
simulator = PipelineAgent(None, None, policy_usr, None, 'user')
agent_sys = PipelineAgent(None, dst_sys, policy_sys, None, 'sys')
evaluator = MultiWozEvaluator()
sess = BiSession(agent_sys, simulator, None, evaluator)
task_success = 0
evaluator_success = 0
for seed in range(100):
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
sess.init_session()
sys_response = []
cur_success = 0
for i in range(40):
sys_response, user_response, session_over, reward = sess.next_turn(
sys_response)
if reward == 80:
cur_success = 1
task_success += 1
if session_over is True:
break
# logging.debug('Current task success: {}, the evaluator result: {}.'.format(cur_success, sess.evaluator.task_success()))
evaluator_success += sess.evaluator.task_success()
logging.debug('Task success rate: {} and evaluator result: {}.'.format(
task_success / 100, evaluator_success / 100))
return task_success / 100, evaluator_success / 100
import sys
import json
from tqdm import tqdm
import torch
import torch.nn as nn
import torch.nn.functional as F
import ontology
from reader import Reader, Vocab
from config import Config
from convlab2.nlu.jointBERT.multiwoz import BERTNLU
from convlab2.dst.rule.multiwoz import RuleDST
nlu = BERTNLU()
dst = RuleDST()
config = Config()
parser = config.parser
config = parser.parse_args()
vocab = Vocab(config)
vocab.load("save/vocab")
reader = Reader(vocab, config)
reader.load_data("train")
data = json.load(open("data/MultiWOZ_2.1/dev_data.json", "r"))
max_iter = len(list(reader.make_batch(reader.dev)))
iterator = reader.make_batch(reader.dev)
t = tqdm(enumerate(iterator), total=max_iter, ncols=250)