def test_end2end():
# go to README.md of each model for more information
# BERT nlu
sys_nlu = BERTNLU()
# simple rule DST
sys_dst = RuleDST()
# rule policy
sys_policy = PPOPolicy()
# template NLG
sys_nlg = TemplateNLG(is_user=False)
# assemble
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, name='sys')
# BERT nlu trained on sys utterance
user_nlu = BERTNLU(mode='sys', config_file='multiwoz_sys_context.json',
model_file='https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_sys_context.zip')
# not use dst
user_dst = None
# rule policy
user_policy = RulePolicy(character='usr')
# template NLG
user_nlg = TemplateNLG(is_user=True)
# assemble
user_agent = PipelineAgent(user_nlu, user_dst, user_policy, user_nlg, name='user')
analyzer = Analyzer(user_agent=user_agent, dataset='multiwoz')
set_seed(20200202)
analyzer.comprehensive_analyze(sys_agent=sys_agent, model_name='BERTNLU-RuleDST-PPOPolicy-TemplateNLG', total_dialog=1000)
def __init__(self, model_file=DEFAULT_MODEL_URL, name="Dialog"):
super(Dialog, self).__init__(name=name)
if not os.path.exists(os.path.join(DEFAULT_DIRECTORY,'multiwoz/data')):
os.mkdir(os.path.join(DEFAULT_DIRECTORY,'multiwoz/data'))
### download multiwoz data
print('down load data from', DEFAULT_ARCHIVE_FILE_URL)
if not os.path.exists(os.path.join(DEFAULT_DIRECTORY,'multiwoz/save')):
os.mkdir(os.path.join(DEFAULT_DIRECTORY,'multiwoz/save'))
### download trained model
print('down load model from', DEFAULT_MODEL_URL)
model_path = ""
config = Config()
parser = config.parser
config = parser.parse_args()
with open("assets/never_split.txt") as f:
never_split = f.read().split("\n")
self.tokenizer = BertTokenizer("assets/vocab.txt", never_split=never_split)
self.nlu = BERTNLU()
self.dst_ = DST(config).cuda()
ckpt = torch.load("save/model_Sun_Jun_21_07:08:48_2020.pt", map_location = lambda storage, loc: storage.cuda(local_rank))
self.dst_.load_state_dict(ckpt["model"])
self.dst_.eval()
self.policy = RulePolicy()
self.nlg = TemplateNLG(is_user=False)
self.init_session()
self.slot_mapping = {
"leave": "leaveAt",
"arrive": "arriveBy"
}
def set_user(user):
# MILU
user_nlu = BERTNLU()
# not use dst
user_dst = None
# rule policy
user_policy = RulePolicy(character='usr')
#'attraction', 'hotel', 'restaurant', 'train', 'taxi', 'hospital', 'police'
if user:
if user == '7':
user_policy.policy.goal_generator = GoalGenerator_7()
if user == 'attraction':
user_policy.policy.goal_generator = GoalGenerator_attraction()
elif user == 'hospital':
user_policy.policy.goal_generator = GoalGenerator_hospital()
elif user == 'hotel':
user_policy.policy.goal_generator = GoalGenerator_hotel()
elif user == 'police':
user_policy.policy.goal_generator = GoalGenerator_police()
elif user == 'restaurant':
user_policy.policy.goal_generator = GoalGenerator_restaurant()
elif user == 'taxi':
user_policy.policy.goal_generator = GoalGenerator_taxi()
elif user == 'train':
user_policy.policy.goal_generator = GoalGenerator_train()
# template NLG
user_nlg = TemplateNLG(is_user=True)
return user_nlu, user_dst, user_policy, user_nlg
def build_user_agent_bertnlu():
user_nlu = BERTNLU()
user_dst = None
user_policy = RulePolicy(character='usr')
user_nlg = TemplateNLG(is_user=True)
user_agent = PipelineAgent(user_nlu, user_dst, user_policy, user_nlg, 'user')
return user_agent
def build_sys_agent_svmnlu():
sys_nlu = SVMNLU()
sys_dst = RuleDST()
sys_policy = RulePolicy(character='sys')
sys_nlg = TemplateNLG(is_user=False)
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')
return sys_agent
def fine_tune(pos_action, neg_action, tokenizer, model):
nlg_usr = TemplateNLG(is_user=True)
nlg_sys = TemplateNLG(is_user=False)
pos_train_usr_utter = []
pos_train_sys_utter = []
neg_train_usr_utter = []
neg_train_sys_utter = []
for turn in pos_action:
if turn[0] != [] and turn[1] != []:
s_u = nlg_usr.generate(turn[0])
s_a = nlg_sys.generate(turn[1])
pos_train_usr_utter.append(s_u)
pos_train_sys_utter.append(s_a)
for turn in neg_action:
if turn[0] != [] and turn[1] != []:
s_u = nlg_usr.generate(turn[0])
s_a = nlg_sys.generate(turn[1])
neg_train_usr_utter.append(s_u)
neg_train_sys_utter.append(s_a)
train_usr_utter = pos_train_usr_utter + neg_train_usr_utter
train_sys_utter = pos_train_sys_utter + neg_train_sys_utter
train_encoding = tokenizer(train_usr_utter,
train_sys_utter,
padding=True,
truncation=True,
max_length=80)
train_encoding['label'] = [1] * len(pos_train_usr_utter) + [0] * len(
neg_train_usr_utter)
train_dataset = Dataset.from_dict(train_encoding)
train_dataset.set_format('torch',
columns=['input_ids', 'attention_mask', 'label'])
save_dir = os.path.join(root_dir,
'convlab2/policy/dqn/NLE/save/script_fine_tune')
log_dir = os.path.join(
root_dir, 'convlab2/policy/dqn/NLE/save/script_fine_tune/logs')
training_args = TrainingArguments(
output_dir=save_dir,
num_train_epochs=2,
per_device_train_batch_size=32,
per_device_eval_batch_size=128,
warmup_steps=500,
weight_decay=0.01,
evaluate_during_training=False,
logging_dir=log_dir,
)
trainer = Trainer(
model=model,
args=training_args,
train_dataset=train_dataset,
)
trainer.train()
trainer.save_model(os.path.join(save_dir, 'fine_tune_checkpoint'))
def set_system(sys_policy, sys_path):
# BERT nlu
sys_nlu = BERTNLU()
# simple rule DST
sys_dst = RuleDST()
# rule policy
sys_policy = get_policy(sys_policy, sys_path)
# template NLG
sys_nlg = TemplateNLG(is_user=False)
return sys_nlu, sys_dst, sys_policy, sys_nlg
def __init__(self, vocab, db, config):
super(DIALOG, self).__init__()
self.vocab = vocab
self.db = db
self.pointer_size = config.pointer_size
self.max_belief_len = config.max_belief_len
self.nlu = BERTNLU() # fixed, not finetuning
self.context_encoder = ContextEncoder(vocab.vocab_size, config.hidden_size, config.hidden_size, config.dropout, config.num_layers, vocab.word2idx["<pad>"])
self.belief_decoder = BeliefDecoder(vocab, self.context_encoder.embedding, config.hidden_size, config.dropout, config.num_layers, config.max_value_len)
self.policy = RulePolicy()
self.nlg = TemplateNLG(is_user=False)
# self.action_decoder = ActionDecoder(vocab, self.context_encoder.embedding, config.hidden_size, config.pointer_size, config.dropout, config.num_layers, config.max_act_len)
# self.response_decoder = ResponseDecoder(vocab, self.context_encoder.embedding, config.hidden_size, config.pointer_size, config.dropout, config.num_layers, config.max_sentence_len)
self.load_embedding() # load Glove & Kazuma embedding
def build_sys_agent_svmnlu(use_nlu=True):
sys_nlu = SVMNLU(mode='all')
sys_dst = RuleDST()
sys_policy = RulePolicy(character='sys')
sys_nlg = TemplateNLG(is_user=False)
if use_nlu:
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')
else:
sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, 'sys')
return sys_agent
def build_sys_agent_bertnlu_context(use_nlu=True):
sys_nlu = BERTNLU(mode='all', config_file='multiwoz_all_context.json',
model_file='https://tatk-data.s3-ap-northeast-1.amazonaws.com/bert_multiwoz_all_context.zip')
sys_dst = RuleDST()
sys_policy = RulePolicy(character='sys')
sys_nlg = TemplateNLG(is_user=False)
if use_nlu:
sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')
else:
sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, 'sys')
return sys_agent
def build_user_agent_svmnlu(use_nlu=True):
user_nlu = SVMNLU(mode='all')
user_dst = None
user_policy = RulePolicy(character='usr')
user_nlg = TemplateNLG(is_user=True)
if use_nlu:
user_agent = PipelineAgent(user_nlu, None, user_policy, user_nlg, 'user')
else:
user_agent = PipelineAgent(None, None, user_policy, None, 'user')
return user_agent
def build_user_agent_bertnlu(use_nlu=True):
user_nlu = BERTNLU(mode='all', config_file='multiwoz_all.json',
model_file='https://tatk-data.s3-ap-northeast-1.amazonaws.com/bert_multiwoz_all.zip')
user_dst = None
user_policy = RulePolicy(character='usr')
user_nlg = TemplateNLG(is_user=True)
if use_nlu:
user_agent = PipelineAgent(user_nlu, None, user_policy, user_nlg, 'user')
else:
user_agent = PipelineAgent(None, None, user_policy, None, 'user')
return user_agent
def __init__(self, dataset='multiwoz-test'):
user_nlu = BERTNLU(
mode='sys',
config_file='multiwoz_sys_context.json',
model_file=
'https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_sys_context.zip'
)
user_dst = None
user_policy = RulePolicy(character='usr')
user_nlg = TemplateNLG(is_user=True)
user_agent = PipelineAgent(user_nlu,
user_dst,
user_policy,
user_nlg,
name='user')
dataset, _ = data.split_name(dataset)
super().__init__(user_agent, dataset)
from convlab2.nlg.template.multiwoz import TemplateNLG
from convlab2.evaluator.multiwoz_eval import MultiWozEvaluator
import random
import numpy as np
from pprint import pprint
rgi_queue = PriorityQueue(maxsize=0)
rgo_queue = PriorityQueue(maxsize=0)
app = Flask(__name__)
# sys_nlu = BERTNLU()
sys_nlu = MILU()
sys_dst = RuleDST()
sys_policy = RulePolicy(character='sys')
sys_nlg = TemplateNLG(is_user=False)
agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys')
print(agent.response('I am looking for a hotel'))
@app.route('/', methods=['GET', 'POST'])
def process():
try:
in_request = request.json
print(in_request)
except:
return "invalid input: {}".format(in_request)
rgi_queue.put(in_request)
rgi_queue.join()
print("\t role=usr/sys")
sys.exit()
dataset_name = sys.argv[1]
model_name = sys.argv[2]
role = sys.argv[3]
if dataset_name == 'MultiWOZ':
if model_name == 'SCLSTM':
from convlab2.nlg.sclstm.multiwoz import SCLSTM
if role == 'usr':
model = SCLSTM(is_user=True, use_cuda=True)
elif role == 'sys':
model = SCLSTM(is_user=False, use_cuda=True)
elif model_name == 'TemplateNLG':
from convlab2.nlg.template.multiwoz import TemplateNLG
if role == 'usr':
model = TemplateNLG(is_user=True)
elif role == 'sys':
model = TemplateNLG(is_user=False)
else:
raise Exception("Available models: SCLSTM, TEMPLATE")
from convlab2.util.dataloader.module_dataloader import SingleTurnNLGDataloader
from convlab2.util.dataloader.dataset_dataloader import MultiWOZDataloader
dataloader = SingleTurnNLGDataloader(
dataset_dataloader=MultiWOZDataloader())
data = dataloader.load_data(data_key='test', role=role)['test']
dialog_acts = []
golden_utts = []
gen_utts = []
gen_slots = []
class Dialog(Agent):
def __init__(self, model_file=DEFAULT_MODEL_URL, name="Dialog"):
super(Dialog, self).__init__(name=name)
if not os.path.exists(os.path.join(DEFAULT_DIRECTORY,'multiwoz/data')):
os.mkdir(os.path.join(DEFAULT_DIRECTORY,'multiwoz/data'))
### download multiwoz data
print('down load data from', DEFAULT_ARCHIVE_FILE_URL)
if not os.path.exists(os.path.join(DEFAULT_DIRECTORY,'multiwoz/save')):
os.mkdir(os.path.join(DEFAULT_DIRECTORY,'multiwoz/save'))
### download trained model
print('down load model from', DEFAULT_MODEL_URL)
model_path = ""
config = Config()
parser = config.parser
config = parser.parse_args()
with open("assets/never_split.txt") as f:
never_split = f.read().split("\n")
self.tokenizer = BertTokenizer("assets/vocab.txt", never_split=never_split)
self.nlu = BERTNLU()
self.dst_ = DST(config).cuda()
ckpt = torch.load("save/model_Sun_Jun_21_07:08:48_2020.pt", map_location = lambda storage, loc: storage.cuda(local_rank))
self.dst_.load_state_dict(ckpt["model"])
self.dst_.eval()
self.policy = RulePolicy()
self.nlg = TemplateNLG(is_user=False)
self.init_session()
self.slot_mapping = {
"leave": "leaveAt",
"arrive": "arriveBy"
}
def init_session(self):
self.nlu.init_session()
self.policy.init_session()
self.nlg.init_session()
self.history = []
self.state = default_state()
pass
def response(self, user):
self.history.append(["user", user])
user_action = []
self.input_action = self.nlu.predict(user, context=[x[1] for x in self.history[:-1]])
self.input_action = deepcopy(self.input_action)
for act in self.input_action:
intent, domain, slot, value = act
if intent == "Request":
user_action.append(act)
if not self.state["request_state"].get(domain):
self.state["request_state"][domain] = {}
if slot not in self.state["request_state"][domain]:
self.state['request_state'][domain][slot] = 0
context = " ".join([utterance[1] for utterance in self.history])
context = context[-MAX_CONTEXT_LENGTH:]
context = self.tokenizer.encode(context)
context = torch.tensor(context, dtype=torch.int64).unsqueeze(dim=0).cuda() # [1, len]
belief_gen = self.dst_(None, context, 0, test=True)[0] # [slots, len]
for slot_idx, domain_slot in enumerate(ontology.all_info_slots):
domain, slot = domain_slot.split("-")
slot = self.slot_mapping.get(slot, slot)
value = belief_gen[slot_idx][:-1] # remove <EOS>
value = self.tokenizer.decode(value)
if value != "none":
if slot in self.state["belief_state"][domain]["book"].keys():
if self.state["belief_state"][domain]["book"][slot] == "":
action = ["Inform", domain.capitalize(), REF_USR_DA[domain].get(slot, slot), value]
user_action.append(action)
self.state["belief_state"][domain]["book"][slot] = value
elif slot in self.state["belief_state"][domain]["semi"].keys():
if self.state["belief_state"][domain]["semi"][slot] == "":
action = ["Inform", domain.capitalize(), REF_USR_DA[domain].get(slot, slot), value]
user_action.append(action)
self.state["belief_state"][domain]["semi"][slot] = value
self.state["user_action"] = user_action
self.output_action = deepcopy(self.policy.predict(self.state))
model_response = self.nlg.generate(self.output_action)
self.history.append(["sys", model_response])
return model_response
if __name__ == '__main__':
# BERT nlu
sys_nlu = BERTNLU()
# simple rule DST
sys_dst = RuleDST()
# rule policy
sys_policy = RulePolicy()
from convlab2.policy.mgail.multiwoz import MGAIL
policy_sys = MGAIL()
policy_sys.load(
'/home/nightop/ConvLab-2/convlab2/policy/mgail/multiwoz/save/all/99')
# template NLG
sys_nlg = TemplateNLG(is_user=False)
# assemble
sys_agent = PipelineAgent(sys_nlu,
sys_dst,
sys_policy,
sys_nlg,
name='sys')
# MILU
user_nlu = BERTNLU()
# not use dst
user_dst = None
# rule policy
user_policy = RulePolicy(character='usr')
#user_policy.policy.goal_generator = GoalGenerator_7()
#user_policy.policy.goal_generator = GoalGenerator_restaurant()
for gen in das2utts[das]['gens']:
refs.append([s.split() for s in das2utts[das]['refs']])
gens.append(gen.split())
bleu = corpus_bleu(refs, gens, weights=(0.25, 0.25, 0.25, 0.25), smoothing_function=SmoothingFunction().method1)
return bleu
if __name__ == '__main__':
if len(sys.argv) != 2:
print("usage:")
print("\t python evaluate.py data_key")
print("\t data_key=usr/sys/all")
sys.exit()
data_key = sys.argv[1]
if data_key=='all' or data_key=='usr':
model_usr = TemplateNLG(is_user=True)
if data_key=='all' or data_key=='sys':
model_sys = TemplateNLG(is_user=False)
archive = zipfile.ZipFile('../../../../data/multiwoz/test.json.zip', 'r')
test_data = json.load(archive.open('test.json'))
dialog_acts = []
golden_utts = []
gen_utts = []
gen_slots = []
sen_num = 0
sess_num = 0
for no, sess in list(test_data.items()):
sess_num+=1
