def test_end2end(): # go to README.md of each model for more information # BERT nlu sys_nlu = BERTNLU() # simple rule DST sys_dst = RuleDST() # rule policy sys_policy = PPOPolicy() # template NLG sys_nlg = TemplateNLG(is_user=False) # assemble sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, name='sys') # BERT nlu trained on sys utterance user_nlu = BERTNLU(mode='sys', config_file='multiwoz_sys_context.json', model_file='https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_sys_context.zip') # not use dst user_dst = None # rule policy user_policy = RulePolicy(character='usr') # template NLG user_nlg = TemplateNLG(is_user=True) # assemble user_agent = PipelineAgent(user_nlu, user_dst, user_policy, user_nlg, name='user') analyzer = Analyzer(user_agent=user_agent, dataset='multiwoz') set_seed(20200202) analyzer.comprehensive_analyze(sys_agent=sys_agent, model_name='BERTNLU-RuleDST-PPOPolicy-TemplateNLG', total_dialog=1000)
def __init__(self, model_file=DEFAULT_MODEL_URL, name="Dialog"): super(Dialog, self).__init__(name=name) if not os.path.exists(os.path.join(DEFAULT_DIRECTORY,'multiwoz/data')): os.mkdir(os.path.join(DEFAULT_DIRECTORY,'multiwoz/data')) ### download multiwoz data print('down load data from', DEFAULT_ARCHIVE_FILE_URL) if not os.path.exists(os.path.join(DEFAULT_DIRECTORY,'multiwoz/save')): os.mkdir(os.path.join(DEFAULT_DIRECTORY,'multiwoz/save')) ### download trained model print('down load model from', DEFAULT_MODEL_URL) model_path = "" config = Config() parser = config.parser config = parser.parse_args() with open("assets/never_split.txt") as f: never_split = f.read().split("\n") self.tokenizer = BertTokenizer("assets/vocab.txt", never_split=never_split) self.nlu = BERTNLU() self.dst_ = DST(config).cuda() ckpt = torch.load("save/model_Sun_Jun_21_07:08:48_2020.pt", map_location = lambda storage, loc: storage.cuda(local_rank)) self.dst_.load_state_dict(ckpt["model"]) self.dst_.eval() self.policy = RulePolicy() self.nlg = TemplateNLG(is_user=False) self.init_session() self.slot_mapping = { "leave": "leaveAt", "arrive": "arriveBy" }
def set_user(user): # MILU user_nlu = BERTNLU() # not use dst user_dst = None # rule policy user_policy = RulePolicy(character='usr') #'attraction', 'hotel', 'restaurant', 'train', 'taxi', 'hospital', 'police' if user: if user == '7': user_policy.policy.goal_generator = GoalGenerator_7() if user == 'attraction': user_policy.policy.goal_generator = GoalGenerator_attraction() elif user == 'hospital': user_policy.policy.goal_generator = GoalGenerator_hospital() elif user == 'hotel': user_policy.policy.goal_generator = GoalGenerator_hotel() elif user == 'police': user_policy.policy.goal_generator = GoalGenerator_police() elif user == 'restaurant': user_policy.policy.goal_generator = GoalGenerator_restaurant() elif user == 'taxi': user_policy.policy.goal_generator = GoalGenerator_taxi() elif user == 'train': user_policy.policy.goal_generator = GoalGenerator_train() # template NLG user_nlg = TemplateNLG(is_user=True) return user_nlu, user_dst, user_policy, user_nlg
def build_user_agent_bertnlu(): user_nlu = BERTNLU() user_dst = None user_policy = RulePolicy(character='usr') user_nlg = TemplateNLG(is_user=True) user_agent = PipelineAgent(user_nlu, user_dst, user_policy, user_nlg, 'user') return user_agent
def build_sys_agent_svmnlu(): sys_nlu = SVMNLU() sys_dst = RuleDST() sys_policy = RulePolicy(character='sys') sys_nlg = TemplateNLG(is_user=False) sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys') return sys_agent
def fine_tune(pos_action, neg_action, tokenizer, model): nlg_usr = TemplateNLG(is_user=True) nlg_sys = TemplateNLG(is_user=False) pos_train_usr_utter = [] pos_train_sys_utter = [] neg_train_usr_utter = [] neg_train_sys_utter = [] for turn in pos_action: if turn[0] != [] and turn[1] != []: s_u = nlg_usr.generate(turn[0]) s_a = nlg_sys.generate(turn[1]) pos_train_usr_utter.append(s_u) pos_train_sys_utter.append(s_a) for turn in neg_action: if turn[0] != [] and turn[1] != []: s_u = nlg_usr.generate(turn[0]) s_a = nlg_sys.generate(turn[1]) neg_train_usr_utter.append(s_u) neg_train_sys_utter.append(s_a) train_usr_utter = pos_train_usr_utter + neg_train_usr_utter train_sys_utter = pos_train_sys_utter + neg_train_sys_utter train_encoding = tokenizer(train_usr_utter, train_sys_utter, padding=True, truncation=True, max_length=80) train_encoding['label'] = [1] * len(pos_train_usr_utter) + [0] * len( neg_train_usr_utter) train_dataset = Dataset.from_dict(train_encoding) train_dataset.set_format('torch', columns=['input_ids', 'attention_mask', 'label']) save_dir = os.path.join(root_dir, 'convlab2/policy/dqn/NLE/save/script_fine_tune') log_dir = os.path.join( root_dir, 'convlab2/policy/dqn/NLE/save/script_fine_tune/logs') training_args = TrainingArguments( output_dir=save_dir, num_train_epochs=2, per_device_train_batch_size=32, per_device_eval_batch_size=128, warmup_steps=500, weight_decay=0.01, evaluate_during_training=False, logging_dir=log_dir, ) trainer = Trainer( model=model, args=training_args, train_dataset=train_dataset, ) trainer.train() trainer.save_model(os.path.join(save_dir, 'fine_tune_checkpoint'))
def set_system(sys_policy, sys_path): # BERT nlu sys_nlu = BERTNLU() # simple rule DST sys_dst = RuleDST() # rule policy sys_policy = get_policy(sys_policy, sys_path) # template NLG sys_nlg = TemplateNLG(is_user=False) return sys_nlu, sys_dst, sys_policy, sys_nlg
def __init__(self, vocab, db, config): super(DIALOG, self).__init__() self.vocab = vocab self.db = db self.pointer_size = config.pointer_size self.max_belief_len = config.max_belief_len self.nlu = BERTNLU() # fixed, not finetuning self.context_encoder = ContextEncoder(vocab.vocab_size, config.hidden_size, config.hidden_size, config.dropout, config.num_layers, vocab.word2idx["<pad>"]) self.belief_decoder = BeliefDecoder(vocab, self.context_encoder.embedding, config.hidden_size, config.dropout, config.num_layers, config.max_value_len) self.policy = RulePolicy() self.nlg = TemplateNLG(is_user=False) # self.action_decoder = ActionDecoder(vocab, self.context_encoder.embedding, config.hidden_size, config.pointer_size, config.dropout, config.num_layers, config.max_act_len) # self.response_decoder = ResponseDecoder(vocab, self.context_encoder.embedding, config.hidden_size, config.pointer_size, config.dropout, config.num_layers, config.max_sentence_len) self.load_embedding() # load Glove & Kazuma embedding
def build_sys_agent_svmnlu(use_nlu=True): sys_nlu = SVMNLU(mode='all') sys_dst = RuleDST() sys_policy = RulePolicy(character='sys') sys_nlg = TemplateNLG(is_user=False) if use_nlu: sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys') else: sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, 'sys') return sys_agent
def build_sys_agent_bertnlu_context(use_nlu=True): sys_nlu = BERTNLU(mode='all', config_file='multiwoz_all_context.json', model_file='https://tatk-data.s3-ap-northeast-1.amazonaws.com/bert_multiwoz_all_context.zip') sys_dst = RuleDST() sys_policy = RulePolicy(character='sys') sys_nlg = TemplateNLG(is_user=False) if use_nlu: sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys') else: sys_agent = PipelineAgent(None, sys_dst, sys_policy, None, 'sys') return sys_agent
def build_user_agent_svmnlu(use_nlu=True): user_nlu = SVMNLU(mode='all') user_dst = None user_policy = RulePolicy(character='usr') user_nlg = TemplateNLG(is_user=True) if use_nlu: user_agent = PipelineAgent(user_nlu, None, user_policy, user_nlg, 'user') else: user_agent = PipelineAgent(None, None, user_policy, None, 'user') return user_agent
def build_user_agent_bertnlu(use_nlu=True): user_nlu = BERTNLU(mode='all', config_file='multiwoz_all.json', model_file='https://tatk-data.s3-ap-northeast-1.amazonaws.com/bert_multiwoz_all.zip') user_dst = None user_policy = RulePolicy(character='usr') user_nlg = TemplateNLG(is_user=True) if use_nlu: user_agent = PipelineAgent(user_nlu, None, user_policy, user_nlg, 'user') else: user_agent = PipelineAgent(None, None, user_policy, None, 'user') return user_agent
def __init__(self, dataset='multiwoz-test'): user_nlu = BERTNLU( mode='sys', config_file='multiwoz_sys_context.json', model_file= 'https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_sys_context.zip' ) user_dst = None user_policy = RulePolicy(character='usr') user_nlg = TemplateNLG(is_user=True) user_agent = PipelineAgent(user_nlu, user_dst, user_policy, user_nlg, name='user') dataset, _ = data.split_name(dataset) super().__init__(user_agent, dataset)
from convlab2.nlg.template.multiwoz import TemplateNLG from convlab2.evaluator.multiwoz_eval import MultiWozEvaluator import random import numpy as np from pprint import pprint rgi_queue = PriorityQueue(maxsize=0) rgo_queue = PriorityQueue(maxsize=0) app = Flask(__name__) # sys_nlu = BERTNLU() sys_nlu = MILU() sys_dst = RuleDST() sys_policy = RulePolicy(character='sys') sys_nlg = TemplateNLG(is_user=False) agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, 'sys') print(agent.response('I am looking for a hotel')) @app.route('/', methods=['GET', 'POST']) def process(): try: in_request = request.json print(in_request) except: return "invalid input: {}".format(in_request) rgi_queue.put(in_request) rgi_queue.join()
print("\t role=usr/sys") sys.exit() dataset_name = sys.argv[1] model_name = sys.argv[2] role = sys.argv[3] if dataset_name == 'MultiWOZ': if model_name == 'SCLSTM': from convlab2.nlg.sclstm.multiwoz import SCLSTM if role == 'usr': model = SCLSTM(is_user=True, use_cuda=True) elif role == 'sys': model = SCLSTM(is_user=False, use_cuda=True) elif model_name == 'TemplateNLG': from convlab2.nlg.template.multiwoz import TemplateNLG if role == 'usr': model = TemplateNLG(is_user=True) elif role == 'sys': model = TemplateNLG(is_user=False) else: raise Exception("Available models: SCLSTM, TEMPLATE") from convlab2.util.dataloader.module_dataloader import SingleTurnNLGDataloader from convlab2.util.dataloader.dataset_dataloader import MultiWOZDataloader dataloader = SingleTurnNLGDataloader( dataset_dataloader=MultiWOZDataloader()) data = dataloader.load_data(data_key='test', role=role)['test'] dialog_acts = [] golden_utts = [] gen_utts = [] gen_slots = []
class Dialog(Agent): def __init__(self, model_file=DEFAULT_MODEL_URL, name="Dialog"): super(Dialog, self).__init__(name=name) if not os.path.exists(os.path.join(DEFAULT_DIRECTORY,'multiwoz/data')): os.mkdir(os.path.join(DEFAULT_DIRECTORY,'multiwoz/data')) ### download multiwoz data print('down load data from', DEFAULT_ARCHIVE_FILE_URL) if not os.path.exists(os.path.join(DEFAULT_DIRECTORY,'multiwoz/save')): os.mkdir(os.path.join(DEFAULT_DIRECTORY,'multiwoz/save')) ### download trained model print('down load model from', DEFAULT_MODEL_URL) model_path = "" config = Config() parser = config.parser config = parser.parse_args() with open("assets/never_split.txt") as f: never_split = f.read().split("\n") self.tokenizer = BertTokenizer("assets/vocab.txt", never_split=never_split) self.nlu = BERTNLU() self.dst_ = DST(config).cuda() ckpt = torch.load("save/model_Sun_Jun_21_07:08:48_2020.pt", map_location = lambda storage, loc: storage.cuda(local_rank)) self.dst_.load_state_dict(ckpt["model"]) self.dst_.eval() self.policy = RulePolicy() self.nlg = TemplateNLG(is_user=False) self.init_session() self.slot_mapping = { "leave": "leaveAt", "arrive": "arriveBy" } def init_session(self): self.nlu.init_session() self.policy.init_session() self.nlg.init_session() self.history = [] self.state = default_state() pass def response(self, user): self.history.append(["user", user]) user_action = [] self.input_action = self.nlu.predict(user, context=[x[1] for x in self.history[:-1]]) self.input_action = deepcopy(self.input_action) for act in self.input_action: intent, domain, slot, value = act if intent == "Request": user_action.append(act) if not self.state["request_state"].get(domain): self.state["request_state"][domain] = {} if slot not in self.state["request_state"][domain]: self.state['request_state'][domain][slot] = 0 context = " ".join([utterance[1] for utterance in self.history]) context = context[-MAX_CONTEXT_LENGTH:] context = self.tokenizer.encode(context) context = torch.tensor(context, dtype=torch.int64).unsqueeze(dim=0).cuda() # [1, len] belief_gen = self.dst_(None, context, 0, test=True)[0] # [slots, len] for slot_idx, domain_slot in enumerate(ontology.all_info_slots): domain, slot = domain_slot.split("-") slot = self.slot_mapping.get(slot, slot) value = belief_gen[slot_idx][:-1] # remove <EOS> value = self.tokenizer.decode(value) if value != "none": if slot in self.state["belief_state"][domain]["book"].keys(): if self.state["belief_state"][domain]["book"][slot] == "": action = ["Inform", domain.capitalize(), REF_USR_DA[domain].get(slot, slot), value] user_action.append(action) self.state["belief_state"][domain]["book"][slot] = value elif slot in self.state["belief_state"][domain]["semi"].keys(): if self.state["belief_state"][domain]["semi"][slot] == "": action = ["Inform", domain.capitalize(), REF_USR_DA[domain].get(slot, slot), value] user_action.append(action) self.state["belief_state"][domain]["semi"][slot] = value self.state["user_action"] = user_action self.output_action = deepcopy(self.policy.predict(self.state)) model_response = self.nlg.generate(self.output_action) self.history.append(["sys", model_response]) return model_response
if __name__ == '__main__': # BERT nlu sys_nlu = BERTNLU() # simple rule DST sys_dst = RuleDST() # rule policy sys_policy = RulePolicy() from convlab2.policy.mgail.multiwoz import MGAIL policy_sys = MGAIL() policy_sys.load( '/home/nightop/ConvLab-2/convlab2/policy/mgail/multiwoz/save/all/99') # template NLG sys_nlg = TemplateNLG(is_user=False) # assemble sys_agent = PipelineAgent(sys_nlu, sys_dst, sys_policy, sys_nlg, name='sys') # MILU user_nlu = BERTNLU() # not use dst user_dst = None # rule policy user_policy = RulePolicy(character='usr') #user_policy.policy.goal_generator = GoalGenerator_7() #user_policy.policy.goal_generator = GoalGenerator_restaurant()
for gen in das2utts[das]['gens']: refs.append([s.split() for s in das2utts[das]['refs']]) gens.append(gen.split()) bleu = corpus_bleu(refs, gens, weights=(0.25, 0.25, 0.25, 0.25), smoothing_function=SmoothingFunction().method1) return bleu if __name__ == '__main__': if len(sys.argv) != 2: print("usage:") print("\t python evaluate.py data_key") print("\t data_key=usr/sys/all") sys.exit() data_key = sys.argv[1] if data_key=='all' or data_key=='usr': model_usr = TemplateNLG(is_user=True) if data_key=='all' or data_key=='sys': model_sys = TemplateNLG(is_user=False) archive = zipfile.ZipFile('../../../../data/multiwoz/test.json.zip', 'r') test_data = json.load(archive.open('test.json')) dialog_acts = [] golden_utts = [] gen_utts = [] gen_slots = [] sen_num = 0 sess_num = 0 for no, sess in list(test_data.items()): sess_num+=1