def __init__(self): Tracker.__init__(self) self.state = init_state() prefix = os.path.dirname(os.path.dirname(convlab.__file__)) self.value_dict = json.load( open(prefix + '/data/multiwoz/value_dict.json')) # changed part # self.domain_exclude = ['hotel', 'Hotel'] # self.domain_exclude = ['Restaurant', 'Attraction', 'Taxi', 'Train', 'restaurant', 'attraction', 'taxi', 'train'] self.domain_exclude = []
def __init__(self, data_dir='data/mdbt'): Tracker.__init__(self) # data profile self.data_dir = data_dir self.validation_url = os.path.join(self.data_dir, 'data/validate.json') self.word_vectors_url = os.path.join( self.data_dir, 'word-vectors/paragram_300_sl999.txt') self.training_url = os.path.join(self.data_dir, 'data/train.json') self.ontology_url = os.path.join(self.data_dir, 'data/ontology.json') self.testing_url = os.path.join(self.data_dir, 'data/test.json') self.model_url = os.path.join(self.data_dir, 'models/model-1') self.graph_url = os.path.join(self.data_dir, 'graphs/graph-1') self.results_url = os.path.join(self.data_dir, 'results/log-1.txt') self.kb_url = os.path.join(self.data_dir, 'data/') # not used self.train_model_url = os.path.join(self.data_dir, 'train_models/model-1') self.train_graph_url = os.path.join(self.data_dir, 'train_graph/graph-1') print('Configuring MDBT model...') self.word_vectors = load_word_vectors(self.word_vectors_url) # Load the ontology and extract the feature vectors self.ontology, self.ontology_vectors, self.slots = load_ontology( self.ontology_url, self.word_vectors) # Load and process the training data self.dialogues, self.actual_dialogues = load_woz_data( self.testing_url, self.word_vectors, self.ontology) self.no_dialogues = len(self.dialogues) self.model_variables = model_definition(self.ontology_vectors, len(self.ontology), self.slots, num_hidden=None, bidir=True, net_type=None, test=True, dev='cpu') self.state = init_state() _config = tf.ConfigProto() _config.gpu_options.allow_growth = True _config.allow_soft_placement = True self.sess = tf.Session(config=_config) self.param_restored = False self.det_dic = {} for domain, dic in REF_USR_DA.items(): for key, value in dic.items(): assert '-' not in key self.det_dic[key.lower()] = key + '-' + domain self.det_dic[value.lower()] = key + '-' + domain self.value_dict = json.load( open(os.path.join(self.data_dir, '../multiwoz/value_dict.json')))
def __init__(self, archive_file=DEFAULT_ARCHIVE_FILE, cuda_device=DEFAULT_CUDA_DEVICE, model_file=None): if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for MDRG is specified!") archive_file = cached_path(model_file) temp_path = tempfile.mkdtemp() zip_ref = zipfile.ZipFile(archive_file, 'r') zip_ref.extractall(temp_path) zip_ref.close() self.dic = pickle.load( open(os.path.join(temp_path, 'mdrg/svdic.pkl'), 'rb')) # Load dictionaries with open(os.path.join(temp_path, 'mdrg/input_lang.index2word.json')) as f: input_lang_index2word = json.load(f) with open(os.path.join(temp_path, 'mdrg/input_lang.word2index.json')) as f: input_lang_word2index = json.load(f) with open(os.path.join(temp_path, 'mdrg/output_lang.index2word.json')) as f: output_lang_index2word = json.load(f) with open(os.path.join(temp_path, 'mdrg/output_lang.word2index.json')) as f: output_lang_word2index = json.load(f) self.response_model = Model(args, input_lang_index2word, output_lang_index2word, input_lang_word2index, output_lang_word2index) self.response_model.loadModel(os.path.join(temp_path, 'mdrg/mdrg')) shutil.rmtree(temp_path) self.prev_state = init_state() self.prev_active_domain = None
def predict(self, state): """ Predict the dialog act of a natural language utterance and apply error model. Args: utterance (str): A natural language utterance. Returns: output (dict): The dialog act of utterance. """ state_vector = self.state_encoder.encode(state) instance = self.dataset_reader.text_to_instance(state_vector) outputs = self.model.forward_on_instance(instance) dialacts = self.action_decoder.decode(outputs["actions"], state) if dialacts == {'general-bye': [['none', 'none']]}: outputs["probs"][outputs["actions"]] = 0 outputs["actions"] = np.argmax(outputs["probs"]) dialacts = self.action_decoder.decode(outputs["actions"], state) if state == init_state(): dialacts = {} return dialacts
def init_session(self): self.state = init_state() if not self.param_restored: self.load_param() self.param_restored = True
self.model.eval() def predict(self, state): """ Predict the dialog act of a natural language utterance and apply error model. Args: utterance (str): A natural language utterance. Returns: output (dict): The dialog act of utterance. """ state_vector = self.state_encoder.encode(state) instance = self.dataset_reader.text_to_instance(state_vector) outputs = self.model.forward_on_instance(instance) dialacts = self.action_decoder.decode(outputs["actions"], state) if dialacts == {'general-bye': [['none', 'none']]}: outputs["probs"][outputs["actions"]] = 0 outputs["actions"] = np.argmax(outputs["probs"]) dialacts = self.action_decoder.decode(outputs["actions"], state) if state == init_state(): dialacts = {} return dialacts if __name__ == "__main__": policy = VanillaMLEPolicy() pprint(policy.predict(init_state()))
def predict_response(self, state): history = [] for i in range(len(state['history'])): for j in range(len(state['history'][i])): history.append(state['history'][i][j]) e_idx = len(history) s_idx = max(0, e_idx - self.config.backward_size) context = [] for turn in history[s_idx: e_idx]: # turn = pad_to(config.max_utt_len, turn, do_pad=False) context.append(turn) if len(state['history']) == 1: self.prev_state = init_state() prepared_data = {} prepared_data['context'] = [] prepared_data['response'] = {} prev_bstate = deepcopy(self.prev_state['belief_state']) state_history = state['history'] bstate = deepcopy(state['belief_state']) # mark_not_mentioned(prev_state) active_domain = self.get_active_domain( self.prev_active_domain, prev_bstate, bstate) domain_mark_not_mentioned(bstate, active_domain) top_results, num_results = None, None for usr in context: words = usr.split() usr = delexicalize.delexicalise(' '.join(words), self.dic) # parsing reference number GIVEN belief state usr = delexicaliseReferenceNumber(usr, bstate) # changes to numbers only here digitpat = re.compile('\d+') usr = re.sub(digitpat, '[value_count]', usr) # add database pointer pointer_vector, top_results, num_results = addDBPointer(bstate) # add booking pointer pointer_vector = addBookingPointer(bstate, pointer_vector) belief_summary = get_summary_bstate(bstate) usr_utt = [BOS] + usr.split() + [EOS] packed_val = {} packed_val['bs'] = belief_summary packed_val['db'] = pointer_vector packed_val['utt'] = self.corpus._sent2id(usr_utt) prepared_data['context'].append(packed_val) prepared_data['response']['bs'] = prepared_data['context'][-1]['bs'] prepared_data['response']['db'] = prepared_data['context'][-1]['db'] results = [Pack(context=prepared_data['context'], response=prepared_data['response'])] data_feed = prepare_batch_gen(results, self.config) outputs = self.model_predict(data_feed) if active_domain is not None and active_domain in num_results: num_results = num_results[active_domain] else: num_results = 0 if active_domain is not None and active_domain in top_results: top_results = {active_domain: top_results[active_domain]} else: top_results = {} state_with_history = deepcopy(bstate) state_with_history['history'] = deepcopy(state_history) response = self.populate_template( outputs, top_results, num_results, state_with_history) import pprint pprint.pprint("============") pprint.pprint('usr:'******'agent:') pprint.pprint(response) pprint.pprint("============") return response, active_domain
def reset(): self.prev_state = init_state()
def __init__(self, archive_file=DEFAULT_ARCHIVE_FILE, cuda_device=DEFAULT_CUDA_DEVICE, model_file=None): SysPolicy.__init__(self) if not os.path.isfile(archive_file): if not model_file: raise Exception("No model for LaRL is specified!") archive_file = cached_path(model_file) temp_path = tempfile.mkdtemp() zip_ref = zipfile.ZipFile(archive_file, 'r') zip_ref.extractall(temp_path) zip_ref.close() self.prev_state = init_state() self.prev_active_domain = None domain_name = 'object_division' domain_info = domain.get_domain(domain_name) data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data') train_data_path = os.path.join(data_path, 'norm-multi-woz', 'train_dials.json') if not os.path.exists(train_data_path): zipped_file = os.path.join(data_path, 'norm-multi-woz.zip') archive = zipfile.ZipFile(zipped_file, 'r') archive.extractall(data_path) norm_multiwoz_path = os.path.join(data_path, 'norm-multi-woz') with open(os.path.join(norm_multiwoz_path, 'input_lang.index2word.json')) as f: self.input_lang_index2word = json.load(f) with open(os.path.join(norm_multiwoz_path, 'input_lang.word2index.json')) as f: self.input_lang_word2index = json.load(f) with open(os.path.join(norm_multiwoz_path, 'output_lang.index2word.json')) as f: self.output_lang_index2word = json.load(f) with open(os.path.join(norm_multiwoz_path, 'output_lang.word2index.json')) as f: self.output_lang_word2index = json.load(f) config = Pack( seed=10, train_path=train_data_path, max_vocab_size=1000, last_n_model=5, max_utt_len=50, max_dec_len=50, backward_size=2, batch_size=1, use_gpu=True, op='adam', init_lr=0.001, l2_norm=1e-05, momentum=0.0, grad_clip=5.0, dropout=0.5, max_epoch=100, embed_size=100, num_layers=1, utt_rnn_cell='gru', utt_cell_size=300, bi_utt_cell=True, enc_use_attn=True, dec_use_attn=True, dec_rnn_cell='lstm', dec_cell_size=300, dec_attn_mode='cat', y_size=10, k_size=20, beta=0.001, simple_posterior=True, contextual_posterior=True, use_mi=False, use_pr=True, use_diversity=False, # beam_size=20, fix_batch=True, fix_train_batch=False, avg_type='word', print_step=300, ckpt_step=1416, improve_threshold=0.996, patient_increase=2.0, save_model=True, early_stop=False, gen_type='greedy', preview_batch_num=None, k=domain_info.input_length(), init_range=0.1, pretrain_folder='2019-09-20-21-43-06-sl_cat', forward_only=False ) config.use_gpu = config.use_gpu and torch.cuda.is_available() self.corpus = corpora_inference.NormMultiWozCorpus(config) self.model = SysPerfectBD2Cat(self.corpus, config) self.config = config if config.use_gpu: self.model.load_state_dict(torch.load( os.path.join(temp_path, 'larl_model/best-model'))) self.model.cuda() else: self.model.load_state_dict(torch.load(os.path.join( temp_path, 'larl_model/best-model'), map_location=lambda storage, loc: storage)) self.model.eval() self.dic = pickle.load( open(os.path.join(temp_path, 'larl_model/svdic.pkl'), 'rb'))
def init_session(self): self.state = init_state() if not self.param_restored: self.restore()
def init_session(self): self.state = init_state()
def __init__(self): Tracker.__init__(self) self.state = init_state() prefix = os.path.dirname(os.path.dirname(convlab.__file__)) self.value_dict = json.load( open(prefix + '/data/multiwoz/value_dict.json'))