def __init__(self):
Tracker.__init__(self)
self.state = init_state()
prefix = os.path.dirname(os.path.dirname(convlab.__file__))
self.value_dict = json.load(
open(prefix + '/data/multiwoz/value_dict.json'))
# changed part
# self.domain_exclude = ['hotel', 'Hotel']
# self.domain_exclude = ['Restaurant', 'Attraction', 'Taxi', 'Train', 'restaurant', 'attraction', 'taxi', 'train']
self.domain_exclude = []
def __init__(self, data_dir='data/mdbt'):
Tracker.__init__(self)
# data profile
self.data_dir = data_dir
self.validation_url = os.path.join(self.data_dir, 'data/validate.json')
self.word_vectors_url = os.path.join(
self.data_dir, 'word-vectors/paragram_300_sl999.txt')
self.training_url = os.path.join(self.data_dir, 'data/train.json')
self.ontology_url = os.path.join(self.data_dir, 'data/ontology.json')
self.testing_url = os.path.join(self.data_dir, 'data/test.json')
self.model_url = os.path.join(self.data_dir, 'models/model-1')
self.graph_url = os.path.join(self.data_dir, 'graphs/graph-1')
self.results_url = os.path.join(self.data_dir, 'results/log-1.txt')
self.kb_url = os.path.join(self.data_dir, 'data/') # not used
self.train_model_url = os.path.join(self.data_dir,
'train_models/model-1')
self.train_graph_url = os.path.join(self.data_dir,
'train_graph/graph-1')
print('Configuring MDBT model...')
self.word_vectors = load_word_vectors(self.word_vectors_url)
# Load the ontology and extract the feature vectors
self.ontology, self.ontology_vectors, self.slots = load_ontology(
self.ontology_url, self.word_vectors)
# Load and process the training data
self.dialogues, self.actual_dialogues = load_woz_data(
self.testing_url, self.word_vectors, self.ontology)
self.no_dialogues = len(self.dialogues)
self.model_variables = model_definition(self.ontology_vectors,
len(self.ontology),
self.slots,
num_hidden=None,
bidir=True,
net_type=None,
test=True,
dev='cpu')
self.state = init_state()
_config = tf.ConfigProto()
_config.gpu_options.allow_growth = True
_config.allow_soft_placement = True
self.sess = tf.Session(config=_config)
self.param_restored = False
self.det_dic = {}
for domain, dic in REF_USR_DA.items():
for key, value in dic.items():
assert '-' not in key
self.det_dic[key.lower()] = key + '-' + domain
self.det_dic[value.lower()] = key + '-' + domain
self.value_dict = json.load(
open(os.path.join(self.data_dir, '../multiwoz/value_dict.json')))
def __init__(self,
archive_file=DEFAULT_ARCHIVE_FILE,
cuda_device=DEFAULT_CUDA_DEVICE,
model_file=None):
if not os.path.isfile(archive_file):
if not model_file:
raise Exception("No model for MDRG is specified!")
archive_file = cached_path(model_file)
temp_path = tempfile.mkdtemp()
zip_ref = zipfile.ZipFile(archive_file, 'r')
zip_ref.extractall(temp_path)
zip_ref.close()
self.dic = pickle.load(
open(os.path.join(temp_path, 'mdrg/svdic.pkl'), 'rb'))
# Load dictionaries
with open(os.path.join(temp_path,
'mdrg/input_lang.index2word.json')) as f:
input_lang_index2word = json.load(f)
with open(os.path.join(temp_path,
'mdrg/input_lang.word2index.json')) as f:
input_lang_word2index = json.load(f)
with open(os.path.join(temp_path,
'mdrg/output_lang.index2word.json')) as f:
output_lang_index2word = json.load(f)
with open(os.path.join(temp_path,
'mdrg/output_lang.word2index.json')) as f:
output_lang_word2index = json.load(f)
self.response_model = Model(args, input_lang_index2word,
output_lang_index2word,
input_lang_word2index,
output_lang_word2index)
self.response_model.loadModel(os.path.join(temp_path, 'mdrg/mdrg'))
shutil.rmtree(temp_path)
self.prev_state = init_state()
self.prev_active_domain = None
def predict(self, state):
"""
Predict the dialog act of a natural language utterance and apply error model.
Args:
utterance (str): A natural language utterance.
Returns:
output (dict): The dialog act of utterance.
"""
state_vector = self.state_encoder.encode(state)
instance = self.dataset_reader.text_to_instance(state_vector)
outputs = self.model.forward_on_instance(instance)
dialacts = self.action_decoder.decode(outputs["actions"], state)
if dialacts == {'general-bye': [['none', 'none']]}:
outputs["probs"][outputs["actions"]] = 0
outputs["actions"] = np.argmax(outputs["probs"])
dialacts = self.action_decoder.decode(outputs["actions"], state)
if state == init_state():
dialacts = {}
return dialacts
def init_session(self):
self.state = init_state()
if not self.param_restored:
self.load_param()
self.param_restored = True
self.model.eval()
def predict(self, state):
"""
Predict the dialog act of a natural language utterance and apply error model.
Args:
utterance (str): A natural language utterance.
Returns:
output (dict): The dialog act of utterance.
"""
state_vector = self.state_encoder.encode(state)
instance = self.dataset_reader.text_to_instance(state_vector)
outputs = self.model.forward_on_instance(instance)
dialacts = self.action_decoder.decode(outputs["actions"], state)
if dialacts == {'general-bye': [['none', 'none']]}:
outputs["probs"][outputs["actions"]] = 0
outputs["actions"] = np.argmax(outputs["probs"])
dialacts = self.action_decoder.decode(outputs["actions"], state)
if state == init_state():
dialacts = {}
return dialacts
if __name__ == "__main__":
policy = VanillaMLEPolicy()
pprint(policy.predict(init_state()))
def predict_response(self, state):
history = []
for i in range(len(state['history'])):
for j in range(len(state['history'][i])):
history.append(state['history'][i][j])
e_idx = len(history)
s_idx = max(0, e_idx - self.config.backward_size)
context = []
for turn in history[s_idx: e_idx]:
# turn = pad_to(config.max_utt_len, turn, do_pad=False)
context.append(turn)
if len(state['history']) == 1:
self.prev_state = init_state()
prepared_data = {}
prepared_data['context'] = []
prepared_data['response'] = {}
prev_bstate = deepcopy(self.prev_state['belief_state'])
state_history = state['history']
bstate = deepcopy(state['belief_state'])
# mark_not_mentioned(prev_state)
active_domain = self.get_active_domain(
self.prev_active_domain, prev_bstate, bstate)
domain_mark_not_mentioned(bstate, active_domain)
top_results, num_results = None, None
for usr in context:
words = usr.split()
usr = delexicalize.delexicalise(' '.join(words), self.dic)
# parsing reference number GIVEN belief state
usr = delexicaliseReferenceNumber(usr, bstate)
# changes to numbers only here
digitpat = re.compile('\d+')
usr = re.sub(digitpat, '[value_count]', usr)
# add database pointer
pointer_vector, top_results, num_results = addDBPointer(bstate)
# add booking pointer
pointer_vector = addBookingPointer(bstate, pointer_vector)
belief_summary = get_summary_bstate(bstate)
usr_utt = [BOS] + usr.split() + [EOS]
packed_val = {}
packed_val['bs'] = belief_summary
packed_val['db'] = pointer_vector
packed_val['utt'] = self.corpus._sent2id(usr_utt)
prepared_data['context'].append(packed_val)
prepared_data['response']['bs'] = prepared_data['context'][-1]['bs']
prepared_data['response']['db'] = prepared_data['context'][-1]['db']
results = [Pack(context=prepared_data['context'],
response=prepared_data['response'])]
data_feed = prepare_batch_gen(results, self.config)
outputs = self.model_predict(data_feed)
if active_domain is not None and active_domain in num_results:
num_results = num_results[active_domain]
else:
num_results = 0
if active_domain is not None and active_domain in top_results:
top_results = {active_domain: top_results[active_domain]}
else:
top_results = {}
state_with_history = deepcopy(bstate)
state_with_history['history'] = deepcopy(state_history)
response = self.populate_template(
outputs, top_results, num_results, state_with_history)
import pprint
pprint.pprint("============")
pprint.pprint('usr:'******'agent:')
pprint.pprint(response)
pprint.pprint("============")
return response, active_domain
def reset():
self.prev_state = init_state()
def __init__(self,
archive_file=DEFAULT_ARCHIVE_FILE,
cuda_device=DEFAULT_CUDA_DEVICE,
model_file=None):
SysPolicy.__init__(self)
if not os.path.isfile(archive_file):
if not model_file:
raise Exception("No model for LaRL is specified!")
archive_file = cached_path(model_file)
temp_path = tempfile.mkdtemp()
zip_ref = zipfile.ZipFile(archive_file, 'r')
zip_ref.extractall(temp_path)
zip_ref.close()
self.prev_state = init_state()
self.prev_active_domain = None
domain_name = 'object_division'
domain_info = domain.get_domain(domain_name)
data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data')
train_data_path = os.path.join(data_path, 'norm-multi-woz', 'train_dials.json')
if not os.path.exists(train_data_path):
zipped_file = os.path.join(data_path, 'norm-multi-woz.zip')
archive = zipfile.ZipFile(zipped_file, 'r')
archive.extractall(data_path)
norm_multiwoz_path = os.path.join(data_path, 'norm-multi-woz')
with open(os.path.join(norm_multiwoz_path, 'input_lang.index2word.json')) as f:
self.input_lang_index2word = json.load(f)
with open(os.path.join(norm_multiwoz_path, 'input_lang.word2index.json')) as f:
self.input_lang_word2index = json.load(f)
with open(os.path.join(norm_multiwoz_path, 'output_lang.index2word.json')) as f:
self.output_lang_index2word = json.load(f)
with open(os.path.join(norm_multiwoz_path, 'output_lang.word2index.json')) as f:
self.output_lang_word2index = json.load(f)
config = Pack(
seed=10,
train_path=train_data_path,
max_vocab_size=1000,
last_n_model=5,
max_utt_len=50,
max_dec_len=50,
backward_size=2,
batch_size=1,
use_gpu=True,
op='adam',
init_lr=0.001,
l2_norm=1e-05,
momentum=0.0,
grad_clip=5.0,
dropout=0.5,
max_epoch=100,
embed_size=100,
num_layers=1,
utt_rnn_cell='gru',
utt_cell_size=300,
bi_utt_cell=True,
enc_use_attn=True,
dec_use_attn=True,
dec_rnn_cell='lstm',
dec_cell_size=300,
dec_attn_mode='cat',
y_size=10,
k_size=20,
beta=0.001,
simple_posterior=True,
contextual_posterior=True,
use_mi=False,
use_pr=True,
use_diversity=False,
#
beam_size=20,
fix_batch=True,
fix_train_batch=False,
avg_type='word',
print_step=300,
ckpt_step=1416,
improve_threshold=0.996,
patient_increase=2.0,
save_model=True,
early_stop=False,
gen_type='greedy',
preview_batch_num=None,
k=domain_info.input_length(),
init_range=0.1,
pretrain_folder='2019-09-20-21-43-06-sl_cat',
forward_only=False
)
config.use_gpu = config.use_gpu and torch.cuda.is_available()
self.corpus = corpora_inference.NormMultiWozCorpus(config)
self.model = SysPerfectBD2Cat(self.corpus, config)
self.config = config
if config.use_gpu:
self.model.load_state_dict(torch.load(
os.path.join(temp_path, 'larl_model/best-model')))
self.model.cuda()
else:
self.model.load_state_dict(torch.load(os.path.join(
temp_path, 'larl_model/best-model'), map_location=lambda storage, loc: storage))
self.model.eval()
self.dic = pickle.load(
open(os.path.join(temp_path, 'larl_model/svdic.pkl'), 'rb'))
def init_session(self):
self.state = init_state()
if not self.param_restored:
self.restore()
def init_session(self):
self.state = init_state()
def __init__(self):
Tracker.__init__(self)
self.state = init_state()
prefix = os.path.dirname(os.path.dirname(convlab.__file__))
self.value_dict = json.load(
open(prefix + '/data/multiwoz/value_dict.json'))
