def _build_data(self, root_dir, processed_dir):
self.data = {}
data_loader = ActPolicyDataloader(
dataset_dataloader=MultiWOZDataloader())
for part in ['train', 'val', 'test']:
self.data[part] = []
raw_data = data_loader.load_data(data_key=part,
role='system')[part]
for belief_state, context_dialog_act, terminal, dialog_act in \
zip(raw_data['belief_state'], raw_data['context_dialog_act'], raw_data['terminal'], raw_data['dialog_act']):
state = default_state()
state['belief_state'] = belief_state
state['user_action'] = context_dialog_act[-1]
state['system_action'] = context_dialog_act[-2] if len(
context_dialog_act) > 1 else {}
state['terminal'] = terminal
action = dialog_act
self.data[part].append([
self.vector.state_vectorize(state),
self.vector.action_vectorize(action)
])
os.makedirs(processed_dir)
for part in ['train', 'val', 'test']:
with open(os.path.join(processed_dir, '{}.pkl'.format(part)),
'wb') as f:
pickle.dump(self.data[part], f)
def fake_state():
user_action = {
'Hotel-Request': [['Name', '?']],
'Train-Inform': [['Day', 'don\'t care']]
}
from tatk.util.multiwoz.state import default_state
init_belief_state = default_state()['belief_state']
kb_results = [None, None]
kb_results[0] = {
'name': 'xxx_train',
'day': 'tuesday',
'dest': 'cam',
'phone': '123-3333',
'area': 'south'
}
kb_results[1] = {
'name': 'xxx_train',
'day': 'tuesday',
'dest': 'cam',
'phone': '123-3333',
'area': 'north'
}
state = {
'user_action': user_action,
'belief_state': init_belief_state,
'kb_results_dict': kb_results,
'hotel-request': [['phone']]
}
'''
state = {'user_action': dict(),
'belief_state: dict(),
'kb_results_dict': kb_results
}
'''
return state
def main():
s = default_state()
s['history'] = [['null', 'I want a korean restaurant in the centre.']]
s['belief_state']['attraction']['semi']['area'] = 'centre'
s['belief_state']['restaurant']['semi']['area'] = 'centre'
s['belief_state']['restaurant']['semi']['food'] = 'korean'
testPolicy = MDRGWordPolicy()
print(testPolicy.predict(s))
def main():
s = default_state()
s['history'] = [['null', 'I want a chinese restaurant']]
# s['belief_state']['attraction']['semi']['area'] = 'centre'
s['belief_state']['restaurant']['semi']['area'] = 'south'
s['belief_state']['restaurant']['semi']['food'] = 'mexican'
testPolicy = MDRGWordPolicy()
print(s)
print(testPolicy.predict(s))
def __init__(self):
DST.__init__(self)
self.state = default_state()
path = os.path.dirname(
os.path.dirname(
os.path.dirname(
os.path.dirname(os.path.dirname(
os.path.abspath(__file__))))))
path = os.path.join(path, 'data/multiwoz/value_dict.json')
self.value_dict = json.load(open(path))
def __init__(self, num=1):
parser = argparse.ArgumentParser(description='S2S')
parser.add_argument('--no_cuda', type=util.str2bool, nargs='?', const=True, default=True,
help='enables CUDA training')
parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)')
parser.add_argument('--no_models', type=int, default=20, help='how many models to evaluate')
parser.add_argument('--original', type=str, default='model/model/', help='Original path.')
parser.add_argument('--dropout', type=float, default=0.0)
parser.add_argument('--use_emb', type=str, default='False')
parser.add_argument('--beam_width', type=int, default=10, help='Beam width used in beamsearch')
parser.add_argument('--write_n_best', type=util.str2bool, nargs='?', const=True, default=False,
help='Write n-best list (n=beam_width)')
parser.add_argument('--model_path', type=str, default='model/model/translate.ckpt',
help='Path to a specific model checkpoint.')
parser.add_argument('--model_dir', type=str, default='data/multi-woz/model/model/')
parser.add_argument('--model_name', type=str, default='translate.ckpt')
parser.add_argument('--valid_output', type=str, default='model/data/val_dials/',
help='Validation Decoding output dir path')
parser.add_argument('--decode_output', type=str, default='model/data/test_dials/',
help='Decoding output dir path')
args = parser.parse_args([])
args.cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
self.device = torch.device("cuda" if args.cuda else "cpu")
with open(os.path.join(os.path.dirname(__file__), args.model_path + '.config'), 'r') as f:
add_args = json.load(f)
# print(add_args)
for k, v in add_args.items():
setattr(args, k, v)
# print(args)
args.mode = 'test'
args.load_param = True
args.dropout = 0.0
assert args.dropout == 0.0
# Start going through models
args.original = args.model_path
args.model_path = args.original
self.model = loadModel(num, args)
self.dial = {"cur": {"log": []}}
self.prev_state = default_state()
self.prev_active_domain = None
def __init__(self, ontology_vectors, ontology, slots, data_dir):
Tracker.__init__(self)
# data profile
self.data_dir = data_dir
self.validation_url = os.path.join(self.data_dir, 'data/validate.json')
self.word_vectors_url = os.path.join(
self.data_dir, 'word-vectors/paragram_300_sl999.txt')
self.training_url = os.path.join(self.data_dir, 'data/train.json')
self.ontology_url = os.path.join(self.data_dir, 'data/ontology.json')
self.testing_url = os.path.join(self.data_dir, 'data/test.json')
self.model_url = os.path.join(self.data_dir, 'models/model-1')
self.graph_url = os.path.join(self.data_dir, 'graphs/graph-1')
self.results_url = os.path.join(self.data_dir, 'results/log-1.txt')
self.kb_url = os.path.join(self.data_dir, 'data/') # not used
self.train_model_url = os.path.join(self.data_dir,
'train_models/model-1')
self.train_graph_url = os.path.join(self.data_dir,
'train_graph/graph-1')
self.model_variables = model_definition(ontology_vectors,
len(ontology),
slots,
num_hidden=None,
bidir=True,
net_type=None,
test=True,
dev='cpu')
self.state = default_state()
_config = tf.ConfigProto()
_config.gpu_options.allow_growth = True
_config.allow_soft_placement = True
self.sess = tf.Session(config=_config)
self.param_restored = False
self.det_dic = {}
for domain, dic in REF_USR_DA.items():
for key, value in dic.items():
assert '-' not in key
self.det_dic[key.lower()] = key + '-' + domain
self.det_dic[value.lower()] = key + '-' + domain
def parent_dir(path, time=1):
for _ in range(time):
path = os.path.dirname(path)
return path
root_dir = parent_dir(os.path.abspath(__file__), 4)
self.value_dict = json.load(
open(os.path.join(root_dir, 'data/multiwoz/value_dict.json')))
def generate_dict(self):
"""
init the dict for mapping state/action into vector
"""
self.act2vec = dict((a, i) for i, a in enumerate(self.da_voc))
self.vec2act = dict((v, k) for k, v in self.act2vec.items())
self.da_dim = len(self.da_voc)
self.opp2vec = dict((a, i) for i, a in enumerate(self.da_voc_opp))
self.da_opp_dim = len(self.da_voc_opp)
self.belief_state_dim = 0
for domain in self.belief_domains:
for slot, value in default_state()['belief_state'][domain.lower()]['semi'].items():
self.belief_state_dim += 1
self.state_dim = self.da_opp_dim + self.da_dim + self.belief_state_dim + \
len(self.db_domains) + 6 * len(self.db_domains) + 1
def _build_data(self, root_dir, processed_dir):
self.data = {}
for part in ['train', 'val', 'test']:
self.data[part] = []
raw_data = ActPolicyDataloader(data_key=part, role='sys')
for turn in raw_data:
state = default_state()
state['user_action'] = turn['context_dialog_act'][-1]
state['system_action'] = turn['context_dialog_act'][-2]
state['terminal'] = turn['terminal']
action = turn['dialog_act']
self.data[part].append([
self.vector.state_vectorize(state),
self.vector.action_vectorize(action)
])
os.makedirs(processed_dir)
for part in ['train', 'val', 'test']:
with open(os.path.join(processed_dir, '{}.pkl'.format(part)),
'wb') as f:
pickle.dump(self.data[part], f)
def setup_class(cls):
# domain_slots will be used for checking request_state
cls.domain_slots = {}
for key, value in REF_SYS_DA.items():
new_key = key.lower()
new_value = set(value.values())
cls.domain_slots[new_key] = new_value
cls.default_belief_state = default_state()['belief_state']
cls.usr_acts = [
{
"Hotel-Inform": [["Area", "east"], ["Stars", "4"]]
},
{
"Hotel-Inform": [["Parking", "yes"], ["Internet", "yes"]]
},
{},
{
"Hotel-Inform": [["Day", "friday"]],
"Hotel-Request": [["Ref", "?"]]
},
{
"Train-Inform": [["Dest", "bishops stortford"],
["Day", "friday"], ["Depart", "cambridge"]]
},
{
"Train-Inform": [["Arrive", "19:45"]]
},
{
"Train-Request": [["Leave", "?"], ["Time", "?"],
["Ticket", "?"]]
},
{
"Hotel-Inform": [["Stay", "4"], ["Day", "monday"],
["People", "3"]]
},
]
def _build_data(self, root_dir, processed_dir):
raw_data = {}
for part in ['train', 'val', 'test']:
archive = zipfile.ZipFile(
os.path.join(root_dir,
'data/multiwoz/{}.json.zip'.format(part)), 'r')
with archive.open('{}.json'.format(part), 'r') as f:
raw_data[part] = json.load(f)
self.data = {}
for part in ['train', 'val', 'test']:
self.data[part] = []
for key in raw_data[part]:
sess = raw_data[part][key]['log']
state = default_state()
action = {}
for i, turn in enumerate(sess):
if i % 2 == 0:
state['user_action'] = turn['dialog_act']
if i + 2 == len(sess):
state['terminal'] = True
else:
state['belief_state'] = turn['metadata']
action = turn['dialog_act']
self.data[part].append([
self.vector.state_vectorize(state),
self.vector.action_vectorize(action)
])
state['system_action'] = turn['dialog_act']
os.makedirs(processed_dir)
for part in ['train', 'val', 'test']:
with open(os.path.join(processed_dir, '{}.pkl'.format(part)),
'wb') as f:
pickle.dump(self.data[part], f)
def update(self, user_act=None):
"""Update the dialog state."""
if type(user_act) is not str:
raise Exception(
'Expected user_act to be <class \'str\'> type, but get {}.'.
format(type(user_act)))
prev_state = self.state
if not os.path.exists(os.path.join(self.data_dir, "results")):
os.makedirs(os.path.join(self.data_dir, "results"))
global train_batch_size
model_variables = self.model_variables
(user, sys_res, no_turns, user_uttr_len, sys_uttr_len, labels,
domain_labels, domain_accuracy, slot_accuracy, value_accuracy,
value_f1, train_step, keep_prob, predictions, true_predictions,
[y, _]) = model_variables
# generate fake dialogue based on history (this os to reuse the original MDBT code)
# actual_history = prev_state['history'] # [[sys, user], [sys, user], ...]
actual_history = copy.deepcopy(
prev_state['history']) # [[sys, user], [sys, user], ...]
actual_history = [['null']]
actual_history[-1].append(user_act)
actual_history = self.normalize_history(actual_history)
if len(actual_history) == 0:
actual_history = [[
'', user_act if len(user_act) > 0 else 'fake user act'
]]
fake_dialogue = {}
turn_no = 0
for _sys, _user in actual_history:
turn = {}
turn['system'] = _sys
fake_user = {}
fake_user['text'] = _user
fake_user['belief_state'] = default_state()['belief_state']
turn['user'] = fake_user
key = str(turn_no)
fake_dialogue[key] = turn
turn_no += 1
context, actual_context = process_history([fake_dialogue],
self.word_vectors,
self.ontology)
batch_user, batch_sys, batch_labels, batch_domain_labels, batch_user_uttr_len, batch_sys_uttr_len, \
batch_no_turns = generate_batch(context, 0, 1, len(self.ontology)) # old feature
# run model
[pred, y_pred] = self.sess.run(
[predictions, y],
feed_dict={
user: batch_user,
sys_res: batch_sys,
labels: batch_labels,
domain_labels: batch_domain_labels,
user_uttr_len: batch_user_uttr_len,
sys_uttr_len: batch_sys_uttr_len,
no_turns: batch_no_turns,
keep_prob: 1.0
})
# convert to str output
dialgs, _, _ = track_dialogue(actual_context, self.ontology, pred,
y_pred)
assert len(dialgs) >= 1
last_turn = dialgs[0][-1]
predictions = last_turn['prediction']
new_belief_state = copy.deepcopy(prev_state['belief_state'])
# updaet belief state
for item in predictions:
item = item.lower()
domain, slot, value = item.strip().split('-')
value = value[::-1].split(':', 1)[1][::-1]
if slot == 'price range':
slot = 'pricerange'
if slot not in ['name', 'book']:
if domain not in new_belief_state:
raise Exception(
'Error: domain <{}> not in belief state'.format(
domain))
slot = REF_SYS_DA[domain.capitalize()].get(slot, slot)
assert 'semi' in new_belief_state[domain]
assert 'book' in new_belief_state[domain]
if 'book' in slot:
assert slot.startswith('book ')
slot = slot.strip().split()[1]
domain_dic = new_belief_state[domain]
if slot in domain_dic['semi']:
new_belief_state[domain]['semi'][slot] = normalize_value(
self.value_dict, domain, slot, value)
elif slot in domain_dic['book']:
new_belief_state[domain]['book'][slot] = value
elif slot.lower() in domain_dic['book']:
new_belief_state[domain]['book'][slot.lower()] = value
else:
with open('mdbt_unknown_slot.log', 'a+') as f:
f.write(
'unknown slot name <{}> with value <{}> of domain <{}>\nitem: {}\n\n'
.format(slot, value, domain, item))
new_request_state = copy.deepcopy(prev_state['request_state'])
# update request_state
user_request_slot = self.detect_requestable_slots(user_act)
for domain in user_request_slot:
for key in user_request_slot[domain]:
if domain not in new_request_state:
new_request_state[domain] = {}
if key not in new_request_state[domain]:
new_request_state[domain][key] = user_request_slot[domain][
key]
# update state
new_state = copy.deepcopy(dict(prev_state))
new_state['belief_state'] = new_belief_state
new_state['request_state'] = new_request_state
self.state = new_state
return self.state
def init_session(self):
self.state = default_state()
if not self.param_restored:
self.restore()
def init_session(self):
"""Initialize ``self.state`` with a default state, which ``tatk.util.multiwoz.state.default_state`` returns."""
self.state = default_state()
