def __init__(self, ontology_vectors, ontology, slots, data_dir):
DST.__init__(self)
# data profile
self.data_dir = data_dir
self.validation_url = os.path.join(self.data_dir, 'data/validate.json')
self.word_vectors_url = os.path.join(
self.data_dir, 'word-vectors/paragram_300_sl999.txt')
self.training_url = os.path.join(self.data_dir, 'data/train.json')
self.ontology_url = os.path.join(self.data_dir, 'data/ontology.json')
self.testing_url = os.path.join(self.data_dir, 'data/test.json')
self.model_url = os.path.join(self.data_dir, 'models/model-1')
self.graph_url = os.path.join(self.data_dir, 'graphs/graph-1')
self.results_url = os.path.join(self.data_dir, 'results/log-1.txt')
self.kb_url = os.path.join(self.data_dir, 'data/') # not used
self.train_model_url = os.path.join(self.data_dir,
'train_models/model-1')
self.train_graph_url = os.path.join(self.data_dir,
'train_graph/graph-1')
self.model_variables = model_definition(ontology_vectors,
len(ontology),
slots,
num_hidden=None,
bidir=True,
net_type=None,
test=True,
dev='cpu')
self.state = default_state()
_config = tf.ConfigProto()
_config.gpu_options.allow_growth = True
_config.allow_soft_placement = True
self.sess = tf.Session(config=_config)
self.param_restored = False
self.det_dic = {}
for domain, dic in REF_USR_DA.items():
for key, value in dic.items():
assert '-' not in key
self.det_dic[key.lower()] = key + '-' + domain
self.det_dic[value.lower()] = key + '-' + domain
def parent_dir(path, time=1):
for _ in range(time):
path = os.path.dirname(path)
return path
root_dir = parent_dir(os.path.abspath(__file__), 4)
self.value_dict = json.load(
open(os.path.join(root_dir, 'data/multiwoz/value_dict.json')))
def __init__(
self,
model_file='https://convlab.blob.core.windows.net/convlab-2/sumbt.tar.gz',
arg_path=os.path.join(SUMBT_PATH, 'config.json'),
eval_slots=multiwoz_slot_list_en):
DST.__init__(self)
# if not os.path.exists(data_dir):
# if model_file == '':
# raise Exception(
# 'Please provide remote model file path in config')
# resp = urllib.request.urlretrieve(model_file)[0]
# temp_file = tarfile.open(resp)
# temp_file.extractall('data')
# assert os.path.exists(data_dir)
args = json.load(open(arg_path))
args = SimpleNamespace(**args)
self.args = args
data_dir = os.path.join(ROOT_PATH, args.data_dir)
if args.lang == 'zh':
convert_to_glue_format = convert_to_glue_format_zh
default_state = default_state_zh
processor = ProcessorZh(args)
eval_slots = multiwoz_slot_list_zh
else:
convert_to_glue_format = convert_to_glue_format_en
default_state = default_state_en
processor = ProcessorEn(args)
eval_slots = multiwoz_slot_list_en
self.processor = processor
label_list = processor.get_labels()
num_labels = [len(labels) for labels in label_list
] # number of slot-values in each slot-type
# tokenizer
self.tokenizer = AutoTokenizer.from_pretrained(
args.bert_model_name, cache_dir=args.bert_model_cache_dir)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
self.device = torch.device("cuda" if USE_CUDA else "cpu")
self.sumbt_model = BeliefTracker(args, num_labels, self.device)
if USE_CUDA and N_GPU > 1:
self.sumbt_model = torch.nn.DataParallel(self.sumbt_model)
if args.fp16:
self.sumbt_model.half()
self.sumbt_model.to(self.device)
## Get slot-value embeddings
self.label_token_ids, self.label_len = [], []
for labels in label_list:
token_ids, lens = get_label_embedding(labels,
args.max_label_length,
self.tokenizer, self.device)
self.label_token_ids.append(token_ids)
self.label_len.append(lens)
self.label_map = [{label: i
for i, label in enumerate(labels)}
for labels in label_list]
self.label_map_inv = [{i: label
for i, label in enumerate(labels)}
for labels in label_list]
self.label_list = label_list
self.target_slot = processor.target_slot
## Get domain-slot-type embeddings
self.slot_token_ids, self.slot_len = \
get_label_embedding(processor.target_slot, args.max_label_length, self.tokenizer, self.device)
self.args = args
self.state = default_state()
self.param_restored = False
if USE_CUDA and N_GPU == 1:
self.sumbt_model.initialize_slot_value_lookup(
self.label_token_ids, self.slot_token_ids)
elif USE_CUDA and N_GPU > 1:
self.sumbt_model.module.initialize_slot_value_lookup(
self.label_token_ids, self.slot_token_ids)
self.det_dic = {}
for domain, dic in REF_USR_DA.items():
for key, value in dic.items():
assert '-' not in key
self.det_dic[key.lower()] = key + '-' + domain
self.det_dic[value.lower()] = key + '-' + domain
self.cached_res = {}
convert_to_glue_format(os.path.join(ROOT_PATH, args.data_dir),
SUMBT_PATH, args)
if not os.path.isdir(os.path.join(SUMBT_PATH, args.output_dir)):
os.makedirs(os.path.join(SUMBT_PATH, args.output_dir))
self.train_examples = processor.get_train_examples(os.path.join(
SUMBT_PATH, args.tmp_data_dir),
accumulation=False)
self.dev_examples = processor.get_dev_examples(os.path.join(
SUMBT_PATH, args.tmp_data_dir),
accumulation=False)
self.test_examples = processor.get_test_examples(os.path.join(
SUMBT_PATH, args.tmp_data_dir),
accumulation=False)
self.eval_slots = eval_slots
def __init__(self, argpath=None, mode='en', training=False):
super(MultiWOZTRADE, self).__init__()
if argpath is None:
argpath = 'model/TRADE-multiwozdst/HDD400BSZ32DR0.2ACC-0.3591'
args['path'] = argpath
self.init_session()
self.multiwoz_root = os.path.dirname(os.path.abspath(__file__))
self.download_model()
self.download_data()
directory = argpath.split("/")
HDD = directory[2].split('HDD')[1].split('BSZ')[0]
decoder = directory[1].split('-')[0]
BSZ = int(args['batch']) if args['batch'] else int(
directory[2].split('BSZ')[1].split('DR')[0])
args["decoder"] = decoder
def parent_dir(path, time=1):
for _ in range(time):
path = os.path.dirname(path)
return path
root_dir = parent_dir(os.path.abspath(__file__), 5)
# load ontology variables
self.value_dict = json.load(
open(os.path.join(root_dir, 'data/multiwoz/value_dict.json')))
self.det_dic = {}
for domain, dic in REF_USR_DA.items():
for key, value in dic.items():
assert '-' not in key
self.det_dic[key.lower()] = key + '-' + domain
self.det_dic[value.lower()] = key + '-' + domain
train, dev, test, test_special, lang, SLOTS_LIST, gating_dict, max_word = prepare_data_seq(
False, 'dst', False, batch_size=4)
self.slot_list = SLOTS_LIST
self.test_set = test
hidden_size = int(HDD)
lang = lang
path = argpath
lr = 0
task = 'dst'
dropout = 0
slots = SLOTS_LIST
gating_dict = gating_dict
nb_train_vocab = max_word
self.mode = mode
self.name = "TRADE"
self.task = task
self.hidden_size = hidden_size
self.lang = lang[0]
self.mem_lang = lang[1]
self.lr = lr
self.dropout = dropout
self.slots = slots[0]
self.slot_temp = slots[2]
self.gating_dict = gating_dict
self.nb_gate = len(gating_dict)
self.cross_entorpy = nn.CrossEntropyLoss()
self.init_run_vars(training)
self.encoder = EncoderRNN(self.lang.n_words,
hidden_size,
self.dropout,
mode=mode)
self.decoder = Generator(self.lang, self.encoder.embedding,
self.lang.n_words, hidden_size, self.dropout,
self.slots, self.nb_gate)
model_root = os.path.dirname(os.path.abspath(__file__))
if path:
path = os.path.join(model_root, path)
map_loc = 'cpu' if not torch.cuda.is_available() else None
self.encoder.load_state_dict(
torch.load(str(path) + '/enc.pr', map_location=map_loc))
self.decoder.load_state_dict(
torch.load(str(path) + '/dec.pr', map_location=map_loc))
if torch.cuda.is_available():
self.encoder = self.encoder.cuda()
self.decoder = self.decoder.cuda()
# Initialize optimizers and criterion
self.optimizer = optim.Adam(self.parameters(), lr=lr)
self.scheduler = lr_scheduler.ReduceLROnPlateau(self.optimizer,
mode='max',
factor=0.5,
patience=1,
min_lr=0.0001,
verbose=True)
self.reset()
if USE_CUDA:
self.encoder.cuda()
self.decoder.cuda()