def __init__(self):
super(TradeDST, self).__init__()
# load config
common_config_path = os.path.join(get_config_path(),
TradeDST.common_config_name)
common_config = json.load(open(common_config_path))
model_config_path = os.path.join(get_config_path(),
TradeDST.model_config_name)
model_config = json.load(open(model_config_path))
model_config.update(common_config)
self.model_config = model_config
self.model_config['data_path'] = os.path.join(
get_data_path(), 'crosswoz/dst_trade_data')
self.model_config['n_gpus'] = 0 if self.model_config[
'device'] == 'cpu' else torch.cuda.device_count()
self.model_config['device'] = torch.device(self.model_config['device'])
if model_config['load_embedding']:
model_config['hidden_size'] = 300
# download data
for model_key, url in TradeDST.model_urls.items():
dst = os.path.join(self.model_config['data_path'], model_key)
if model_key.endswith('pth'):
file_name = 'trained_model_path'
elif model_key.endswith('pkl'):
file_name = model_key.rsplit('-', maxsplit=1)[0]
else:
file_name = model_key.split('.')[0] # ontology
self.model_config[file_name] = dst
if not os.path.exists(dst) or not self.model_config['use_cache']:
download_from_url(url, dst)
# load date & model
ontology = json.load(
open(self.model_config['ontology'], 'r', encoding='utf8'))
self.all_slots = get_slot_information(ontology)
self.gate2id = {'ptr': 0, 'none': 1}
self.id2gate = {id_: gate for gate, id_ in self.gate2id.items()}
self.lang = pickle.load(open(self.model_config['lang'], 'rb'))
self.mem_lang = pickle.load(open(self.model_config['mem-lang'], 'rb'))
model = Trade(
lang=self.lang,
vocab_size=len(self.lang.index2word),
hidden_size=self.model_config['hidden_size'],
dropout=self.model_config['dropout'],
num_encoder_layers=self.model_config['num_encoder_layers'],
num_decoder_layers=self.model_config['num_decoder_layers'],
pad_id=self.model_config['pad_id'],
slots=self.all_slots,
num_gates=len(self.gate2id),
unk_mask=self.model_config['unk_mask'])
model.load_state_dict(
torch.load(self.model_config['trained_model_path']))
self.model = model.to(self.model_config['device']).eval()
print(f'>>> {self.model_config["trained_model_path"]} loaded ...')
self.state = default_state()
print('>>> State initialized ...')
def prepare_data_seq(config):
eval_batch = (config["eval_batch_size"]
if config["eval_batch_size"] else config["batch_size"])
train_file_path = config["train_dials"]
dev_file_path = config["dev_dials"]
test_file_path = config["test_dials"]
ontology_file_path = config["ontology"]
# load domain-slot pairs from ontology
ontology = json.load(open(ontology_file_path, "r", encoding="utf8"))
slots = get_slot_information(ontology)
gating_dict = {"ptr": 0, "none": 1}
# Vocabulary
lang_name = "lang-all.pkl" if config["all_vocab"] else "lang-train.pkl"
mem_lang_name = "mem-lang-all.pkl" if config[
"all_vocab"] else "mem-lang-train.pkl"
if config["debug"]:
lang_name = "debug-" + lang_name
mem_lang_name = "debug-" + mem_lang_name
lang_file_path = os.path.join(config["data_path"], lang_name)
mem_lang_file_path = os.path.join(config["data_path"], mem_lang_name)
load_lang = False
if (os.path.exists(lang_file_path) and
os.path.exists(mem_lang_file_path)) and not config["clean_cache"]:
print("Loading saved lang files...")
load_lang = True
with open(lang_file_path, "rb") as f:
lang = pickle.load(f)
with open(mem_lang_file_path, "rb") as f:
mem_lang = pickle.load(f)
else:
lang, mem_lang = Lang(config), Lang(config)
# 都包含了 ontology 中的 domain 和 slot，之后分别包含 utterance 和 domain-slot-value
lang.index_words(slots, "slot")
mem_lang.index_words(slots, "slot")
# 生成 dataloader
pair_train, train_max_len = read_langs(train_file_path, gating_dict, slots,
"train", lang, mem_lang, load_lang,
config)
train_loader = get_seq(
pair_train,
lang,
mem_lang,
config["batch_size"],
config["n_gpus"],
shuffle=True,
config=config,
)
train_vocab_size = lang.n_words
pair_dev, dev_max_len = read_langs(dev_file_path, gating_dict, slots,
"dev", lang, mem_lang, load_lang,
config)
dev_loader = get_seq(
pair_dev,
lang,
mem_lang,
eval_batch,
config["n_gpus"],
shuffle=False,
config=config,
)
pair_test, test_max_len = read_langs(test_file_path, gating_dict, slots,
"tests", lang, mem_lang, load_lang,
config)
test_loader = get_seq(
pair_test,
lang,
mem_lang,
eval_batch,
config["n_gpus"],
shuffle=False,
config=config,
)
# 保存中间数据
if (not (os.path.exists(lang_file_path)
and os.path.exists(mem_lang_file_path)) or config["clean_cache"]):
print("Dumping lang files...")
with open(lang_file_path, "wb") as f:
pickle.dump(lang, f)
with open(mem_lang_file_path, "wb") as f:
pickle.dump(mem_lang, f)
emb_dump_path = os.path.join(config["data_path"],
f"emb{len(lang.index2word)}")
if (not os.path.exists(emb_dump_path)
or config["clean_cache"]) and config["load_embedding"]:
dump_pretrained_emb(config["orig_pretrained_embedding"],
lang.index2word, emb_dump_path)
max_dialogue_history_length = max(train_max_len, dev_max_len,
test_max_len) + 1
print("Read %s pairs train" % len(pair_train))
print("Read %s pairs dev" % len(pair_dev))
print("Read %s pairs tests" % len(pair_test))
print("Vocab_size: %s " % lang.n_words)
print("Vocab_size Training %s" % train_vocab_size)
print("Vocab_size Belief %s" % mem_lang.n_words)
print("Max. length of dialog words for RNN: %s " %
max_dialogue_history_length)
langs = [lang, mem_lang]
# dataloader, dataloader, dataloader, dataloader, List[Lang], List[Dict[str, str]], Dict[str, int], int
return train_loader, dev_loader, test_loader, langs, slots, gating_dict
def __init__(self):
super(TradeDST, self).__init__()
# load config
common_config_path = os.path.join(get_config_path(),
TradeDST.common_config_name)
common_config = json.load(open(common_config_path))
model_config_path = os.path.join(get_config_path(),
TradeDST.model_config_name)
model_config = json.load(open(model_config_path))
model_config.update(common_config)
self.model_config = model_config
self.model_config["data_path"] = os.path.join(
get_data_path(), "crosswoz/dst_trade_data")
self.model_config["n_gpus"] = (0 if self.model_config["device"]
== "cpu" else torch.cuda.device_count())
self.model_config["device"] = torch.device(self.model_config["device"])
if model_config["load_embedding"]:
model_config["hidden_size"] = 300
# download data
for model_key, url in TradeDST.model_urls.items():
dst = os.path.join(self.model_config["data_path"], model_key)
if model_key.endswith("pth"):
file_name = "trained_model_path"
elif model_key.endswith("pkl"):
file_name = model_key.rsplit("-", maxsplit=1)[0]
else:
file_name = model_key.split(".")[0] # ontology
self.model_config[file_name] = dst
if not os.path.exists(dst) or not self.model_config["use_cache"]:
download_from_url(url, dst)
# load date & model
ontology = json.load(
open(self.model_config["ontology"], "r", encoding="utf8"))
self.all_slots = get_slot_information(ontology)
self.gate2id = {"ptr": 0, "none": 1}
self.id2gate = {id_: gate for gate, id_ in self.gate2id.items()}
self.lang = pickle.load(open(self.model_config["lang"], "rb"))
self.mem_lang = pickle.load(open(self.model_config["mem-lang"], "rb"))
model = Trade(
lang=self.lang,
vocab_size=len(self.lang.index2word),
hidden_size=self.model_config["hidden_size"],
dropout=self.model_config["dropout"],
num_encoder_layers=self.model_config["num_encoder_layers"],
num_decoder_layers=self.model_config["num_decoder_layers"],
pad_id=self.model_config["pad_id"],
slots=self.all_slots,
num_gates=len(self.gate2id),
unk_mask=self.model_config["unk_mask"],
)
model.load_state_dict(
torch.load(self.model_config["trained_model_path"]))
self.model = model.to(self.model_config["device"]).eval()
print(f'>>> {self.model_config["trained_model_path"]} loaded ...')
self.state = default_state()
print(">>> State initialized ...")