def __init__(self):
DST.__init__(self)
self.state = default_state()
path = os.path.dirname(
os.path.dirname(
os.path.dirname(
os.path.dirname(os.path.dirname(
os.path.abspath(__file__))))))
path = os.path.join(path, 'data/multiwoz/value_dict.json')
self.value_dict = json.load(open(path))
def __init__(self, data_dir=DATA_PATH, eval_slots=multiwoz_zh_slot_list):
DST.__init__(self)
self.init_data()
processor = Processor(args)
self.processor = processor
label_list = processor.get_labels()
num_labels = [len(labels) for labels in label_list] # number of slot-values in each slot-type
# tokenizer
self.tokenizer = BertTokenizer.from_pretrained(args.bert_model_name, cache_dir=args.bert_model_cache_dir)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
self.device = torch.device("cuda" if USE_CUDA else "cpu")
self.sumbt_model = BeliefTracker(args, num_labels, self.device)
if USE_CUDA and N_GPU > 1:
self.sumbt_model = torch.nn.DataParallel(self.sumbt_model)
if args.fp16:
self.sumbt_model.half()
self.sumbt_model.to(self.device)
## Get slot-value embeddings
self.label_token_ids, self.label_len = [], []
for labels in label_list:
token_ids, lens = get_label_embedding(labels, args.max_label_length, self.tokenizer, self.device)
self.label_token_ids.append(token_ids)
self.label_len.append(lens)
self.label_map = [{label: i for i, label in enumerate(labels)} for labels in label_list]
self.label_map_inv = [{i: label for i, label in enumerate(labels)} for labels in label_list]
self.label_list = label_list
self.target_slot = processor.target_slot
## Get domain-slot-type embeddings
self.slot_token_ids, self.slot_len = \
get_label_embedding(processor.target_slot, args.max_label_length, self.tokenizer, self.device)
self.args = args
self.state = default_state()
self.param_restored = False
if USE_CUDA and N_GPU == 1:
self.sumbt_model.initialize_slot_value_lookup(self.label_token_ids, self.slot_token_ids)
elif USE_CUDA and N_GPU > 1:
self.sumbt_model.module.initialize_slot_value_lookup(self.label_token_ids, self.slot_token_ids)
self.cached_res = {}
convert_to_glue_format(DATA_PATH, SUMBT_PATH)
if not os.path.isdir(os.path.join(SUMBT_PATH, args.output_dir)):
os.makedirs(os.path.join(SUMBT_PATH, args.output_dir))
self.train_examples = processor.get_train_examples(os.path.join(SUMBT_PATH, args.tmp_data_dir), accumulation=False)
self.dev_examples = processor.get_dev_examples(os.path.join(SUMBT_PATH, args.tmp_data_dir), accumulation=False)
self.test_examples = processor.get_test_examples(os.path.join(SUMBT_PATH, args.tmp_data_dir), accumulation=False)
self.eval_slots = eval_slots
def __init__(self, ontology_vectors, ontology, slots, data_dir):
DST.__init__(self)
# data profile
self.data_dir = data_dir
self.validation_url = os.path.join(self.data_dir, 'data/validate.json')
self.word_vectors_url = os.path.join(
self.data_dir, 'word-vectors/paragram_300_sl999.txt')
self.training_url = os.path.join(self.data_dir, 'data/train.json')
self.ontology_url = os.path.join(self.data_dir, 'data/ontology.json')
self.testing_url = os.path.join(self.data_dir, 'data/test.json')
self.model_url = os.path.join(self.data_dir, 'models/model-1')
self.graph_url = os.path.join(self.data_dir, 'graphs/graph-1')
self.results_url = os.path.join(self.data_dir, 'results/log-1.txt')
self.kb_url = os.path.join(self.data_dir, 'data/') # not used
self.train_model_url = os.path.join(self.data_dir,
'train_models/model-1')
self.train_graph_url = os.path.join(self.data_dir,
'train_graph/graph-1')
self.model_variables = model_definition(ontology_vectors,
len(ontology),
slots,
num_hidden=None,
bidir=True,
net_type=None,
test=True,
dev='cpu')
self.state = default_state()
_config = tf.ConfigProto()
_config.gpu_options.allow_growth = True
_config.allow_soft_placement = True
self.sess = tf.Session(config=_config)
self.param_restored = False
self.det_dic = {}
for domain, dic in REF_USR_DA.items():
for key, value in dic.items():
assert '-' not in key
self.det_dic[key.lower()] = key + '-' + domain
self.det_dic[value.lower()] = key + '-' + domain
def parent_dir(path, time=1):
for _ in range(time):
path = os.path.dirname(path)
return path
root_dir = parent_dir(os.path.abspath(__file__), 4)
self.value_dict = json.load(
open(os.path.join(root_dir, 'data/multiwoz/value_dict.json')))
def __init__(self, data_dir=DATA_PATH):
DST.__init__(self)
# if not os.path.exists(data_dir):
# if model_file == '':
# raise Exception(
# 'Please provide remote model file path in config')
# resp = urllib.request.urlretrieve(model_file)[0]
# temp_file = tarfile.open(resp)
# temp_file.extractall('data')
# assert os.path.exists(data_dir)
processor = Processor(args)
self.processor = processor
# values of each slot e.g. values_list
label_list = processor.get_labels()
num_labels = [len(labels) for labels in label_list] # number of slot-values in each slot-type
# tokenizer
self.tokenizer = BertTokenizer.from_pretrained(args.bert_model_name, cache_dir=args.bert_model_cache_dir)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
self.device = torch.device("cuda" if USE_CUDA else "cpu")
self.sumbt_model = BeliefTracker(args, num_labels, self.device)
if USE_CUDA and N_GPU > 1:
self.sumbt_model = torch.nn.DataParallel(self.sumbt_model)
if args.fp16:
self.sumbt_model.half()
self.sumbt_model.to(self.device)
## Get slot-value embeddings
self.label_token_ids, self.label_len = [], []
for labels in label_list:
# encoding values
token_ids, lens = get_label_embedding(labels, args.max_label_length, self.tokenizer, self.device)
self.label_token_ids.append(token_ids)
self.label_len.append(lens)
self.label_map = [{label: i for i, label in enumerate(labels)} for labels in label_list]
self.label_map_inv = [{i: label for i, label in enumerate(labels)} for labels in label_list]
self.label_list = label_list
self.target_slot = processor.target_slot
## Get domain-slot-type embeddings
self.slot_token_ids, self.slot_len = \
get_label_embedding(processor.target_slot, args.max_label_length, self.tokenizer, self.device)
self.args = args
self.state = default_state()
self.param_restored = False
if USE_CUDA and N_GPU == 1:
self.sumbt_model.initialize_slot_value_lookup(self.label_token_ids, self.slot_token_ids)
elif USE_CUDA and N_GPU > 1:
self.sumbt_model.module.initialize_slot_value_lookup(self.label_token_ids, self.slot_token_ids)
self.cached_res = {}
convert_to_glue_format(DATA_PATH, SUMBT_PATH)
if not os.path.isdir(os.path.join(SUMBT_PATH, args.output_dir)):
os.makedirs(os.path.join(SUMBT_PATH, args.output_dir))
self.train_examples = processor.get_train_examples(os.path.join(SUMBT_PATH, args.tmp_data_dir), accumulation=False)
self.dev_examples = processor.get_dev_examples(os.path.join(SUMBT_PATH, args.tmp_data_dir), accumulation=False)
self.test_examples = processor.get_test_examples(os.path.join(SUMBT_PATH, args.tmp_data_dir), accumulation=False)
def __init__(
self,
model_file='https://convlab.blob.core.windows.net/convlab-2/sumbt.tar.gz',
arg_path=os.path.join(SUMBT_PATH, 'config.json'),
eval_slots=multiwoz_slot_list_en):
DST.__init__(self)
# if not os.path.exists(data_dir):
# if model_file == '':
# raise Exception(
# 'Please provide remote model file path in config')
# resp = urllib.request.urlretrieve(model_file)[0]
# temp_file = tarfile.open(resp)
# temp_file.extractall('data')
# assert os.path.exists(data_dir)
args = json.load(open(arg_path))
args = SimpleNamespace(**args)
self.args = args
data_dir = os.path.join(ROOT_PATH, args.data_dir)
if args.lang == 'zh':
convert_to_glue_format = convert_to_glue_format_zh
default_state = default_state_zh
processor = ProcessorZh(args)
eval_slots = multiwoz_slot_list_zh
else:
convert_to_glue_format = convert_to_glue_format_en
default_state = default_state_en
processor = ProcessorEn(args)
eval_slots = multiwoz_slot_list_en
self.processor = processor
label_list = processor.get_labels()
num_labels = [len(labels) for labels in label_list
] # number of slot-values in each slot-type
# tokenizer
self.tokenizer = AutoTokenizer.from_pretrained(
args.bert_model_name, cache_dir=args.bert_model_cache_dir)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
self.device = torch.device("cuda" if USE_CUDA else "cpu")
self.sumbt_model = BeliefTracker(args, num_labels, self.device)
if USE_CUDA and N_GPU > 1:
self.sumbt_model = torch.nn.DataParallel(self.sumbt_model)
if args.fp16:
self.sumbt_model.half()
self.sumbt_model.to(self.device)
## Get slot-value embeddings
self.label_token_ids, self.label_len = [], []
for labels in label_list:
token_ids, lens = get_label_embedding(labels,
args.max_label_length,
self.tokenizer, self.device)
self.label_token_ids.append(token_ids)
self.label_len.append(lens)
self.label_map = [{label: i
for i, label in enumerate(labels)}
for labels in label_list]
self.label_map_inv = [{i: label
for i, label in enumerate(labels)}
for labels in label_list]
self.label_list = label_list
self.target_slot = processor.target_slot
## Get domain-slot-type embeddings
self.slot_token_ids, self.slot_len = \
get_label_embedding(processor.target_slot, args.max_label_length, self.tokenizer, self.device)
self.args = args
self.state = default_state()
self.param_restored = False
if USE_CUDA and N_GPU == 1:
self.sumbt_model.initialize_slot_value_lookup(
self.label_token_ids, self.slot_token_ids)
elif USE_CUDA and N_GPU > 1:
self.sumbt_model.module.initialize_slot_value_lookup(
self.label_token_ids, self.slot_token_ids)
self.det_dic = {}
for domain, dic in REF_USR_DA.items():
for key, value in dic.items():
assert '-' not in key
self.det_dic[key.lower()] = key + '-' + domain
self.det_dic[value.lower()] = key + '-' + domain
self.cached_res = {}
convert_to_glue_format(os.path.join(ROOT_PATH, args.data_dir),
SUMBT_PATH, args)
if not os.path.isdir(os.path.join(SUMBT_PATH, args.output_dir)):
os.makedirs(os.path.join(SUMBT_PATH, args.output_dir))
self.train_examples = processor.get_train_examples(os.path.join(
SUMBT_PATH, args.tmp_data_dir),
accumulation=False)
self.dev_examples = processor.get_dev_examples(os.path.join(
SUMBT_PATH, args.tmp_data_dir),
accumulation=False)
self.test_examples = processor.get_test_examples(os.path.join(
SUMBT_PATH, args.tmp_data_dir),
accumulation=False)
self.eval_slots = eval_slots