def download_data(
self,
data_url="https://convlab.blob.core.windows.net/convlab-2/trade_crosswoz_data.zip"
):
"""Automatically download the pretrained model and necessary data."""
if os.path.exists(os.path.join(self.crosswoz_root, 'data/crosswoz')) and \
os.path.exists(os.path.join(self.crosswoz_root, 'data/dev_dials.json')):
return
data_dir = os.path.join(self.crosswoz_root, 'data')
if not os.path.exists(data_dir):
os.mkdir(data_dir)
zip_file_path = os.path.join(data_dir, 'trade_crosswoz_data.zip')
if not os.path.exists(os.path.join(data_dir,
'trade_crosswoz_data.zip')):
print('downloading crosswoz TRADE data files...')
cached_path(data_url, data_dir)
files = os.listdir(data_dir)
target_file = ''
for name in files:
if name.endswith('.json'):
target_file = name[:-5]
try:
assert target_file in files
except Exception as e:
print(
'allennlp download file error: TRADE Cross model download failed.'
)
raise e
shutil.copyfile(os.path.join(data_dir, target_file), zip_file_path)
with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
print('unzipping data file ...')
zip_ref.extractall(data_dir)
def download_data(self, data_url="https://convlab.blob.core.windows.net/convlab-2/comer.zip"):
"""Automatically download the pretrained model and necessary data."""
if os.path.exists(os.path.join(self.multiwoz_root, 'data/mwoz2_dm.dict')) and \
os.path.exists(os.path.join(self.multiwoz_root, 'data/mwoz2_sl.dict')) and \
os.path.exists(os.path.join(self.multiwoz_root, 'data/save_data.tgt.dict')):
return
print(os.path.join(self.multiwoz_root, 'data/mwoz2_dm.dict'))
print(os.path.join(self.multiwoz_root, 'data/mwoz2_sl.dict'))
print(os.path.join(self.multiwoz_root, 'data/save_data.tgt.dict'))
data_dir = os.path.join(self.multiwoz_root, 'data')
if not os.path.exists(data_dir):
os.mkdir(data_dir)
zip_file_path = os.path.join(data_dir, 'comer.zip')
if not os.path.exists(os.path.join(data_dir, 'comer.zip')):
print('downloading multiwoz COMER data files...')
cached_path(data_url, data_dir)
files = os.listdir(data_dir)
target_file = ''
for name in files:
if name.endswith('.json'):
target_file = name[:-5]
try:
assert target_file in files
except Exception as e:
print('allennlp download file error: COMER model download failed.')
raise e
shutil.copyfile(os.path.join(data_dir, target_file), zip_file_path)
print("Unzipping comer.zip...")
with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
zip_ref.extractall(data_dir)
print("Moving files and dirs...")
os.system("mv data/data bak/")
os.system("rm -rf data/")
os.system("mv bak/ data/")
def download_model(
self,
model_url="https://convlab.blob.core.windows.net/convlab-2/trade_multiwoz_model.zip"
):
"""Automatically download the pretrained model and necessary data."""
if os.path.exists(
os.path.join(self.multiwoz_root, 'model/TRADE-multiwozdst')):
return
model_dir = os.path.join(self.multiwoz_root, 'model')
if not os.path.exists(model_dir):
os.mkdir(model_dir)
zip_file_path = os.path.join(model_dir, 'trade_multiwoz_model.zip')
if not os.path.exists(
os.path.join(model_dir, 'trade_multiwoz_model.zip')):
print('downloading multiwoz Trade model files...')
cached_path(model_url, model_dir)
files = os.listdir(model_dir)
target_file = ''
for name in files:
if name.endswith('.json'):
target_file = name[:-5]
try:
assert target_file in files
except Exception as e:
print(
'allennlp download file error: TRADE Cross model download failed.'
)
raise e
shutil.copyfile(os.path.join(model_dir, target_file),
zip_file_path)
with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
print('unzipping model file ...')
zip_ref.extractall(model_dir)
def __init__(self, model_file=DEFAULT_MODEL_URL, name='DAMD'):
"""
Sequicity initialization
Args:
model_file (str):
trained model path or url.
Example:
damd = Damd()
"""
super(Damd, self).__init__(name=name)
if not os.path.exists(os.path.join(DEFAULT_DIRECTORY,
'multiwoz/data')):
print('down load data from', DEFAULT_ARCHIVE_FILE_URL)
archive_file = cached_path(DEFAULT_ARCHIVE_FILE_URL)
archive = zipfile.ZipFile(archive_file, 'r')
print('unzip to', os.path.join(DEFAULT_DIRECTORY, 'multiwoz/'))
archive.extractall(os.path.join(DEFAULT_DIRECTORY, 'multiwoz/'))
archive.close()
model_path = os.path.join(os.path.join(DEFAULT_DIRECTORY, 'multiwoz/'),
cfg.eval_load_path)
if not os.path.exists(model_path):
model_dir = os.path.dirname(model_path)
if not os.path.exists(model_dir):
os.makedirs(model_dir)
print('Load from model_file param')
print('down load data from', model_file)
archive_file = cached_path(model_file)
archive = zipfile.ZipFile(archive_file, 'r')
print('unzip to', model_dir)
archive.extractall(model_dir)
archive.close()
self.reader = MultiWozReader()
self.m = DAMD(self.reader)
if cfg.cuda and torch.cuda.is_available():
self.m = self.m.cuda() #cfg.cuda_device[0]
if cfg.limit_bspn_vocab:
self.reader.bspn_masks_tensor = {}
for key, values in self.reader.bspn_masks.items():
v_ = cuda_(torch.Tensor(values).long())
self.reader.bspn_masks_tensor[key] = v_
if cfg.limit_aspn_vocab:
self.reader.aspn_masks_tensor = {}
for key, values in self.reader.aspn_masks.items():
v_ = cuda_(torch.Tensor(values).long())
self.reader.aspn_masks_tensor[key] = v_
cfg.model_path = os.path.join(model_path, 'model.pkl')
self.load_model(cfg.model_path)
self.m.eval()
self.init_session()
def __init__(self,
archive_file=DEFAULT_ARCHIVE_FILE,
model_file='https://convlab.blob.core.windows.net/convlab-2/mle_policy_crosswoz.zip'):
root_dir = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.json'), 'r') as f:
cfg = json.load(f)
voc_file = os.path.join(root_dir, 'data/crosswoz/sys_da_voc.json')
voc_opp_file = os.path.join(root_dir, 'data/crosswoz/usr_da_voc.json')
self.vector = CrossWozVector(sys_da_voc_json=voc_file, usr_da_voc_json=voc_opp_file)
self.policy = MultiDiscretePolicy(self.vector.state_dim, cfg['h_dim'], self.vector.sys_da_dim).to(device=DEVICE)
if not os.path.isfile(archive_file):
if not model_file:
raise Exception("No model for MLE Policy is specified!")
archive_file = cached_path(model_file)
model_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'save')
if not os.path.exists(model_dir):
os.mkdir(model_dir)
if not os.path.exists(os.path.join(model_dir, 'best_mle.pol.mdl')):
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(model_dir)
self.load_from_pretrained(archive_file, model_file, cfg['load'])
def load_weights(self):
DEFAULT_DIRECTORY = os.path.join(os.path.dirname(os.path.abspath(__file__)), "models")
DEFAULT_ARCHIVE_FILE = os.path.join(DEFAULT_DIRECTORY, "sumbt_multiwoz.zip")
archive_file = DEFAULT_ARCHIVE_FILE
model_file = 'https://tatk-data.s3-ap-northeast-1.amazonaws.com/sumbt_multiwoz.zip'
if not os.path.isfile(archive_file):
if not model_file:
raise Exception("No model for SC-LSTM is specified!")
archive_file = cached_path(model_file)
model_dir = os.path.dirname(os.path.abspath(__file__))
if not os.path.exists(os.path.join(model_dir, 'resource')):
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(model_dir)
model_ckpt = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'pytorch_model.bin')
# model_ckpt = os.path.join(OUTPUT_DIR, "pytorch_model.bin")
model = self.belief_tracker
# in the case that slot and values are different between the training and evaluation
ptr_model = torch.load(model_ckpt)
print('loading pretrained weights')
if N_GPU == 1:
state = model.state_dict()
state.update(ptr_model)
model.load_state_dict(state)
else:
# print("Evaluate using only one device!")
model.module.load_state_dict(ptr_model)
model.to(DEVICE)
def __init__(self, mode):
"""
SVM NLU initialization.
Args:
mode (str):
can be either `'usr'`, `'sys'` or `'all'`, representing which side of data the model was trained on.
Example:
nlu = SVMNLU(mode='usr')
"""
assert mode == 'usr' or mode == 'sys' or mode == 'all'
model_file = 'https://tatk-data.s3-ap-northeast-1.amazonaws.com/svm_multiwoz_{}.zip'.format(mode)
config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs/multiwoz_{}.cfg'.format(mode))
self.config = configparser.ConfigParser()
self.config.read(config_file)
self.c = Classifier.classifier(self.config)
model_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
self.config.get("train", "output"))
model_dir = os.path.dirname(model_path)
if not os.path.exists(model_path):
if not os.path.exists(model_dir):
os.makedirs(model_dir)
print('Load from model_file param')
archive_file = cached_path(model_file)
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
archive.close()
self.c.load(model_path)
def __init__(self, archive_file, model_file=None, use_cuda=False):
if not os.path.isfile(archive_file):
if not model_file:
raise Exception("No model for DA-predictor is specified!")
archive_file = cached_path(model_file)
model_dir = os.path.dirname(os.path.abspath(__file__))
if not os.path.exists(os.path.join(model_dir, 'checkpoints')):
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(model_dir)
load_dir = os.path.join(model_dir,
"checkpoints/predictor/save_step_15120")
self.db = Database()
if not os.path.exists(load_dir):
archive = zipfile.ZipFile(f'{load_dir}.zip', 'r')
archive.extractall(os.path.dirname(load_dir))
self.tokenizer = BertTokenizer.from_pretrained("bert-base-uncased",
do_lower_case=True)
self.max_seq_length = 256
self.domain = 'restaurant'
self.model = BertForSequenceClassification.from_pretrained(
load_dir,
cache_dir=os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),
'distributed_{}'.format(-1)),
num_labels=44)
self.device = 'cuda' if use_cuda else 'cpu'
self.model.to(self.device)
def __init__(self,
archive_file=DEFAULT_ARCHIVE_FILE,
cuda_device=DEFAULT_CUDA_DEVICE,
model_file=None,
context_size=3):
""" Constructor for NLU class. """
self.context_size = context_size
check_for_gpu(cuda_device)
if not os.path.isfile(archive_file):
if not model_file:
raise Exception("No model for MILU is specified!")
archive_file = cached_path(model_file)
archive = load_archive(archive_file, cuda_device=cuda_device)
self.tokenizer = SpacyWordSplitter(language="en_core_web_sm")
_special_case = [{ORTH: u"id", LEMMA: u"id"}]
self.tokenizer.spacy.tokenizer.add_special_case(u"id", _special_case)
dataset_reader_params = archive.config["dataset_reader"]
self.dataset_reader = DatasetReader.from_params(dataset_reader_params)
self.model = archive.model
self.model.eval()
def __init__(self,
archive_file=DEFAULT_ARCHIVE_FILE,
use_cuda=False,
is_user=False,
model_file='https://convlab.blob.core.windows.net/convlab-2/nlg_sclstm_camrest.zip'):
if not os.path.isfile(archive_file):
if not model_file:
raise Exception("No model for SC-LSTM is specified!")
archive_file = cached_path(model_file)
model_dir = os.path.dirname(os.path.abspath(__file__))
if not os.path.exists(os.path.join(model_dir, 'resource')):
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(model_dir)
self.USE_CUDA = use_cuda
self.args, self.config = parse(is_user)
self.dataset = SimpleDatasetCam(self.config)
# get model hyper-parameters
hidden_size = self.config.getint('MODEL', 'hidden_size')
# get feat size
d_size = self.dataset.da_size + self.dataset.sv_size # len of 1-hot feat
vocab_size = len(self.dataset.word2index)
self.model = LMDeep('sclstm', vocab_size, vocab_size, hidden_size, d_size, n_layer=self.args['n_layer'],
use_cuda=use_cuda)
model_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), self.args['model_path'])
# print(model_path)
assert os.path.isfile(model_path)
self.model.load_state_dict(torch.load(model_path))
self.model.eval()
if use_cuda:
self.model.cuda()
def load_from_pretrained(self, archive_file, model_file, filename):
if not os.path.isfile(archive_file):
if not model_file:
raise Exception("No model for PPO Policy is specified!")
archive_file = cached_path(model_file)
model_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'save')
if not os.path.exists(model_dir):
os.mkdir(model_dir)
if not os.path.exists(os.path.join(model_dir, 'best_ppo.pol.mdl')):
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(model_dir)
policy_mdl = os.path.join(os.path.dirname(os.path.abspath(__file__)),
filename + '_ppo.pol.mdl')
if os.path.exists(policy_mdl):
self.policy.load_state_dict(
torch.load(policy_mdl, map_location=DEVICE))
logging.info(
'<<dialog policy>> loaded checkpoint from file: {}'.format(
policy_mdl))
value_mdl = os.path.join(os.path.dirname(os.path.abspath(__file__)),
filename + '_ppo.val.mdl')
if os.path.exists(value_mdl):
self.value.load_state_dict(
torch.load(value_mdl, map_location=DEVICE))
logging.info(
'<<dialog policy>> loaded checkpoint from file: {}'.format(
value_mdl))
def __init__(self, model_file=DEFAULT_MODEL_URL, name='Sequicity'):
"""
Sequicity initialization
Args:
model_file (str):
trained model path or url. default="https://convlab.blob.core.windows.net/convlab-2/sequicity_multiwoz.zip"
Example:
sequicity = Sequicity()
"""
super(Sequicity, self).__init__(name=name)
nltk.download('punkt')
config_file = DEFAULT_CONFIG_FILE
c = json.load(open(config_file))
cfg.init_handler(c['tsdf_init'])
if not os.path.exists(os.path.join(DEFAULT_DIRECTORY,
'multiwoz/data')):
print('down load data from', DEFAULT_ARCHIVE_FILE_URL)
archive_file = cached_path(DEFAULT_ARCHIVE_FILE_URL)
archive = zipfile.ZipFile(archive_file, 'r')
print('unzip to', os.path.join(DEFAULT_DIRECTORY, 'multiwoz/'))
archive.extractall(os.path.join(DEFAULT_DIRECTORY, 'multiwoz/'))
archive.close()
model_path = os.path.join(DEFAULT_DIRECTORY,
c['tsdf_init']['model_path'])
if not os.path.exists(model_path):
model_dir = os.path.dirname(model_path)
if not os.path.exists(model_dir):
os.makedirs(model_dir)
print('Load from model_file param')
print('down load data from', model_file)
archive_file = cached_path(model_file)
archive = zipfile.ZipFile(archive_file, 'r')
print('unzip to', model_dir)
archive.extractall(model_dir)
archive.close()
torch.manual_seed(cfg.seed)
torch.cuda.manual_seed(cfg.seed)
random.seed(cfg.seed)
np.random.seed(cfg.seed)
self.m = Model('multiwoz')
self.m.count_params()
self.m.load_model()
self.init_session()
def __init__(
self,
mode='all',
config_file='multiwoz_all_context.json',
model_file='https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_all_context.zip'
):
assert mode == 'usr' or mode == 'sys' or mode == 'all'
self.mode = mode
config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'configs/{}'.format(config_file))
config = json.load(open(config_file))
# print(config['DEVICE'])
# DEVICE = config['DEVICE']
DEVICE = 'cpu' if not torch.cuda.is_available() else 'cuda:0'
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
data_dir = os.path.join(root_dir, config['data_dir'])
output_dir = os.path.join(root_dir, config['output_dir'])
if not os.path.exists(os.path.join(data_dir, 'intent_vocab.json')):
preprocess(mode)
intent_vocab = json.load(
open(os.path.join(data_dir, 'intent_vocab.json')))
tag_vocab = json.load(open(os.path.join(data_dir, 'tag_vocab.json')))
dataloader = Dataloader(
intent_vocab=intent_vocab,
tag_vocab=tag_vocab,
pretrained_weights=config['model']['pretrained_weights'])
print('intent num:', len(intent_vocab))
print('tag num:', len(tag_vocab))
best_model_path = os.path.join(output_dir, 'pytorch_model.bin')
if not os.path.exists(best_model_path):
if not os.path.exists(output_dir):
os.makedirs(output_dir)
print('Load from model_file param')
archive_file = cached_path(model_file)
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(root_dir)
archive.close()
print('Load from', best_model_path)
model = JointBERT(config['model'], DEVICE, dataloader.tag_dim,
dataloader.intent_dim).cuda()
model.load_state_dict(
torch.load(os.path.join(output_dir, 'pytorch_model.bin'),
map_location=lambda storage, loc: storage.cuda(
torch.cuda.current_device())))
# model.to(DEVICE)
model.eval()
self.model = model
self.use_context = config['model']['context']
self.dataloader = dataloader
self.nlp = spacy.load('en_core_web_sm')
print("BERTNLU loaded")
def __init__(self, model_file=DEFAULT_MODEL_URL):
"""
Sequicity initialization
Args:
model_file (str):
trained model path or url. default="https://tatk-data.s3-ap-northeast-1.amazonaws.com/sequicity_camrest.zip"
Example:
sequicity = Sequicity()
"""
config_file = DEFAULT_CONFIG_FILE
c = json.load(open(config_file))
cfg.init_handler(c['tsdf_init'])
if not os.path.exists(os.path.join(DEFAULT_DIRECTORY, 'camrest/data')):
print('down load data from', DEFAULT_ARCHIVE_FILE_URL)
archive_file = cached_path(DEFAULT_ARCHIVE_FILE_URL)
archive = zipfile.ZipFile(archive_file, 'r')
print('unzip to', os.path.join(DEFAULT_DIRECTORY, 'camrest/'))
archive.extractall(os.path.join(DEFAULT_DIRECTORY, 'camrest/'))
archive.close()
model_path = os.path.join(DEFAULT_DIRECTORY,
c['tsdf_init']['model_path'])
if not os.path.exists(model_path):
model_dir = os.path.dirname(model_path)
if not os.path.exists(model_dir):
os.makedirs(model_dir)
print('Load from model_file param')
print('down load data from', model_file)
archive_file = cached_path(model_file)
archive = zipfile.ZipFile(archive_file, 'r')
print('unzip to', model_dir)
archive.extractall(model_dir)
archive.close()
torch.manual_seed(cfg.seed)
torch.cuda.manual_seed(cfg.seed)
random.seed(cfg.seed)
np.random.seed(cfg.seed)
self.m = Model('camrest')
self.m.count_params()
self.m.load_model()
self.init_session()
def auto_download(self):
"""Automatically download the pretrained model and necessary data."""
if os.path.exists(os.path.join(self.config_path, 'model/rnn_model_state_dict.th')) and \
os.path.exists(os.path.join(self.config_path, 'selection_model_state_dict.th')):
return
models_dir = os.path.join(self.config_path, 'models')
cached_path(self.file_url, models_dir)
files = os.listdir(models_dir)
target_file = ''
for name in files:
if name.endswith('.json'):
target_file = name[:-5]
try:
assert target_file in files
except Exception as e:
print('allennlp download file error: RnnRollout Deal_or_Not data download failed.')
raise e
zip_file_path = os.path.join(models_dir, target_file + '.zip')
shutil.copyfile(os.path.join(models_dir, target_file), zip_file_path)
with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
zip_ref.extractall(models_dir)
def download_model(self):
if not os.path.isdir(DOWNLOAD_DIRECTORY):
os.mkdir(DOWNLOAD_DIRECTORY)
# model_file = os.path.join(DOWNLOAD_DIRECTORY, 'pytorch_model.zip')
# if not os.path.isfile(model_file):
model_file = 'https://convlab.blob.core.windows.net/convlab-2/sumbt.tar.gz'
import tarfile
if not os.path.isfile(os.path.join(DOWNLOAD_DIRECTORY, 'pytorch_model.bin')):
archive_file = cached_path(model_file)
# archive = zipfile.ZipFile(archive_file, 'r')
t = tarfile.open(archive_file)
t.extractall(path=DOWNLOAD_DIRECTORY)
def __init__(
self,
mode='all',
config_file='crosswoz_all_context.json',
model_file='https://convlab.blob.core.windows.net/convlab-2/bert_crosswoz_all_context.zip'
):
assert mode == 'usr' or mode == 'sys' or mode == 'all'
config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'configs/{}'.format(config_file))
config = json.load(open(config_file))
DEVICE = config['DEVICE']
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
data_dir = os.path.join(root_dir, config['data_dir'])
output_dir = os.path.join(root_dir, config['output_dir'])
if not os.path.exists(os.path.join(data_dir, 'intent_vocab.json')):
preprocess(mode, config['model']['pretrained_weights'])
intent_vocab = json.load(
open(os.path.join(data_dir, 'intent_vocab.json')))
tag_vocab = json.load(open(os.path.join(data_dir, 'tag_vocab.json')))
dataloader = Dataloader(
intent_vocab=intent_vocab,
tag_vocab=tag_vocab,
pretrained_weights=config['model']['pretrained_weights'])
print('intent num:', len(intent_vocab))
print('tag num:', len(tag_vocab))
best_model_path = os.path.join(output_dir, 'pytorch_model.bin')
if not os.path.exists(best_model_path):
if not os.path.exists(output_dir):
os.makedirs(output_dir)
print('Load from model_file param')
archive_file = cached_path(model_file)
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(root_dir)
archive.close()
print('Load from', best_model_path)
model = JointBERT(config['model'], DEVICE, dataloader.tag_dim,
dataloader.intent_dim)
model.load_state_dict(
torch.load(os.path.join(output_dir, 'pytorch_model.bin'), DEVICE))
model.to(DEVICE)
model.eval()
self.model = model
self.dataloader = dataloader
print("BERTNLU loaded")
def auto_download(self):
"""Automatically download the pretrained model and necessary data."""
if os.path.exists(os.path.join(self.data_dir, 'models')) and \
os.path.exists(os.path.join(self.data_dir, 'data')) and \
os.path.exists(os.path.join(self.data_dir, 'word-vectors')):
return
cached_path(self.file_url, self.data_dir)
files = os.listdir(self.data_dir)
target_file = ''
for name in files:
if name.endswith('.json'):
target_file = name[:-5]
try:
assert target_file in files
except Exception as e:
print(
'allennlp download file error: MDBT Multiwoz data download failed.'
)
raise e
zip_file_path = os.path.join(self.data_dir, target_file + '.zip')
shutil.copyfile(os.path.join(self.data_dir, target_file),
zip_file_path)
with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
zip_ref.extractall(self.data_dir)
def load(self, archive_file, model_file, filename):
if not os.path.isfile(archive_file):
if not model_file:
raise Exception("No model for VHUS Policy is specified!")
archive_file = cached_path(model_file)
model_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'save')
if not os.path.exists(model_dir):
os.mkdir(model_dir)
if not os.path.exists(os.path.join(model_dir, 'best_simulator.mdl')):
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(model_dir)
user_mdl = os.path.join(os.path.dirname(os.path.abspath(__file__)), filename + '_simulator.mdl')
if os.path.exists(user_mdl):
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
self.user.load_state_dict(torch.load(user_mdl, map_location=DEVICE))
print('<<user simulator>> loaded checkpoint from file: {}'.format(user_mdl))
def __init__(
self,
archive_file=DEFAULT_ARCHIVE_FILE,
use_cuda=False,
is_user=False,
model_file='https://tatk-data.s3-ap-northeast-1.amazonaws.com/nlg_sclstm_multiwoz.zip'
):
if not os.path.isfile(archive_file):
if not model_file:
raise Exception("No model for SC-LSTM is specified!")
archive_file = cached_path(model_file)
model_dir = os.path.dirname(os.path.abspath(__file__))
if not os.path.exists(os.path.join(model_dir, 'resource')):
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(model_dir)
self.USE_CUDA = use_cuda
self.args, self.config = parse(is_user)
self.dataset = SimpleDatasetWoz(self.config)
# get model hyper-parameters
hidden_size = self.config.getint('MODEL', 'hidden_size')
# get feat size
d_size = self.dataset.do_size + self.dataset.da_size + self.dataset.sv_size # len of 1-hot feat
vocab_size = len(self.dataset.word2index)
self.model = LMDeep('sclstm',
vocab_size,
vocab_size,
hidden_size,
d_size,
n_layer=self.args['n_layer'],
use_cuda=use_cuda)
model_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
self.args['model_path'])
# print(model_path)
assert os.path.isfile(model_path)
self.model.load_state_dict(torch.load(model_path))
self.model.eval()
for name, param in self.model.named_parameters():
print(name, param.shape, param.device, param.requires_grad)
if use_cuda:
self.model.cuda()
def auto_download():
model_path = os.path.join(os.path.dirname(__file__), 'model')
data_path = os.path.join(os.path.dirname(__file__), 'data')
db_path = os.path.join(os.path.dirname(__file__), 'db')
root_path = os.path.dirname(__file__)
urls = {model_path: 'https://tatk-data.s3-ap-northeast-1.amazonaws.com/mdrg_model.zip',
data_path: 'https://tatk-data.s3-ap-northeast-1.amazonaws.com/mdrg_data.zip',
db_path: 'https://tatk-data.s3-ap-northeast-1.amazonaws.com/mdrg_db.zip'}
for path in [model_path, data_path, db_path]:
if not os.path.exists(path):
file_url = urls[path]
print('Downloading from: ', file_url)
archive_file = cached_path(file_url)
print('Extracting...')
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(root_path)
def __init__(self, archive_file, model_file=None, use_cuda=False):
if not os.path.isfile(archive_file):
if not model_file:
raise Exception("No model for HDSA is specified!")
archive_file = cached_path(model_file)
model_dir = os.path.dirname(os.path.abspath(__file__))
if not os.path.exists(os.path.join(model_dir, 'checkpoints')):
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(model_dir)
with open(os.path.join(model_dir, "data/vocab.json"), 'r') as f:
vocabulary = json.load(f)
vocab, ivocab = vocabulary['vocab'], vocabulary['rev']
self.tokenizer = Tokenizer(vocab, ivocab, False)
self.max_seq_length = 50
self.history = []
self.kbs = {}
self.decoder = TableSemanticDecoder(
vocab_size=self.tokenizer.vocab_len,
d_word_vec=128,
n_layers=3,
d_model=128,
n_head=4,
dropout=0.2)
self.device = 'cuda' if use_cuda else 'cpu'
self.decoder.to(self.device)
checkpoint_file = os.path.join(
model_dir, "checkpoints/generator/BERT_dim128_w_domain_exp")
if use_cuda:
self.decoder.load_state_dict(torch.load(checkpoint_file))
else:
self.decoder.load_state_dict(torch.load(checkpoint_file, 'cpu'))
with open(os.path.join(model_dir, 'data/svdic.pkl'), 'rb') as f:
self.dic = pickle.load(f)
with open(os.path.join(model_dir, 'data/mapping.pair')) as f:
self.replacements = []
for line in f:
tok_from, tok_to = line.replace('\n', '').split('\t')
self.replacements.append(
(' ' + tok_from + ' ', ' ' + tok_to + ' '))
def load(self, archive_file, model_file, filename):
if not os.path.isfile(archive_file):
if not model_file:
raise Exception("No model for MLE Policy is specified!")
archive_file = cached_path(model_file)
model_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'save')
if not os.path.exists(model_dir):
os.mkdir(model_dir)
if not os.path.exists(os.path.join(model_dir, 'best_mle.pol.mdl')):
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(model_dir)
policy_mdl = os.path.join(os.path.dirname(os.path.abspath(__file__)),
filename + '_mle.pol.mdl')
if os.path.exists(policy_mdl):
self.policy.load_state_dict(torch.load(policy_mdl))
print('<<dialog policy>> loaded checkpoint from file: {}'.format(
policy_mdl))
def __init__(
self,
archive_file=DEFAULT_ARCHIVE_FILE,
cuda_device=DEFAULT_CUDA_DEVICE,
model_file="https://convlab.blob.core.windows.net/convlab-2/new_milu(20200922)_multiwoz_all_context.tar.gz",
context_size=3):
""" Constructor for NLU class. """
self.context_size = context_size
cuda_device = 0 if torch.cuda.is_available() else DEFAULT_CUDA_DEVICE
check_for_gpu(cuda_device)
if not os.path.isfile(archive_file):
if not model_file:
raise Exception("No model for MILU is specified!")
archive_file = cached_path(model_file)
archive = load_archive(archive_file, cuda_device=cuda_device)
self.tokenizer = SpacyWordSplitter(language="en_core_web_sm")
_special_case = [{ORTH: u"id", LEMMA: u"id"}]
self.tokenizer.spacy.tokenizer.add_special_case(u"id", _special_case)
with open(
os.path.join(get_root_path(),
'data/multiwoz/db/postcode.json'), 'r') as f:
token_list = json.load(f)
for token in token_list:
token = token.strip()
self.tokenizer.spacy.tokenizer.add_special_case(
token, [{
ORTH: token,
LEMMA: token,
POS: u'NOUN'
}])
dataset_reader_params = archive.config["dataset_reader"]
self.dataset_reader = DatasetReader.from_params(dataset_reader_params)
self.model = archive.model
self.model.eval()
def __init__(
self,
archive_file=DEFAULT_ARCHIVE_FILE,
use_cuda=True,
is_user=False,
model_file='https://convlab.blob.core.windows.net/convlab-2/nlg-gpt-multiwoz.zip'
):
model_dir = os.path.dirname(os.path.abspath(__file__))
if not os.path.isfile(archive_file):
archive_file = cached_path(model_file)
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(model_dir)
self.model_name_or_path = os.path.join(model_dir, 'multiwoz')
self.length = 50
self.num_samples = 5
self.temperature = 1.0
self.repetition_penalty = 1.0
self.top_k = 50
self.top_p = 0.9
self.seed = 42
self.stop_token = '<|endoftext|>'
self.device = torch.device(
"cuda" if torch.cuda.is_available() and use_cuda else "cpu")
set_seed(self.seed, torch.cuda.device_count())
model_class, tokenizer_class = GPT2LMHeadModel, GPT2Tokenizer
self.tokenizer = tokenizer_class.from_pretrained(
self.model_name_or_path)
self.model = model_class.from_pretrained(self.model_name_or_path)
self.model.to(self.device)
self.model.eval()
if self.length < 0 and self.model.config.max_position_embeddings > 0:
self.length = self.model.config.max_position_embeddings
elif 0 < self.model.config.max_position_embeddings < self.length:
self.length = self.model.config.max_position_embeddings # No generation bigger than model size
elif self.length < 0:
self.length = self.MAX_LENGTH # avoid infinite loop
def auto_download():
model_path = os.path.join(os.path.dirname(__file__), 'model')
data_path = os.path.join(os.path.dirname(__file__), 'data')
db_path = os.path.join(os.path.dirname(__file__), 'db')
root_path = os.path.dirname(__file__)
urls = {
model_path:
'https://convlab.blob.core.windows.net/convlab-2/mdrg_model.zip',
data_path:
'https://convlab.blob.core.windows.net/convlab-2/mdrg_data.zip',
db_path: 'https://convlab.blob.core.windows.net/convlab-2/mdrg_db.zip'
}
for path in [model_path, data_path, db_path]:
if not os.path.exists(path):
file_url = urls[path]
print('Downloading from: ', file_url)
archive_file = cached_path(file_url)
print('Extracting...')
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(root_path)
def __init__(
self,
archive_file=DEFAULT_ARCHIVE_FILE,
cuda_device=DEFAULT_CUDA_DEVICE,
model_file="https://convlab.blob.core.windows.net/convlab-2/larl.zip"
):
if not os.path.isfile(archive_file):
if not model_file:
raise Exception("No model for LaRL is specified!")
archive_file = cached_path(model_file)
temp_path = os.path.dirname(os.path.abspath(__file__))
#print(temp_path)
zip_ref = zipfile.ZipFile(archive_file, 'r')
zip_ref.extractall(temp_path)
zip_ref.close()
self.prev_state = default_state()
self.prev_active_domain = None
domain_name = 'object_division'
domain_info = domain.get_domain(domain_name)
self.db = Database()
data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'data')
train_data_path = os.path.join(data_path, 'train_dials.json')
if not os.path.exists(train_data_path):
zipped_file = os.path.join(data_path, 'norm-multi-woz.zip')
archive = zipfile.ZipFile(zipped_file, 'r')
archive.extractall(data_path)
norm_multiwoz_path = data_path
with open(
os.path.join(norm_multiwoz_path,
'input_lang.index2word.json')) as f:
self.input_lang_index2word = json.load(f)
with open(
os.path.join(norm_multiwoz_path,
'input_lang.word2index.json')) as f:
self.input_lang_word2index = json.load(f)
with open(
os.path.join(norm_multiwoz_path,
'output_lang.index2word.json')) as f:
self.output_lang_index2word = json.load(f)
with open(
os.path.join(norm_multiwoz_path,
'output_lang.word2index.json')) as f:
self.output_lang_word2index = json.load(f)
config = Pack(
seed=10,
train_path=train_data_path,
max_vocab_size=1000,
last_n_model=5,
max_utt_len=50,
max_dec_len=50,
backward_size=2,
batch_size=1,
use_gpu=True,
op='adam',
init_lr=0.001,
l2_norm=1e-05,
momentum=0.0,
grad_clip=5.0,
dropout=0.5,
max_epoch=100,
embed_size=100,
num_layers=1,
utt_rnn_cell='gru',
utt_cell_size=300,
bi_utt_cell=True,
enc_use_attn=True,
dec_use_attn=True,
dec_rnn_cell='lstm',
dec_cell_size=300,
dec_attn_mode='cat',
y_size=10,
k_size=20,
beta=0.001,
simple_posterior=True,
contextual_posterior=True,
use_mi=False,
use_pr=True,
use_diversity=False,
#
beam_size=20,
fix_batch=True,
fix_train_batch=False,
avg_type='word',
print_step=300,
ckpt_step=1416,
improve_threshold=0.996,
patient_increase=2.0,
save_model=True,
early_stop=False,
gen_type='greedy',
preview_batch_num=None,
k=domain_info.input_length(),
init_range=0.1,
pretrain_folder='2019-09-20-21-43-06-sl_cat',
forward_only=False)
config.use_gpu = config.use_gpu and torch.cuda.is_available()
self.corpus = corpora_inference.NormMultiWozCorpus(config)
self.model = SysPerfectBD2Cat(self.corpus, config)
self.config = config
if config.use_gpu:
self.model.load_state_dict(
torch.load(os.path.join(temp_path, 'larl_model/best-model')))
self.model.cuda()
else:
self.model.load_state_dict(
torch.load(os.path.join(temp_path, 'larl_model/best-model'),
map_location=lambda storage, loc: storage))
self.model.eval()
self.dic = pickle.load(
open(os.path.join(temp_path, 'larl_model/svdic.pkl'), 'rb'))
def _read(self, file_path):
# if `file_path` is a URL, redirect to the cache
file_path = cached_path(file_path)
if file_path.endswith("zip"):
archive = zipfile.ZipFile(file_path, "r")
data_file = archive.open(os.path.basename(file_path)[:-4])
else:
data_file = open(file_path, "r")
logger.info("Reading instances from lines in file at: %s", file_path)
dialogs = json.load(data_file)
for dial_name in dialogs:
dialog = dialogs[dial_name]["log"]
context_tokens_list = []
for i, turn in enumerate(dialog):
if self._agent and self._agent == "user" and i % 2 != 1:
continue
if self._agent and self._agent == "system" and i % 2 != 0:
continue
tokens = turn["text"].split()
dialog_act = {}
for dacts in turn["span_info"]:
if dacts[0] not in dialog_act:
dialog_act[dacts[0]] = []
dialog_act[dacts[0]].append(
[dacts[1], " ".join(tokens[dacts[3]:dacts[4] + 1])])
spans = turn["span_info"]
tags = []
for j in range(len(tokens)):
for span in spans:
if j == span[3]:
tags.append("B-" + span[0] + "+" + span[1])
break
if j > span[3] and j <= span[4]:
tags.append("I-" + span[0] + "+" + span[1])
break
else:
tags.append("O")
intents = []
for dacts in turn["dialog_act"]:
for dact in turn["dialog_act"][dacts]:
if dacts not in dialog_act or dact[0] not in [
sv[0] for sv in dialog_act[dacts]
]:
if dact[1] in [
"none", "?", "yes", "no", "do nt care",
"do n't care"
]:
intents.append(dacts + "+" + dact[0] + "*" +
dact[1])
for dacts in turn["dialog_act"]:
for dact in turn["dialog_act"][dacts]:
if dacts not in dialog_act:
dialog_act[dacts] = turn["dialog_act"][dacts]
break
elif dact[0] not in [
sv[0] for sv in dialog_act[dacts]
]:
dialog_act[dacts].append(dact)
num_context = random.randint(
0, self._context_size
) if self._random_context_size else self._context_size
if len(context_tokens_list) > 0 and num_context > 0:
wrapped_context_tokens = [
Token(token) for context_tokens in
context_tokens_list[-num_context:]
for token in context_tokens
]
else:
wrapped_context_tokens = [Token("SENT_END")]
wrapped_tokens = [Token(token) for token in tokens]
context_tokens_list.append(tokens + ["SENT_END"])
yield self.text_to_instance(wrapped_context_tokens,
wrapped_tokens, tags, intents,
dialog_act)
def __init__(
self,
mode='all',
config_file='multiwoz_all_context.json',
model_file='https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_all_context.zip'
):
assert mode == 'usr' or mode == 'sys' or mode == 'all'
self.mode = mode
config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'configs/{}'.format(config_file))
config = json.load(open(config_file))
# print(config['DEVICE'])
# DEVICE = config['DEVICE']
DEVICE = 'cpu' if not torch.cuda.is_available() else 'cuda:0'
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
data_dir = os.path.join(root_dir, config['data_dir'])
output_dir = os.path.join(root_dir, config['output_dir'])
if not os.path.exists(os.path.join(data_dir, 'intent_vocab.json')):
preprocess(mode)
intent_vocab = json.load(
open(os.path.join(data_dir, 'intent_vocab.json')))
tag_vocab = json.load(open(os.path.join(data_dir, 'tag_vocab.json')))
dataloader = Dataloader(
intent_vocab=intent_vocab,
tag_vocab=tag_vocab,
pretrained_weights=config['model']['pretrained_weights'])
print('intent num:', len(intent_vocab))
print('tag num:', len(tag_vocab))
best_model_path = os.path.join(output_dir, 'pytorch_model.bin')
if not os.path.exists(best_model_path):
if not os.path.exists(output_dir):
os.makedirs(output_dir)
print('Load from model_file param')
archive_file = cached_path(model_file)
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(root_dir)
archive.close()
print('Load from', best_model_path)
model = JointBERT(config['model'], DEVICE, dataloader.tag_dim,
dataloader.intent_dim)
model.load_state_dict(
torch.load(os.path.join(output_dir, 'pytorch_model.bin'), DEVICE))
model.to(DEVICE)
model.eval()
self.model = model
self.use_context = config['model']['context']
self.dataloader = dataloader
try:
self.nlp = spacy.load("en_core_web_sm")
except Exception:
print('download en_core_web_sm for spacy')
from spacy.cli.download import download as spacy_download
spacy_download("en_core_web_sm")
spacy_model_module = __import__("en_core_web_sm")
self.nlp = spacy_model_module.load()
with open(
os.path.join(get_root_path(),
'data/multiwoz/db/postcode.json'), 'r') as f:
token_list = json.load(f)
for token in token_list:
token = token.strip()
self.nlp.tokenizer.add_special_case(token, [{
ORTH: token,
LEMMA: token,
POS: u'NOUN'
}])
print("BERTNLU loaded")
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
from .utils.config import *
from .models.TRADE import *
from convlab2.util.file_util import cached_path
'''
python3 myTest.py -ds= -path= -bsz=
'''
# download pretrained model
cached_path(args['model_url'], args['model_path'])
import zipfile
with zipfile.ZipFile(args['model_path'] + 'trade_crosswoz_model.zip',
'r') as zip_ref:
zip_ref.extractall(args['model_path'])
directory = args['path'].split("/")
HDD = directory[2].split('HDD')[1].split('BSZ')[0]
decoder = directory[1].split('-')[0]
BSZ = int(args['batch']) if args['batch'] else int(
directory[2].split('BSZ')[1].split('DR')[0])
args["decoder"] = decoder
args["HDD"] = HDD
print("HDD", HDD, "decoder", decoder, "BSZ", BSZ)
if args['dataset'] == 'multiwoz':
from .utils.utils_multiWOZ_DST import *
else:
print("You need to provide the --dataset information")
