def __init__(self, mode, config_file, model_file):
"""
BERT NLU initialization.
Args:
mode (str):
can be either `'usr'`, `'sys'` or `'all'`, representing which side of data the model was trained on.
model_file (str):
model path or url
Example:
nlu = BERTNLU(mode='all', model_file='https://convlab.blob.core.windows.net/models/bert_multiwoz_all_context.zip')
"""
assert mode == 'usr' or mode == 'sys' or mode == 'all'
config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs/{}'.format(config_file))
config = json.load(open(config_file))
DEVICE = config['DEVICE']
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
data_dir = os.path.join(root_dir, config['data_dir'])
output_dir = os.path.join(root_dir, config['output_dir'])
if not os.path.exists(os.path.join(data_dir, 'intent_vocab.json')):
preprocess(mode)
intent_vocab = json.load(open(os.path.join(data_dir, 'intent_vocab.json')))
tag_vocab = json.load(open(os.path.join(data_dir, 'tag_vocab.json')))
dataloader = Dataloader(intent_vocab=intent_vocab, tag_vocab=tag_vocab,
pretrained_weights=config['model']['pretrained_weights'])
print('intent num:', len(intent_vocab))
print('tag num:', len(tag_vocab))
bert_config = BertConfig.from_pretrained(config['model']['pretrained_weights'])
best_model_path = os.path.join(output_dir, 'pytorch_model.bin')
if not os.path.exists(best_model_path):
if not os.path.exists(output_dir):
os.makedirs(output_dir)
print('Load from model_file param')
archive_file = cached_path(model_file)
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(root_dir)
archive.close()
print('Load from', best_model_path)
model = JointBERT(bert_config, config['model'], DEVICE, dataloader.tag_dim, dataloader.intent_dim)
model.load_state_dict(torch.load(os.path.join(output_dir, 'pytorch_model.bin'), DEVICE))
model.to(DEVICE)
model.eval()
self.model = model
self.dataloader = dataloader
print("BERTNLU loaded")
def __init__(self, mode, config_file, model_file):
assert mode == 'usr' or mode == 'sys' or mode == 'all'
config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'configs/{}'.format(config_file))
config = json.load(open(config_file))
DEVICE = config['DEVICE']
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
data_dir = os.path.join(root_dir, config['data_dir'])
output_dir = os.path.join(root_dir, config['output_dir'])
if not os.path.exists(os.path.join(data_dir, 'intent_vocab.json')):
preprocess(mode)
intent_vocab = json.load(
open(os.path.join(data_dir, 'intent_vocab.json')))
tag_vocab = json.load(open(os.path.join(data_dir, 'tag_vocab.json')))
dataloader = Dataloader(
intent_vocab=intent_vocab,
tag_vocab=tag_vocab,
pretrained_weights=config['model']['pretrained_weights'])
print('intent num:', len(intent_vocab))
print('tag num:', len(tag_vocab))
bert_config = BertConfig.from_pretrained(
config['model']['pretrained_weights'])
best_model_path = os.path.join(output_dir, 'bestcheckpoint.tar')
if not os.path.exists(best_model_path):
if not os.path.exists(output_dir):
os.makedirs(output_dir)
print('Load from model_file param')
archive_file = cached_path(model_file)
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(root_dir)
archive.close()
print('Load from', best_model_path)
model = JointBERT(bert_config, config['model'], DEVICE,
dataloader.tag_dim, dataloader.intent_dim)
model.load_state_dict(
torch.load(os.path.join(output_dir, 'pytorch_model.bin'), DEVICE))
model.to(DEVICE)
model.eval()
self.model = model
self.dataloader = dataloader
print("BERTNLU loaded")
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
if config['model']['finetune']:
scheduler.step() # Update learning rate schedule
model.zero_grad()
if step % check_step == 0:
train_slot_loss = train_slot_loss / check_step
train_intent_loss = train_intent_loss / check_step
print('[%d|%d] step' % (step, max_step))
print('\t slot loss:', train_slot_loss)
print('\t intent loss:', train_intent_loss)
predict_golden = {'intent': [], 'slot': [], 'overall': []}
val_slot_loss, val_intent_loss = 0, 0
model.eval()
for pad_batch, ori_batch, real_batch_size in dataloader.yield_batches(batch_size, data_key='val'):
pad_batch = tuple(t.to(DEVICE) for t in pad_batch)
word_seq_tensor, tag_seq_tensor, intent_tensor, word_mask_tensor, tag_mask_tensor, context_seq_tensor, context_mask_tensor = pad_batch
if not config['model']['context']:
context_seq_tensor, context_mask_tensor = None, None
with torch.no_grad():
slot_logits, intent_logits, slot_loss, intent_loss = model.forward(word_seq_tensor,
word_mask_tensor,
tag_seq_tensor,
tag_mask_tensor,
intent_tensor,
context_seq_tensor,
context_mask_tensor)
val_slot_loss += slot_loss.item() * real_batch_size