def __init__(self, mode, config_file, model_file):
"""
BERT NLU initialization.
Args:
mode (str):
can be either `'usr'`, `'sys'` or `'all'`, representing which side of data the model was trained on.
model_file (str):
model path or url
Example:
nlu = BERTNLU(mode='all', model_file='https://convlab.blob.core.windows.net/models/bert_multiwoz_all_context.zip')
"""
assert mode == 'usr' or mode == 'sys' or mode == 'all'
config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs/{}'.format(config_file))
config = json.load(open(config_file))
DEVICE = config['DEVICE']
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
data_dir = os.path.join(root_dir, config['data_dir'])
output_dir = os.path.join(root_dir, config['output_dir'])
if not os.path.exists(os.path.join(data_dir, 'intent_vocab.json')):
preprocess(mode)
intent_vocab = json.load(open(os.path.join(data_dir, 'intent_vocab.json')))
tag_vocab = json.load(open(os.path.join(data_dir, 'tag_vocab.json')))
dataloader = Dataloader(intent_vocab=intent_vocab, tag_vocab=tag_vocab,
pretrained_weights=config['model']['pretrained_weights'])
print('intent num:', len(intent_vocab))
print('tag num:', len(tag_vocab))
bert_config = BertConfig.from_pretrained(config['model']['pretrained_weights'])
best_model_path = os.path.join(output_dir, 'pytorch_model.bin')
if not os.path.exists(best_model_path):
if not os.path.exists(output_dir):
os.makedirs(output_dir)
print('Load from model_file param')
archive_file = cached_path(model_file)
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(root_dir)
archive.close()
print('Load from', best_model_path)
model = JointBERT(bert_config, config['model'], DEVICE, dataloader.tag_dim, dataloader.intent_dim)
model.load_state_dict(torch.load(os.path.join(output_dir, 'pytorch_model.bin'), DEVICE))
model.to(DEVICE)
model.eval()
self.model = model
self.dataloader = dataloader
print("BERTNLU loaded")
def __init__(self, mode, config_file, model_file):
assert mode == 'usr' or mode == 'sys' or mode == 'all'
config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'configs/{}'.format(config_file))
config = json.load(open(config_file))
DEVICE = config['DEVICE']
root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
data_dir = os.path.join(root_dir, config['data_dir'])
output_dir = os.path.join(root_dir, config['output_dir'])
if not os.path.exists(os.path.join(data_dir, 'intent_vocab.json')):
preprocess(mode)
intent_vocab = json.load(
open(os.path.join(data_dir, 'intent_vocab.json')))
tag_vocab = json.load(open(os.path.join(data_dir, 'tag_vocab.json')))
dataloader = Dataloader(
intent_vocab=intent_vocab,
tag_vocab=tag_vocab,
pretrained_weights=config['model']['pretrained_weights'])
print('intent num:', len(intent_vocab))
print('tag num:', len(tag_vocab))
bert_config = BertConfig.from_pretrained(
config['model']['pretrained_weights'])
best_model_path = os.path.join(output_dir, 'bestcheckpoint.tar')
if not os.path.exists(best_model_path):
if not os.path.exists(output_dir):
os.makedirs(output_dir)
print('Load from model_file param')
archive_file = cached_path(model_file)
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(root_dir)
archive.close()
print('Load from', best_model_path)
model = JointBERT(bert_config, config['model'], DEVICE,
dataloader.tag_dim, dataloader.intent_dim)
model.load_state_dict(
torch.load(os.path.join(output_dir, 'pytorch_model.bin'), DEVICE))
model.to(DEVICE)
model.eval()
self.model = model
self.dataloader = dataloader
print("BERTNLU loaded")
pretrained_weights=config['model']['pretrained_weights'])
print('intent num:', len(intent_vocab))
print('tag num:', len(tag_vocab))
for data_key in ['train', 'val', 'test']:
dataloader.load_data(json.load(open(os.path.join(data_dir, '{}_data.json'.format(data_key)))), data_key,
cut_sen_len=config['cut_sen_len'], use_bert_tokenizer=config['use_bert_tokenizer'])
print('{} set size: {}'.format(data_key, len(dataloader.data[data_key])))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
writer = SummaryWriter(log_dir)
model = JointBERT(config['model'], DEVICE, dataloader.tag_dim, dataloader.intent_dim, dataloader.intent_weight)
model.to(DEVICE)
if config['model']['finetune']:
no_decay = ['bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in model.named_parameters() if
not any(nd in n for nd in no_decay) and p.requires_grad],
'weight_decay': config['model']['weight_decay']},
{'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay) and p.requires_grad],
'weight_decay': 0.0}
]
optimizer = AdamW(optimizer_grouped_parameters, lr=config['model']['learning_rate'],
eps=config['model']['adam_epsilon'])
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=config['model']['warmup_steps'],
num_training_steps=config['model']['max_step'])
print('tag num:', len(tag_vocab))
for data_key in ['val', 'test']:
dataloader.load_data(json.load(
open(os.path.join(data_dir, '{}_data.json'.format(data_key)))),
data_key,
cut_sen_len=0,
use_bert_tokenizer=config['use_bert_tokenizer'])
print('{} set size: {}'.format(data_key,
len(dataloader.data[data_key])))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
model = JointBERT(config['model'], DEVICE, dataloader.tag_dim,
dataloader.intent_dim)
model.load_state_dict(
torch.load(os.path.join(output_dir, 'pytorch_model.bin'), DEVICE))
model.to(DEVICE)
model.eval()
batch_size = config['model']['batch_size']
data_key = 'test'
predict_golden = {'intent': [], 'slot': [], 'overall': []}
slot_loss, intent_loss = 0, 0
for pad_batch, ori_batch, real_batch_size in dataloader.yield_batches(
batch_size, data_key=data_key):
pad_batch = tuple(t.to(DEVICE) for t in pad_batch)
word_seq_tensor, tag_seq_tensor, intent_tensor, word_mask_tensor, tag_mask_tensor, context_seq_tensor, context_mask_tensor = pad_batch
if not config['model']['context']:
open(os.path.join(data_dir, '{}_data.json'.format(data_key)))),
data_key)
print('{} set size: {}'.format(data_key,
len(dataloader.data[data_key])))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
writer = SummaryWriter(log_dir)
bert_config = BertConfig.from_pretrained(
config['model']['pretrained_weights'])
model = JointBERT(bert_config, config['model'], DEVICE, dataloader.tag_dim,
dataloader.intent_dim, dataloader.intent_weight)
model.to(DEVICE)
if config['model']['finetune']:
no_decay = ['bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params': [
p for n, p in model.named_parameters()
if not any(nd in n for nd in no_decay) and p.requires_grad
],
'weight_decay':
config['model']['weight_decay']
}, {
'params': [
p for n, p in model.named_parameters()
if any(nd in n for nd in no_decay) and p.requires_grad