def __init__(self, mode, config_file, model_file): """ BERT NLU initialization. Args: mode (str): can be either `'usr'`, `'sys'` or `'all'`, representing which side of data the model was trained on. model_file (str): model path or url Example: nlu = BERTNLU(mode='all', model_file='https://convlab.blob.core.windows.net/models/bert_multiwoz_all_context.zip') """ assert mode == 'usr' or mode == 'sys' or mode == 'all' config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs/{}'.format(config_file)) config = json.load(open(config_file)) DEVICE = config['DEVICE'] root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) data_dir = os.path.join(root_dir, config['data_dir']) output_dir = os.path.join(root_dir, config['output_dir']) if not os.path.exists(os.path.join(data_dir, 'intent_vocab.json')): preprocess(mode) intent_vocab = json.load(open(os.path.join(data_dir, 'intent_vocab.json'))) tag_vocab = json.load(open(os.path.join(data_dir, 'tag_vocab.json'))) dataloader = Dataloader(intent_vocab=intent_vocab, tag_vocab=tag_vocab, pretrained_weights=config['model']['pretrained_weights']) print('intent num:', len(intent_vocab)) print('tag num:', len(tag_vocab)) bert_config = BertConfig.from_pretrained(config['model']['pretrained_weights']) best_model_path = os.path.join(output_dir, 'pytorch_model.bin') if not os.path.exists(best_model_path): if not os.path.exists(output_dir): os.makedirs(output_dir) print('Load from model_file param') archive_file = cached_path(model_file) archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(root_dir) archive.close() print('Load from', best_model_path) model = JointBERT(bert_config, config['model'], DEVICE, dataloader.tag_dim, dataloader.intent_dim) model.load_state_dict(torch.load(os.path.join(output_dir, 'pytorch_model.bin'), DEVICE)) model.to(DEVICE) model.eval() self.model = model self.dataloader = dataloader print("BERTNLU loaded")
def __init__(self, mode, config_file, model_file): assert mode == 'usr' or mode == 'sys' or mode == 'all' config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs/{}'.format(config_file)) config = json.load(open(config_file)) DEVICE = config['DEVICE'] root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) data_dir = os.path.join(root_dir, config['data_dir']) output_dir = os.path.join(root_dir, config['output_dir']) if not os.path.exists(os.path.join(data_dir, 'intent_vocab.json')): preprocess(mode) intent_vocab = json.load( open(os.path.join(data_dir, 'intent_vocab.json'))) tag_vocab = json.load(open(os.path.join(data_dir, 'tag_vocab.json'))) dataloader = Dataloader( intent_vocab=intent_vocab, tag_vocab=tag_vocab, pretrained_weights=config['model']['pretrained_weights']) print('intent num:', len(intent_vocab)) print('tag num:', len(tag_vocab)) bert_config = BertConfig.from_pretrained( config['model']['pretrained_weights']) best_model_path = os.path.join(output_dir, 'bestcheckpoint.tar') if not os.path.exists(best_model_path): if not os.path.exists(output_dir): os.makedirs(output_dir) print('Load from model_file param') archive_file = cached_path(model_file) archive = zipfile.ZipFile(archive_file, 'r') archive.extractall(root_dir) archive.close() print('Load from', best_model_path) model = JointBERT(bert_config, config['model'], DEVICE, dataloader.tag_dim, dataloader.intent_dim) model.load_state_dict( torch.load(os.path.join(output_dir, 'pytorch_model.bin'), DEVICE)) model.to(DEVICE) model.eval() self.model = model self.dataloader = dataloader print("BERTNLU loaded")
pretrained_weights=config['model']['pretrained_weights']) print('intent num:', len(intent_vocab)) print('tag num:', len(tag_vocab)) for data_key in ['train', 'val', 'test']: dataloader.load_data(json.load(open(os.path.join(data_dir, '{}_data.json'.format(data_key)))), data_key, cut_sen_len=config['cut_sen_len'], use_bert_tokenizer=config['use_bert_tokenizer']) print('{} set size: {}'.format(data_key, len(dataloader.data[data_key]))) if not os.path.exists(output_dir): os.makedirs(output_dir) if not os.path.exists(log_dir): os.makedirs(log_dir) writer = SummaryWriter(log_dir) model = JointBERT(config['model'], DEVICE, dataloader.tag_dim, dataloader.intent_dim, dataloader.intent_weight) model.to(DEVICE) if config['model']['finetune']: no_decay = ['bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [ {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay) and p.requires_grad], 'weight_decay': config['model']['weight_decay']}, {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay) and p.requires_grad], 'weight_decay': 0.0} ] optimizer = AdamW(optimizer_grouped_parameters, lr=config['model']['learning_rate'], eps=config['model']['adam_epsilon']) scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=config['model']['warmup_steps'], num_training_steps=config['model']['max_step'])
print('tag num:', len(tag_vocab)) for data_key in ['val', 'test']: dataloader.load_data(json.load( open(os.path.join(data_dir, '{}_data.json'.format(data_key)))), data_key, cut_sen_len=0, use_bert_tokenizer=config['use_bert_tokenizer']) print('{} set size: {}'.format(data_key, len(dataloader.data[data_key]))) if not os.path.exists(output_dir): os.makedirs(output_dir) if not os.path.exists(log_dir): os.makedirs(log_dir) model = JointBERT(config['model'], DEVICE, dataloader.tag_dim, dataloader.intent_dim) model.load_state_dict( torch.load(os.path.join(output_dir, 'pytorch_model.bin'), DEVICE)) model.to(DEVICE) model.eval() batch_size = config['model']['batch_size'] data_key = 'test' predict_golden = {'intent': [], 'slot': [], 'overall': []} slot_loss, intent_loss = 0, 0 for pad_batch, ori_batch, real_batch_size in dataloader.yield_batches( batch_size, data_key=data_key): pad_batch = tuple(t.to(DEVICE) for t in pad_batch) word_seq_tensor, tag_seq_tensor, intent_tensor, word_mask_tensor, tag_mask_tensor, context_seq_tensor, context_mask_tensor = pad_batch if not config['model']['context']:
open(os.path.join(data_dir, '{}_data.json'.format(data_key)))), data_key) print('{} set size: {}'.format(data_key, len(dataloader.data[data_key]))) if not os.path.exists(output_dir): os.makedirs(output_dir) if not os.path.exists(log_dir): os.makedirs(log_dir) writer = SummaryWriter(log_dir) bert_config = BertConfig.from_pretrained( config['model']['pretrained_weights']) model = JointBERT(bert_config, config['model'], DEVICE, dataloader.tag_dim, dataloader.intent_dim, dataloader.intent_weight) model.to(DEVICE) if config['model']['finetune']: no_decay = ['bias', 'LayerNorm.weight'] optimizer_grouped_parameters = [{ 'params': [ p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay) and p.requires_grad ], 'weight_decay': config['model']['weight_decay'] }, { 'params': [ p for n, p in model.named_parameters() if any(nd in n for nd in no_decay) and p.requires_grad