def test(args, device_id, pt): device = "cpu" if args.visible_gpus == '-1' else "cuda" if pt != '': test_from = pt else: test_from = args.best_model logger.info('Loading checkpoint from %s' % test_from) checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if k in model_flags: setattr(args, k, opt[k]) # print(args) config = BertConfig.from_json_file(args.bert_config_path) model = Summarizer(args, device, load_pretrained_bert=False, bert_config=config) model.load_cp(checkpoint) model.eval() logger.info("Test dataset......") test_dataset = torch.load(args.bert_data_path + 'test.data') trainer = build_trainer(args, device_id, model, None) trainer.test(model, test_dataset, device) logger.info("Valid dataset......") test_dataset = torch.load(args.bert_data_path + 'valid.data') trainer = build_trainer(args, device_id, model, None) trainer.test(model, test_dataset, device)
def validate(args, device_id, pt, epoch): device = "cpu" if args.visible_gpus == '-1' else "cuda" if (pt != ''): test_from = pt else: test_from = args.test_from logger.info('Loading checkpoint from %s' % test_from) checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) config = BertConfig.from_json_file(args.bert_config_name) model = Summarizer(args, device, load_pretrained_bert=False, bert_config=config) model.load_cp(checkpoint) model.eval() valid_dataset = torch.load(args.bert_data_path + 'valid.data') trainer = build_trainer(args, device_id, model, None) stats = trainer.validate(valid_dataset, epoch) return stats.xent()
def __init__(self, args): super(BertBiLstmCrf, self).__init__(args) self.args = args self.vector_path = args.vector_path # 预训练词向量的路径 txt self.embedding_dim = args.embedding_dim self.hidden_dim = args.hidden_dim # 隐藏层 self.tag_num = args.tag_num self.batch_size = args.batch_size self.bidirectional = True # BiLstm self.num_layers = args.num_layers self.pad_index = args.pad_index self.dropout = args.dropout # 训练时网络中连接 dropout 的概率 self.save_path = args.save_path embedding_dimension = args.bert_embedding_dim self.embedding = BertModel( config=BertConfig.from_json_file(args.bert_config_json)).to(DEVICE) self.embedding.load_state_dict(torch.load(args.bert_weight)) self.drop = nn.Dropout(0.5) self.lstm = nn.LSTM(embedding_dimension, self.hidden_dim, bidirectional=self.bidirectional, num_layers=self.num_layers, dropout=self.dropout).to(DEVICE) # hidden 除以 2 是因为,双向lstm输出的时候会翻倍,不除以二改成下面的 linear层中 hidden*2 也行 self.linear1 = nn.Linear(self.hidden_dim * 2, self.hidden_dim).to(DEVICE) self.lin_drop = nn.Dropout(0.5) self.linear2 = nn.Linear(self.hidden_dim, self.tag_num + 2).to( DEVICE) # 隐藏层到 label 的线性转换,即维度变换。 self.crf_layer = CRF(self.tag_num).to(DEVICE)
def convert_tf_checkpoint_to_pytorch(): # gave error originally. Solution found at: https://github.com/tensorflow/models/issues/2676 tf_path = 'weights/biobert_v1.1_pubmed/model.ckpt-1000000' init_vars = tf.train.list_variables(tf_path) excluded = ['BERTAdam', '_power', 'global_step'] init_vars = list(filter(lambda x: all([True if e not in x[0] else False for e in excluded]), init_vars)) print(init_vars) names = [] arrays = [] for name, shape in init_vars: print("Loading TF weights {} with shape {}".format(name,shape)) array = tf.train.load_variable(tf_path,name) names.append(name) arrays.append(array) config = BertConfig.from_json_file('weights/biobert_v1.1_pubmed/bert_config.json') print('Building Pytorch model from configuration {}'.format(str(config))) model = BertForPreTraining(config) for name, array in zip(names,arrays): name = name.split('/') # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v # which are not required for using pretrained model if any(n in ["adam_v", "adam_m", "global_step"] for n in name): print("Skipping {}".format("/".join(name))) continue pointer = model for m_name in name: if re.fullmatch(r'[A-Za-z]+_\d+', m_name): l = re.split(r'_(\d+)', m_name) else: l = [m_name] if l[0] == 'kernel' or l[0] == 'gamma': pointer = getattr(pointer, 'weight') elif l[0] == 'output_bias' or l[0] == 'beta': pointer = getattr(pointer, 'bias') elif l[0] == 'output_weights': pointer = getattr(pointer, 'weight') else: pointer = getattr(pointer, l[0]) if len(l) >= 2: num = int(l[1]) pointer = pointer[num] if m_name[-11:] == '_embeddings': pointer = getattr(pointer, 'weight') elif m_name == 'kernel': array = np.transpose(array) try: assert pointer.shape == array.shape except AssertionError as e: e.args += (pointer.shape, array.shape) raise print("Initialize PyTorch weight {}".format(name)) pointer.data = torch.from_numpy(array) # Save pytorch-model print("Save PyTorch model to {}".format('weights/')) torch.save(model.state_dict(), 'weights/pytorch_weight')
def validate(args, device_id, pt, step): device = "cpu" if args.visible_gpus == "-1" else "cuda" if (pt != ""): test_from = pt else: test_from = args.test_from logger.info("Loading checkpoint from %s" % test_from) checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint["opt"]) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) config = BertConfig.from_json_file(args.bert_config_path) model = Summarizer(args, device, load_pretrained_bert=False, bert_config=config) model.load_cp(checkpoint) model.eval() valid_iter = data_loader.Dataloader(args, load_dataset(args, "valid", shuffle=False), args.batch_size, device, shuffle=False, is_test=False) trainer = build_trainer(args, device_id, model, None) stats = trainer.validate(valid_iter, step) return stats.xent()
def __init__(self, args): # 初始化模型,建立encoder和decoder super(Summarizer, self).__init__() if args.mode == "train": self.encoder = BertModel.from_pretrained('bert-base-cased', cache_dir="./temp") elif args.mode == "test": config = BertConfig.from_json_file(args.predict_config) self.encoder = BertModel(config) self.args = args # we choose same hiedden_size with bert embedding self.decoder = nn.GRU(input_size=768, hidden_size=768, num_layers=1) # 初始化参数 self.vocab = torch.load(args.vocab_file) self.vocab_size = len(self.vocab) self.hidden_size = 768 self.w = torch.randn((self.vocab_size, self.hidden_size), requires_grad=True) self.b = torch.randn((self.vocab_size), requires_grad=True) # self.loss_func = nn.SoftMarginLoss() self.optimizer = optim.Adagrad([self.w, self.b], lr=0.01) self.loss_func = nn.BCELoss() self.loss = 0 self.device = args.device
def summarize(args, cp, data): # load path to model checkpoint if cp != '': checkpoint = cp else: checkpoint = args.checkpoint pt = torch.load(checkpoint, map_location=lambda storage, loc: storage) # configure pre-trained bert config = BertConfig.from_json_file(args.bert_path) # configure the device id device = "cpu" if args.visible_gpus == "-1" else "cuda" # load the model using checkpoint model = Summarizer(args, device, load_pretrained_bert=False, bert_config=config) model.load_cp(pt) model.eval() num_sentence = args.num_sentence # build the trainer model bertsum = Trainer(args, device, model) # run inference on trainer model bertsum.infer(data, num_sentence)
def test(args, device_id, pt, step): device = "cpu" if args.visible_gpus == '-1' else "cuda" if (pt != ''): test_from = pt else: test_from = args.test_from logger.info('Loading checkpoint from %s' % test_from) checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) config = BertConfig.from_json_file(args.bert_config_path) model = Summarizer(args, device, load_pretrained_bert=False, bert_config=config) model.load_cp(checkpoint) model.eval() test_iter = data_loader.Dataloader(args, load_dataset(args, 'test', shuffle=False), args.batch_size, device, shuffle=False, is_test=True) trainer = build_trainer(args, device_id, model, None) trainer.test(test_iter, step)
def test_config_to_json_file(self): config_first = BertConfig(vocab_size_or_config_json_file=99, hidden_size=37) json_file_path = "/tmp/config.json" config_first.to_json_file(json_file_path) config_second = BertConfig.from_json_file(json_file_path) os.remove(json_file_path) self.assertEqual(config_second.to_dict(), config_first.to_dict())
def main(): DATA_PATH = Path('/home/ky/text_classification/data/') PATH = Path('/home/ky/text_classification/data/tmp') CLAS_DATA_PATH = PATH / 'class' BERT_PRETRAINED_PATH = Path( '/home/ky/text_classification/chinese_L-12_H-768_A-12/') PYTORCH_PRETRAINED_BERT_CACHE = BERT_PRETRAINED_PATH / 'cache/' PYTORCH_PRETRAINED_BERT_CACHE.mkdir(exist_ok=True) args = { "train_size": -1, "val_size": -1, "full_data_dir": DATA_PATH, "data_dir": PATH, "task_name": "toxic_multilabel", "bert_model": BERT_PRETRAINED_PATH, "output_dir": CLAS_DATA_PATH / 'output', "max_seq_length": 512, "do_train": True, "do_eval": True, "do_lower_case": True, "train_batch_size": 32, "eval_batch_size": 32, "learning_rate": 3e-5, "num_train_epochs": 4.0, "warmup_proportion": 0.1, "no_cuda": True, "local_rank": -1, "seed": 42, "gradient_accumulation_steps": 1, "optimize_on_cpu": False, "fp16": False, "loss_scale": 128 } config_path = '/home/ky/text_classification/BLUC_MODEL/trained_model/config.json' model_path = '/home/ky/text_classification/BLUC_MODEL/trained_model/pytorch_model.bin' tag_path = '/home/ky/text_classification/data/tmp/labels.json' dataset_path = '/home/ky/text_classification/data/dev-20000.tsv' bert_pretrained_path = '/data/disk1/SegPOSdata/Bert_Pretrained/bert-base-chinese' config_a = BertConfig.from_json_file(config_path) device = torch.device('cuda:1') print(device) tokenizer = BertTokenizer.from_pretrained(bert_pretrained_path) dataset = TsvDataset(dataset_path, tokenizer, chunkSize=2**12) tagDict = TagDictionary.load(tag_path) model = BertForMultiLabelSequenceClassification.load( model_path, config_a, 627) predictor = Predictor(model, tagDict, device, maxInputLen=512) acc = predictor.validate(dataset, args['eval_batch_size']) print(len(dataset)) print(acc)
def __init__(self): # configuration self.ROOT_FOLDER = os.path.dirname(__file__) print('ROOT_FOLDER', self.ROOT_FOLDER) DIRECTORIES = { 'ml_hate_speech_path': os.path.join(self.ROOT_FOLDER, 'models/ml_hate_speech_classifier') } # Load a trained model that you have fine-tuned self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") self.bert_model = 'bert-base-multilingual-uncased' self.model_file = os.path.join( DIRECTORIES['ml_hate_speech_path'], 'pytorch_model_epoch_20_seqlen_256.bin') # self.model_file = 'pytorch_model_epoch_20_seqlen_256.bin' print('model_file', self.model_file) print('model_dir', os.listdir(os.path.join(self.ROOT_FOLDER, 'models'))) print('model_dir_s', os.listdir(DIRECTORIES['ml_hate_speech_path'])) print(os.path.isfile(self.model_file)) if not os.path.isfile(self.model_file): print('Please Download the model ...') exit(0) # # model_download_file = "sh " + self.ROOT_FOLDER+'/model_download.sh' # os.system('ls /usr/src/app/project/') # print('model_download_file ',model_download_file, os.path.isfile(model_download_file)) # os.system(model_download_file) # print(self.model_file) if torch.cuda.is_available(): model_state_dict = torch.load(self.model_file) else: print('Loading model ...', self.model_file) model_state_dict = torch.load(self.model_file, map_location='cpu') tokenizer_file = DIRECTORIES[ 'ml_hate_speech_path'] + '/' + self.bert_model + '/vocab.txt' config_file = DIRECTORIES[ 'ml_hate_speech_path'] + '/' + self.bert_model + '/bert_config.json' bert_model_file = DIRECTORIES[ 'ml_hate_speech_path'] + '/' + self.bert_model + '/' #+'-pytorch_model.bin' self.tokenizer = BertTokenizer.from_pretrained(tokenizer_file) config = BertConfig.from_json_file(config_file) self.model = BertForSequenceClassification.from_pretrained( bert_model_file, state_dict=model_state_dict, num_labels=2) # self.model = BertForSequenceClassification.from_pretrained(self.bert_model, state_dict=model_state_dict, # num_labels=2) self.model.to(self.device)
def load_model(model_type): checkpoint = torch.load(f'checkpoints/{model_type}.pt', map_location='cpu') config = BertConfig.from_json_file('models/config.json') model = Summarizer(args=None, device="cpu", load_pretrained_bert=False, bert_config=config) model.load_cp(checkpoint) return model
def evaluate_config(): # args = parser.parse_args() data_dir = '/home/deepracer/DeepRacer/AI_Samurai/Esun_AML/NER_BERT_pytorch/temp' bert_model_dir = '/home/deepracer/DeepRacer/AI_Samurai/Esun_AML/NER_BERT_pytorch/bert-base-chinese-pytorch' model_dir = '/home/deepracer/DeepRacer/AI_Samurai/Esun_AML/NER_BERT_pytorch/experiments/base_model' seed = 23 # help="random seed for initialization" restore_file = 'best' # help="name of the file in `model_dir` containing weights to load" multi_gpu = False # action='store_true', help="Whether to use multiple GPUs if available" fp16 = False # action='store_true', help="Whether to use 16-bit float precision instead of 32-bit" # Load the parameters from json file json_path = os.path.join(model_dir, 'params.json') assert os.path.isfile( json_path), "No json configuration file found at {}".format(json_path) params = Params(json_path) # Use GPUs if available params.device = torch.device( 'cuda' if torch.cuda.is_available() else 'cpu') params.n_gpu = torch.cuda.device_count() params.multi_gpu = multi_gpu # Set the random seed for reproducible experiments random.seed(seed) torch.manual_seed(seed) if params.n_gpu > 0: torch.cuda.manual_seed_all(seed) # set random seed for all GPUs params.seed = seed # Set the logger set_logger(os.path.join(model_dir, 'evaluate.log')) # Create the input data pipeline logging.info("Loading the dataset...") # Initialize the DataLoader data_loader = DataLoader(data_dir, bert_model_dir, params, token_pad_idx=0) logging.info("- done.") # Define the model config_path = os.path.join(bert_model_dir, 'bert_config.json') config = BertConfig.from_json_file(config_path) model = BertForTokenClassification(config, num_labels=len(params.tag2idx)) model.to(params.device) # Reload weights from the saved file load_checkpoint(os.path.join(model_dir, restore_file + '.pth.tar'), model) if fp16: model.half() if params.n_gpu > 1 and multi_gpu: model = torch.nn.DataParallel(model) return data_loader, model, params
def load_model(): config = BertConfig.from_json_file(MODEL_CONFIG_PATH) model = BertForTokenClassification(config, num_labels=len(LABELS)) state_dict = torch.load(MODEL_PATH) model.load_state_dict(state_dict) model.eval() return ( model, BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True), )
def __init__(self, args, device, load_pretrained_bert=False, bert_config=None): super(Summarizer, self).__init__() self.args = args self.device = device self.bert = Bert(args, load_pretrained_bert=True, bert_config=BertConfig.from_json_file( args.bert_config_path)) self.bert_config = BertConfig.from_json_file(args.bert_config_path) if (args.encoder == 'classifier'): self.encoder = Classifier(self.bert_config.hidden_size) elif (args.encoder == 'transformer'): self.encoder = TransformerInterEncoder( self.bert_config.hidden_size, args.ff_size, args.heads, args.dropout, args.inter_layers) elif (args.encoder == 'rnn'): self.encoder = RNNEncoder(bidirectional=True, num_layers=1, input_size=self.bert_config.hidden_size, hidden_size=args.rnn_size, dropout=args.dropout) elif (args.encoder == 'baseline'): bert_config = BertConfig(self.bert_config.vocab_size, hidden_size=args.hidden_size, num_hidden_layers=6, num_attention_heads=8, intermediate_size=args.ff_size) self.bert.model = BertModel(bert_config) self.encoder = Classifier(self.bert_config.hidden_size) if args.param_init != 0.0: for p in self.encoder.parameters(): p.data.uniform_(-args.param_init, args.param_init) if args.param_init_glorot: for p in self.encoder.parameters(): if p.dim() > 1: xavier_uniform_(p) self.to(device)
def load_BFTC_from_TF_ckpt(bert_config, ckpt_path, num_labels): """ Helper function for loading model - workaround to prevent error """ config = BertConfig.from_json_file(bert_config) model = BertForPreTraining(config) load_tf_weights_in_bert(model, ckpt_path) state_dict=model.state_dict() model = BertForTokenClassification(config, num_labels=num_labels) # Load from a PyTorch state_dict old_keys = [] new_keys = [] for key in state_dict.keys(): new_key = None if 'gamma' in key: new_key = key.replace('gamma', 'weight') if 'beta' in key: new_key = key.replace('beta', 'bias') if new_key: old_keys.append(key) new_keys.append(new_key) for old_key, new_key in zip(old_keys, new_keys): state_dict[new_key] = state_dict.pop(old_key) missing_keys = [] unexpected_keys = [] error_msgs = [] # copy state_dict so _load_from_state_dict can modify it metadata = getattr(state_dict, '_metadata', None) state_dict = state_dict.copy() if metadata is not None: state_dict._metadata = metadata def load(module, prefix=''): local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {}) module._load_from_state_dict( state_dict, prefix, local_metadata, True, missing_keys, unexpected_keys, error_msgs) for name, child in module._modules.items(): if child is not None: load(child, prefix + name + '.') start_prefix = '' if not hasattr(model, 'bert') and any(s.startswith('bert.') for s in state_dict.keys()): start_prefix = 'bert.' load(model, prefix=start_prefix) if len(missing_keys) > 0: print("Weights of {} not initialized from pretrained model: {}".format( model.__class__.__name__, missing_keys)) if len(unexpected_keys) > 0: print("Weights from pretrained model not used in {}: {}".format( model.__class__.__name__, unexpected_keys)) if len(error_msgs) > 0: raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format( model.__class__.__name__, "\n\t".join(error_msgs))) return model
def _convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path): # adapated from https://github.com/huggingface/pytorch-pretrained-BERT/blob/master/pytorch_pretrained_bert/convert_tf_checkpoint_to_pytorch.py#L30 # Initialise PyTorch model config = BertConfig.from_json_file(bert_config_file) model = BertForPreTraining(config) # Load weights from tf checkpoint _load_tf_weights_in_bert(model, tf_checkpoint_path) # Save pytorch-model torch.save(model.state_dict(), pytorch_dump_path)
def test_convert_onnx(self): model = BertModel(BertConfig.from_json_file(BERT_CONFIG_PATH)) model.train(False) output = torch.onnx.export( model, self.org_dummy_input, self.model_onnx_path, verbose=True, operator_export_type=OPERATOR_EXPORT_TYPE, input_names=['input_ids', 'token_type_ids', 'attention_mask']) print("Export of torch_model.onnx complete!")
def load_biobert_model(biobert_pth, device): """Read saved state dict for biobert model on disk Args: biobert_pth: str, folder path where model state dictionary and config file are saved """ bert_config_file = os.path.join(biobert_pth, 'bert_config.json') config = BertConfig.from_json_file(bert_config_file) print("Building PyTorch model from configuration: {}".format(str(config))) model = BertForPreTraining(config) model.load_state_dict(torch.load(os.path.join(biobert_pth, 'biobert_statedict.pkl'), map_location=device)) return model
def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path): # Initialise PyTorch model config = BertConfig.from_json_file(bert_config_file) print("Building PyTorch model from configuration: {}".format(str(config))) model = BertForPreTraining(config) # Load weights from tf checkpoint load_tf_weights_in_bert(model, tf_checkpoint_path) # Save pytorch-model print("Save PyTorch model to {}".format(pytorch_dump_path)) torch.save(model.state_dict(), pytorch_dump_path)
def __init__(self, args, w=None, b=None): # 初始化模型,建立encoder和decoder super(Summarizer, self).__init__() if args.mode == "train": self.encoder = BertModel.from_pretrained('bert-base-cased', cache_dir="./temp") self.w = w self.b = b elif args.mode == "test": config = BertConfig.from_json_file(args.predict_config) self.encoder = BertModel(config) self.args = args # we choose same hiedden_size with bert embedding self.decoder = nn.GRU(input_size=768, hidden_size=768, num_layers=1)
def __init__(self, args, Y): super(Bert_seq_cls, self).__init__() print("loading pretrained bert from {}".format(args.bert_dir)) config_file = os.path.join(args.bert_dir, 'bert_config.json') self.config = BertConfig.from_json_file(config_file) print("Model config {}".format(self.config)) self.bert = BertModel.from_pretrained(args.bert_dir) self.dim_reduction = nn.Linear(self.config.hidden_size, args.num_filter_maps) self.dropout = nn.Dropout(self.config.hidden_dropout_prob) self.classifier = nn.Linear(args.num_filter_maps, Y) self.apply(self.init_bert_weights)
def __init__(self, args, tagset_size, embedding_dim, hidden_dim, rnn_layers, dropout_ratio, dropout1, use_cuda=False): super(BERT_LSTM_CRF, self).__init__() self.embedding_dim = embedding_dim self.hidden_dim = hidden_dim self.word_embeds = BertModel(config=BertConfig.from_json_file(args.bert_config_json)) # print(self.word_embeds) self.word_embeds.load_state_dict(torch.load('./ckpts/9134_bert_weight.bin')) self.lstm = nn.LSTM(embedding_dim, hidden_dim, num_layers=rnn_layers, bidirectional=True, dropout=dropout_ratio, batch_first=True) self.rnn_layers = rnn_layers self.dropout1 = nn.Dropout(p=dropout1) self.crf = CRF(target_size=tagset_size, average_batch=True, use_cuda=use_cuda) self.liner = nn.Linear(hidden_dim*2, tagset_size+2) self.tagset_size = tagset_size
def __init__(self, hidden_size=768, linear_out=2, batch_first=True): super(Classifier, self).__init__() self.output_model_file = "lm/pytorch_model.bin" self.output_config_file = "lm/config.json" self.tokenizer = BertTokenizer.from_pretrained("lm", do_lower_case=False) self.config = BertConfig.from_json_file(self.output_config_file) self.model = BertForMaskedLM(self.config) device = torch.device( 'cuda') if torch.cuda.is_available() else torch.device('cpu') self.state_dict = torch.load(self.output_model_file, map_location=device) self.model.load_state_dict(self.state_dict) self.lstm = torch.nn.LSTM(hidden_size, 300) self.linear = torch.nn.Linear(300, linear_out)
def save_state_dict(args, device_id): device = "cpu" if args.visible_gpus == '-1' else "cuda" ckpt = "cnndm_bertsum_classifier_best.pt" logger.info('Loading checkpoint from %s' % ckpt) checkpoint = torch.load(ckpt, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) config = BertConfig.from_json_file(args.bert_config_path) model = Summarizer(args, device, load_pretrained_bert=False, bert_config = config) model.load_cp(checkpoint) model.eval() # save state_dict torch.save(model.state_dict(), "weights.pt")
def load_model(args, device, load_bert=False, checkpoint=None): """Loads a model given by args.encoder using a checkpoint file if specified""" config = BertConfig.from_json_file(args.bert_config_path) if args.encoder == 'bertsum': logger.info('Loading BertSum baseline model') model = BertSum(args, device, load_bert=load_bert, bert_config=config) else: raise ValueError('unknown encoder %s' % args.encoder) # Load checkpointed model if checkpoint: logger.info('Loading checkpoint from %s' % checkpoint) checkpoint = torch.load(checkpoint, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) model.load_cp(checkpoint) # Load optimizer if resuming training, otherwise fresh optimizer if checkpoint and args.mode == 'train': logger.info("Restoring checkpointed optimizer state") #optim = Optimizer.from_opt(model, args, checkpoint) optim = checkpoint['optim'] saved_optimizer_state_dict = optim.optimizer.state_dict() optim.set_parameters(list(model.named_parameters())) optim.optimizer.load_state_dict(saved_optimizer_state_dict) if args.visible_gpus != '-1': for state in optim.optimizer.state.values(): for k, v in state.items(): if torch.is_tensor(v): state[k] = v.cuda() else: logger.info("Using fresh optimizer state") optim = Optimizer.from_opt(model, args) return model, optim
def embed_data(x_data): device = torch.device( 'cuda') if torch.cuda.is_available() else torch.device('cpu') output_model_file = "lm/pytorch_model.bin" output_config_file = "lm/config.json" tokenizer = BertTokenizer.from_pretrained("lm", do_lower_case=False) config = BertConfig.from_json_file(output_config_file) model = BertForMaskedLM(config) state_dict = torch.load(output_model_file, map_location=device) model.load_state_dict(state_dict) entries = [] data_iterator = tqdm(x_data, desc='Loading embeddings') for entry in data_iterator: entries.append(get_embeddings(entry, model)) entries = torch.stack(entries) return entries
def evaluate(args, device_id, checkpoint, step, mode): """ Evaluates a model in either validation or test mode """ if mode not in ['valid', 'test']: raise ValueError('mode must be [test, valid], got %s' % mode) device = "cpu" if args.visible_gpus == '-1' else "cuda" if checkpoint == '': checkpoint = args.test_from logger.info("Evaluating (%s) from checkpoint %s", mode, checkpoint) config = BertConfig.from_json_file(args.bert_config_path) model, _ = load_model(args, device, load_bert=False, checkpoint=checkpoint) model.eval() dataset = data.load(args, mode, device) trainer = build_trainer(args, device_id, model, None) if mode == 'valid': stats = trainer.validate(dataset, step) else: stats = trainer.test(dataset, step) return stats
def __init__(self): super(BioBertModel, self).__init__() self.config_bert = BertConfig.from_json_file( BIO_BERT_DIR + 'biobert_v1.1_pubmed/bert_config.json') self.bert_model = BertModel.from_pretrained(BIO_BERT_DIR + 'biobert_v1.1_pubmed/', from_tf=True) self.dropout = nn.Dropout(DROP_PROB) self.linear_mapping = nn.Linear(BERT_EMBEDDING_DIM, BERT_INTERMED_DIM) self.relu = nn.ReLU() self.batch_norm = nn.BatchNorm1d(BERT_INTERMED_DIM) self.intermed_linear_mapping = nn.Linear(BERT_INTERMED_DIM, BERT_INTERMED_DIM // 2) self.intermed_batch_norm = nn.BatchNorm1d(BERT_INTERMED_DIM // 2) self.intermed_linear_mapping_1 = nn.Linear(BERT_INTERMED_DIM // 2, BERT_INTERMED_DIM // 4) self.intermed_batch_norm_1 = nn.BatchNorm1d(BERT_INTERMED_DIM // 4) self.classification_layer = nn.Linear(BERT_INTERMED_DIM // 2, 1) self.sigmoid = nn.Sigmoid()
def getTranslator(): # set up model device = "cpu" logger.info('Loading checkpoint from %s' % args.test_from) checkpoint = torch.load(args.test_from, map_location=lambda storage, loc: storage) opt = vars(checkpoint['opt']) for k in opt.keys(): if (k in model_flags): setattr(args, k, opt[k]) print(args) config = BertConfig.from_json_file(args.bert_config_path) model = Summarizer(args, device, load_pretrained_bert=False, bert_config=config) model.load_cp(checkpoint) model.eval() return build_trainer(args, -1, model, None)