def train(args, train_loader, test_loader, test_json):
device = torch.device(
"cuda") if torch.cuda.is_available() else torch.device("cpu")
print("loading bert.")
model = BertForTokenClassification.from_pretrained("bert-base-cased",
num_labels=3)
model.to(device)
optim = AdamW(model.parameters(), lr=args.lr)
print("loaded. staring training.")
# best_rationale_acc = 0
for epoch in range(args.num_epoch):
for batch in tqdm(train_loader):
optim.zero_grad()
model.train()
model.zero_grad()
input_ids = batch["input_ids"].to(device)
attention_mask = batch["attention_mask"].to(device)
labels = batch["labels"].to(device)
outputs = model(input_ids,
attention_mask=attention_mask,
labels=labels)
loss = outputs[0]
loss.backward()
optim.step()
if epoch % 3 == 2 and not args.no_logs:
total, correct, pred_rationale = evaluate(args, model, test_loader)
print_score(args, total, correct)
print_human_vs_model(test_json, pred_rationale)
return model
def get_text_reader(reader_name, task, num_labels):
# AILAW Corpus is korean dataset.
# So, model is fixed to Korean Model such as multilingual-BERT, kobert, koelectra, etc.
if reader_name == "bert":
if task == "classification":
model_name = "bert-base-multilingual-cased"
text_reader = BertForSequenceClassification.from_pretrained(model_name, num_labels=num_labels)
else: # ner
model_name = "bert-base-multilingual-cased"
text_reader = BertForTokenClassification.from_pretrained(model_name, num_labels=num_labels)
elif reader_name == "kobert":
if task == "classification":
model_name = "monologg/kobert"
text_reader = BertForSequenceClassification.from_pretrained(model_name, num_labels=num_labels)
else: # ner
model_name = "monologg/kobert"
text_reader = BertForTokenClassification.from_pretrained(model_name, num_labels=num_labels)
elif reader_name == "koelectra":
if task == "classification":
model_name = "monologg/koelectra-base-discriminator"
text_reader = ElectraForSequenceClassification.from_pretrained(model_name, num_labels=num_labels)
else: # ner
model_name = "monologg/koelectra-base-discriminator"
text_reader = ElectraForTokenClassification.from_pretrained(model_name, num_labels=num_labels)
else:
raise KeyError(reader_name)
return text_reader
def launch_bert(training_flag, test_flag):
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
if training_flag is not None:
model = BertForTokenClassification.from_pretrained(
'bert-base-uncased', num_labels=len(tags_vals))
## ---------12 . Optimizer -> weight regularization is a solution to reduce the overfitting of a deep learning
"""
Last keras optimization 2020 (rates from 0.01 seem to be best hyperparamater )for weight regularization for weights layers
from keras.layers import LSTM
from keras.regularizers import l2
model.add(LSTM(32, kernel_regularizer=l2(0.01), recurrent_regularizer=l2(0.01), bias_regularizer=l2(0.01)))
Note : BERT not include beta an gamma parametres for optimization
"""
FULL_FINETUNING = True
if FULL_FINETUNING:
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'gamma', 'beta']
optimizer_grouped_parameters = [{
'params': [
p for n, p in param_optimizer
if not any(nd in n for nd in no_decay)
],
'weight_decay_rate':
0.01
}, {
'params': [
p for n, p in param_optimizer
if any(nd in n for nd in no_decay)
],
'weight_decay_rate':
0.0
}]
else:
param_optimizer = list(model.classifier.named_parameters())
optimizer_grouped_parameters = [{
"params": [p for n, p in param_optimizer]
}]
optimizer = AdamW(optimizer_grouped_parameters, lr=args.lr)
launch_training(training_path=args.training_data,
training_epochs=4,
valid_path=args.validate_data,
training_batch_size=1,
model=model,
model_path=args.save + '/config.json',
tokenizer=tokenizer,
optimizer=optimizer)
if test_flag is not None:
if (args.save is not None):
config = BertConfig.from_json_file(args.save + '/config.json')
model = BertForTokenClassification.from_pretrained(
pretrained_model_name_or_path=args.save + '/pytorch_model.bin',
config=config)
else:
model = BertForTokenClassification.from_pretrained(
'bert-base-uncased', num_labels=len(tags_vals))
launch_test_directory(test_path=test_flag,
model=model,
tokenizer=tokenizer)
def main(num_epochs, learning_rate):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
classes = ["B", "I", "O"]
tokenizer = BertTokenizer.from_pretrained('bert-base-cased',
do_lower_case=False)
tag_to_idx = {t: i for i, t in enumerate(classes)}
tag_to_idx['[PAD]'] = -100
idx_to_tag = {i: t for t, i in tag_to_idx.items()}
train_dataloader, dev_dataloader, dev_sentences, test_dataloader, test_sentences = parse_data(
tokenizer, tag_to_idx, batch_size=16)
print('data loaded and tokenized')
model = BertForTokenClassification.from_pretrained('bert-base-cased',
num_labels=len(classes))
model.to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
print('model instantiated')
model, dev_preds = train_model(tokenizer, tag_to_idx, model, num_epochs,
train_dataloader, optimizer, device,
dev_dataloader, idx_to_tag)
test_preds = evaluate(model, test_dataloader, device, idx_to_tag)
save_preds('dev_preds.txt', dev_preds, dev_sentences)
save_preds('test_preds.txt', test_preds, test_sentences)
def main():
# 各トークンを以下の13クラスのいずれかに分類するような固有表現抽出をしたい．
labels = [
'B-corporation', 'B-creative-work', 'B-group', 'B-location',
'B-person', 'B-product', 'I-corporation', 'I-creative-work', 'I-group',
'I-location', 'I-person', 'I-product', 'O'
]
id2label = {i: label for i, label in enumerate(labels)}
# label2id = {label: i for i, label in enumerate(labels)}
# 利用する学習済みBERTモデルの名前を指定する．
model_name = 'bert-large-cased'
# 学習済みモデルに対応したトークナイザを生成する．
tokenizer = BertTokenizer.from_pretrained(
pretrained_model_name_or_path=model_name, )
# 学習済みモデルから各トークン分類用モデルのインスタンスを生成する．
model = BertForTokenClassification.from_pretrained(
pretrained_model_name_or_path=model_name,
id2label=id2label, # 各トークンに対する出力を13次元にしたいのでこれを渡す．
)
# 一部の重みが初期化されていませんよという警告が出るが（クラス分類する層が
# 初期化されていないのは当然）面倒なので無視する．
print('◆ 適当な文章をモデルに流してみる．→ 14トークン×13クラスの予測結果になっている（サイズが）．')
sentence = 'The Empire State Building officially opened on May 1, 1931.'
inputs = torch.tensor([tokenizer.encode(sentence)]) # ID列をテンソル化して渡す．
outputs = model(inputs)
print(outputs[0].size())
def _train_bert(training_data_retrieval_func):
tokenizer = BertTokenizerFast.from_pretrained(BERT_BASE_MODEL)
tokenizer.add_tokens(ADDITIONAL_SPECIAL_TOKENS)
tokens, labels = training_data_retrieval_func()
train_dataset = _get_datasets(tokens, labels, tokenizer)
model = BertForTokenClassification.from_pretrained(
BERT_BASE_MODEL, num_labels=len(ALL_LABEL_IDS))
model.resize_token_embeddings(len(tokenizer))
run_id = '{}_{}'.format(datetime.datetime.now().strftime('%Y%m%d-%H%M%S'),
utils.get_config('logging.filename'))
training_args = TrainingArguments(
output_dir=f'./bert/results/{run_id}',
logging_dir=f'./bert/logs/{run_id}',
logging_steps=500,
save_steps=2000,
per_device_train_batch_size=8,
num_train_epochs=3,
learning_rate=5e-5,
warmup_steps=0,
weight_decay=0,
)
trainer = Trainer(model=model,
args=training_args,
train_dataset=train_dataset)
trainer.train()
path_to_model = utils._get_cache_path('bert_for_SE_tagging')
model.save_pretrained(path_to_model)
tokenizer.save_pretrained(path_to_model)
def main():
args = docopt(__doc__)
processors = {
'multi-sents': BertMultiSentProcessor,
'uni-sent': BertUniSentProcessor
}
max_input_len = int(args['--max-input-len'])
tokenizer = BertWordPieceTokenizer(str(args['--path-to-vocab']))
processor_constructor = processors[str(args['--mode'])]
processor = processor_constructor(max_input_len, tokenizer)
if args['--ensemble']:
bert_model = BertEnsemble.load_trained(str(args['--path-to-model-dir']))
elif args['--crf']:
bert_model = BertWithCRF.from_pretrained(str(args['--path-to-model-dir']))
elif not args['--rule']:
bert_model = BertForTokenClassification.from_pretrained(str(args['--path-to-model-dir']))
device_no = int(args['--gpu'])
device = torch.device(f'cuda:{device_no}') if device_no > -1 else torch.device('cpu')
if args['--crf']:
bert_extractor = BertWithCRFExtractor(bert_model, tokenizer, max_input_len, device)
elif not args['--rule']:
bert_extractor = BertExtractor(bert_model, tokenizer, max_input_len, device)
else:
bert_extractor = RuleExtractor()
corpus = read_corpus(str(args['--path-to-corpus-dir']))
ents_table = build_ents_table(corpus, processor, bert_extractor, batch_size=int(args['--batch-size']))
ents_table.to_csv(str(args['--path-to-output']), index=False, sep='\t')
return
def __init__(self,
config_name: str,
model_name: str = None,
num_tags: int = 2,
batch_first: bool = True) -> None:
self.batch_first = batch_first
if not os.path.exists(config_name):
raise ValueError("未找到模型配置文件 '{}'".format(config_name))
else:
self.config_name = config_name
if model_name is not None:
if not os.path.exists(model_name):
raise ValueError("未找到模型预训练参数文件 '{}'".format(model_name))
else:
self.model_name = model_name
else:
self.model_name = None
super().__init__()
self.bert_config = BertConfig.from_pretrained(self.config_name)
self.bert_config.num_labels = num_tags
self.model_kwargs = {'config': self.bert_config}
if self.model_name is not None:
self.bertModel = BertForTokenClassification.from_pretrained(
self.model_name, **self.model_kwargs)
else:
self.bertModel = BertForTokenClassification(self.bert_config)
self.crf_model = CRF(num_tags=num_tags, batch_first=batch_first)
def __init__(self):
super(BERTClass, self).__init__()
config = BertConfig.from_pretrained("./bert-base-uncased",
num_labels=len(list(
tag2idx.keys())))
self.l1 = BertForTokenClassification.from_pretrained(
'./bert-base-uncased', config=config)
def load_model(args, test):
# if the model is for testing, attempt to load previous arguments
if test:
try:
prev_args = torch.load(
os.path.join(args.model_dir, "train_args.bin")
)
args.max_length = prev_args.max_length
args.do_lower_case = prev_args.do_lower_case
args.keep_accents = prev_args.keep_accents
except FileNotFoundError:
pass
tokenizer = BertTokenizer.from_pretrained(
args.model_dir,
do_lower_case=args.do_lower_case,
keep_accents=args.keep_accents,
)
model = BertForTokenClassification.from_pretrained(
args.model_dir,
finetuning_task="conll2002",
num_labels=len(LABEL_LIST),
).to(args.device)
if args.n_gpu > 1:
model = torch.nn.DataParallel(model)
return model, tokenizer
def load_model(self,
model_filepath,
config_filepath,
pretrained_model="bert-base-cased"):
"""
Load cybert model.
:param model_filepath: Filepath of the model (.pth or .bin) to
be loaded
:type model_filepath: str
:param label_map_filepath: Config file (.json) to be
used
:type label_map_filepath: str
:param pretrained_model: Name of pretrained model to be loaded from
transformers
repo, default is bert-base-cased
:type pretrained_model: str
Examples
--------
>>> from clx.analytics.cybert import Cybert
>>> cyparse = Cybert()
>>> cyparse.load_model('/path/to/model.pth', '/path/to/config.json')
"""
with open(config_filepath) as f:
config = json.load(f)
self._label_map = {int(k): v for k, v in config["id2label"].items()}
model_state_dict = torch.load(model_filepath)
self._model = BertForTokenClassification.from_pretrained(
pretrained_model,
state_dict=model_state_dict,
num_labels=len(self._label_map),
)
self._model.cuda()
self._model.eval()
def __init__(self,
num_labels=len(id2label.keys()),
from_pretrained='bert-base-uncased'):
super(BertForValueExtraction, self).__init__()
print(f"Loading BertForTokenClassification as {from_pretrained}")
self.token_classifier = BertForTokenClassification.from_pretrained(
from_pretrained, num_labels=num_labels, return_dict=True)
def __init__(self, hparams: Union[Dict, Namespace]):
# NOTE: internal code may pass hparams as dict **kwargs
if isinstance(hparams, Dict):
hparams = Namespace(**hparams)
self.label_ids_to_label = LabelTokenAligner.get_ids_to_label(
hparams.labels)
num_labels = len(self.label_ids_to_label)
super().__init__()
# Enable to access arguments via self.hparams
self.save_hyperparameters(hparams)
self.step_count = 0
self.output_dir = Path(self.hparams.output_dir)
self.cache_dir = self.hparams.cache_dir if self.hparams.cache_dir else None
if self.cache_dir is not None and not os.path.exists(
self.hparams.cache_dir):
os.mkdir(self.cache_dir)
# AutoTokenizer
# trf>=4.0.0: PreTrainedTokenizerFast by default
# NOTE: AutoTokenizer doesn't load PreTrainedTokenizerFast...
self.tokenizer_name = self.hparams.model_name_or_path
self.tokenizer = BertTokenizerFast.from_pretrained(
self.tokenizer_name,
cache_dir=self.cache_dir,
tokenize_chinese_chars=False,
strip_accents=False,
)
# AutoConfig
config_name = self.hparams.model_name_or_path
self.config: PretrainedConfig = BertConfig.from_pretrained(
config_name,
**({
"num_labels": num_labels
} if num_labels is not None else {}),
cache_dir=self.cache_dir,
)
extra_model_params = (
"encoder_layerdrop",
"decoder_layerdrop",
"dropout",
"attention_dropout",
)
for p in extra_model_params:
if getattr(self.hparams, p, None) and hasattr(self.config, p):
setattr(self.config, p, getattr(self.hparams, p, None))
# AutoModelForTokenClassification
self.model: PreTrainedModel = BertForTokenClassification.from_pretrained(
self.hparams.model_name_or_path,
from_tf=bool(".ckpt" in self.hparams.model_name_or_path),
config=self.config,
cache_dir=self.cache_dir,
)
self.scheduler = None
self.optimizer = None
def __init__(self, model_path, num_labels, label_map, device):
super().__init__()
self.model = BertForTokenClassification.from_pretrained(model_path, num_labels=num_labels).to(device)
self.transitions = torch.nn.Parameter(torch.randn(num_labels, num_labels))
# Ok, so we're going to add some constraints here
self.label_map = label_map
self.num_labels = num_labels
def __init__(self):
super(BertClass, self).__init__()
self.l1 = BertForTokenClassification.from_pretrained(
'bert-base-chinese',
num_labels=21,
output_attention=False,
output_hidden_states=False)
def __init__(self,
config_name: str,
model_name: str = None,
num_tags: int = 2,
batch_first: bool = True) -> None:
self.batch_first = batch_first
if not os.path.exists(config_name):
raise ValueError('{} config file not found'.format(config_name))
else:
self.config_name = config_name
if model_name is not None:
if not os.path.exists(model_name):
raise ValueError(' {} model file not found'.format(model_name))
else:
self.model_name = model_name
else:
self.model_name = None
if num_tags <= 0:
raise ValueError(f'invalid number of tags:{num_tags}')
super().__init__()
#bert config文件
self.bert_config = BertConfig.from_pretrained(self.config_name)
self.bert_config.num_tags = num_tags
self.model_kwargs = {'config': self.bert_config}
if self.model_name is not None:
self.bertModel = BertForTokenClassification.from_pretrained(
self.model_name, **self.model_kwargs)
else:
self.bertModel = BertForTokenClassification(self.bert_config)
self.crfModel = CRF(num_tags=num_tags, batch_first=batch_first)
def main():
data_table = pd.read_csv("train_table.csv")
#define bert tokenizer && bert model
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertForTokenClassification.from_pretrained(
'bert-base-uncased', output_hidden_states=True)
model.eval()
print(model.config)
data_embeddings = {}
for index, row in data_table.iterrows():
print(index)
#get sentence_tokens && label_tokens
sentence = row[1]
span = row[2]
#get bert embeddings
span_embeddings = get_bert_embedding(model, tokenizer, sentence, span)
#non-propagandistic span embeddings
if pd.isnull(span):
data_embeddings[sentence] = span_embeddings
#propagandistic span embeddings
else:
data_embeddings[sentence] = (span, row[3], row[4], span_embeddings)
print("Writing to output file...")
torch.save(data_embeddings, "data_embeddings.pt")
print("Done.")
def get_predictions(filename, outputName):
label_list = ['O', 'B-CLEntity', 'I-CLEntity', 'L-CLEntity', 'U-CLEntity']
model = BertForTokenClassification.from_pretrained(
"bert_ner_finetuned_iliad-with-gpu-pattern2.model")
tokenizer = BertTokenizerFast.from_pretrained("bert-base-cased")
book_lines = []
book = open(filename)
for line in book:
line = line.strip()
book_lines.append(line)
book_lines = [line for line in book_lines if line]
pred = []
for line in book_lines:
line_tokens = tokenizer.tokenize(
tokenizer.decode(tokenizer.encode(line)))
line_inputs = tokenizer.encode(line, return_tensors="pt")
line_outputs = model(line_inputs).logits
line_predictions = torch.argmax(line_outputs, dim=2)
line_pred_labels = []
for prediction in line_predictions[0].numpy():
line_pred_labels.append(label_list[prediction])
pred.append(line_pred_labels)
with open(outputName, 'w') as f:
f.write(json.dumps(pred))
return
def load(self, dirpath):
""" Loads a trained model from specified folder on disk.
Parameters
----------
dirpath : str
directory from which model artifacts should be loaded
Returns
-------
self
"""
if not os.path.exists(dirpath):
raise ValueError("Model directory not found: {:s}".format(dirpath))
label_mappings = joblib.load(
os.path.join(dirpath, "label_mappings.pkl"))
self.label2id_ = label_mappings["label2id"]
self.id2label_ = label_mappings["id2label"]
self.special_tokens_ = label_mappings["special_tokens"]
self.model_ = BertForTokenClassification.from_pretrained(
dirpath,
num_labels=len(self.label2id_),
output_attentions=False,
output_hidden_states=False)
self.model_.to(self._device)
self.tokenizer_ = BertTokenizer.from_pretrained(
dirpath, do_basic_tokenize=False)
return self
def __init__(self):
self.tag2idx = {
'B-art': 0,
'B-eve': 1,
'B-geo': 2,
'B-gpe': 3,
'B-nat': 4,
'B-org': 5,
'B-per': 6,
'B-tim': 7,
'I-art': 8,
'I-eve': 9,
'I-geo': 10,
'I-gpe': 11,
'I-nat': 12,
'I-org': 13,
'I-per': 14,
'I-tim': 15,
'O': 16
}
self.idx2tag = {}
for key in list(self.tag2idx.keys()):
self.idx2tag[self.tag2idx[key]] = key
self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',
do_lower_case=True)
self.model = BertForTokenClassification.from_pretrained(
"bert-base-uncased", num_labels=len(self.tag2idx))
self.model.load_state_dict(
torch.load("ner.dataset.4.pth", map_location=torch.device('cpu')))
self.model.eval()
def __init__(self, model_name, num_labels, lr):
super().__init__()
self.save_hyperparameters()
self.bert_tc = BertForTokenClassification.from_pretrained(
model_name,
num_labels=num_labels
)
def __init__(self, hparams, user_tokens=['<newline>', '<bullet>']):
super(BertNerSystem, self).__init__()
self.hparams = hparams
self.hparams.model_type = self.hparams.model_type.lower()
tokenizer = BertTokenizer.from_pretrained(
self.hparams.tokenizer_name if self.hparams.tokenizer_name else
self.hparams.model_name_or_path,
never_split=user_tokens,
do_lower_case=self.hparams.do_lower_case,
cache_dir=self.hparams.cache_dir
if self.hparams.cache_dir else None,
)
config = AutoConfig.from_pretrained(
self.hparams.config_name
if self.hparams.config_name else self.hparams.model_name_or_path,
cache_dir=self.hparams.cache_dir
if self.hparams.cache_dir else None,
output_past=not self.hparams.do_train,
num_labels=self.hparams.num_labels,
)
model = BertForTokenClassification.from_pretrained(
self.hparams.model_name_or_path,
from_tf=bool(".ckpt" in self.hparams.model_name_or_path),
config=config,
cache_dir=self.hparams.cache_dir
if self.hparams.cache_dir else None,
)
self.config, self.tokenizer, self.model = config, tokenizer, model
self.loss = [] # for keeping track of average loss
self.metrics = {}
self.vocab = {v: k for k, v in self.tokenizer.get_vocab().items()}
def main(args):
current_path = os.getcwd()
logging.info(f'current python path {current_path}...')
logging.info('Load data...')
with open(f"{args.dataset}/train_dataset.pkl", "rb") as f:
train_dataset = pickle.load(f)
with open(f"{args.dataset}/valid_dataset.pkl", "rb") as f:
valid_dataset = pickle.load(f)
with open(f"{args.dataset}/test_dataset.pkl", "rb") as f:
test_dataset = pickle.load(f)
logging.info('Making dataloader...')
train_loader = DataLoader(
dataset = train_dataset,
batch_size = args.batch_size,
shuffle = True,
collate_fn = lambda x: Bert_dataset.collate_fn(train_dataset, x)
)
valid_loader = DataLoader(
dataset = valid_dataset,
batch_size = args.batch_size,
collate_fn = lambda x: Bert_dataset.collate_fn(valid_dataset, x)
)
test_loader = DataLoader(
dataset = test_dataset,
batch_size = args.batch_size,
collate_fn = lambda x: Bert_dataset.collate_fn(test_dataset, x)
)
logging.info('Load model and parameters...')
model = BertForTokenClassification.from_pretrained("bert-base-chinese",
num_labels = 3,
output_attentions = False,
output_hidden_states = False
)
trainer = Trainer(model, train_loader, valid_loader)
logging.info('Test validation dataset...')
acc, total_loss = trainer.evaluation(test=False)
print(f"device: {trainer.device} classification acc: {acc: .4f} validation loss: {total_loss:.4f}")
logging.info('Start training...')
trainer.training_process(early_stopping = True,
n_iter_no_change = 5,
max_epoch = args.max_epoch,
save_params = True,
verbose = True,
learning_rate = args.learning_rate,
save_paths = args.save_paths)
logging.info('Training ends!')
logging.info('Test validation dataset...')
acc, total_loss = trainer.evaluation(test=False)
print(f"device: {trainer.device} classification acc: {acc: .4f} validation loss: {total_loss:.4f}")
logging.info('Finish!')
def __init__(self, vocab_size, emb_size, hidden_size, num_labels):
super(bert_chinese_ner, self).__init__()
self.bertconfig = BertConfig.from_pretrained(
bert_chinese_ner.model_path,
num_labels=num_labels,
author="lingze")
self.model = BertForTokenClassification.from_pretrained(
bert_chinese_ner.model_path, config=self.bertconfig)
def get_bert_tokenizer_and_model(training_data_retrieval_func):
path_to_model = utils._get_cache_path('bert_for_SE_tagging')
if not path_to_model.is_dir():
_train_bert(training_data_retrieval_func)
tokenizer = BertTokenizerFast.from_pretrained(path_to_model)
model = BertForTokenClassification.from_pretrained(path_to_model)
model.to('cuda')
return tokenizer, model
def load_frozen_bert(
bert_pretrained_model: str,
bert_state_dict: str = None,
bert_config: BertConfig = None) -> BertForTokenClassification:
if bert_state_dict:
fine_tuned_state_dict = torch.load(bert_state_dict)
bert_token_classifier = BertForTokenClassification.from_pretrained(
pretrained_model_name_or_path=bert_pretrained_model,
state_dict=fine_tuned_state_dict,
config=bert_config)
else:
bert_token_classifier = BertForTokenClassification.from_pretrained(
pretrained_model_name_or_path=bert_pretrained_model,
config=bert_config)
for p in bert_token_classifier.bert.parameters():
p.requires_grad = False
return bert_token_classifier
def build_model_layers(self):
''' builds the layers in the model '''
self.bert = BertForTokenClassification.from_pretrained(
"bert-base-cased",
num_labels=self.num_labels,
output_attentions=False,
output_hidden_states=False)
if self.use_crf:
self.crf = CRF(self.tag_pad_idx, self.pad_token, self.tag_names)
def __init__(self):
"""
Inicializa o modelo treinado para a tarefa específicas (extração dos tipos de entidades estabelecidos para
textos de notícias).
Utiliza o tokenizador para língua portuguesa neuralmind/bert-base-portuguese-cased.
"""
super().__init__(
BertForTokenClassification.from_pretrained(
stringify_path(config.diretorio_modelo_bert_finetuned)))
def __init__(self, config):
super().__init__(config)
self.init_weights()
self.bert = BertForTokenClassification.from_pretrained(config.model,
config=config)
logger.error("\n---------The __init__ of this class will always generate warnings. "\
"Even if everything was initialized correctly. "\
"If you loaded a pretrained model using .from_pretrained() "\
"or loaded the state_dict, ignore the previous errors.\n---------")
def load_model(output_path, device, name='model'):
model_path = os.path.join(output_path, name)
if not os.path.exists(model_path):
raise ValueError('Could not find model at: {}. You first need to train a task model.'.format(model_path))
print(' ➤ Loading fully trained model from: {}'.format(model_path))
model = BertForTokenClassification.from_pretrained(model_path)
model.to(device)
return model
