d_model=args.d_model,
d_inner=args.d_inner,
num_layers=args.num_layers,
num_attn_heads=args.num_heads,
ffn_dropout=args.ffn_dropout,
vocab_size=vocab_size,
max_seq_length=args.max_seq_length,
embedding_dropout=args.embedding_dropout,
learn_positional_encodings=True,
hidden_act="gelu",
**dec_first_sublayer_params)
decoder.restore_from(args.restore_from, local_rank=args.local_rank)
t_log_softmax = nemo_nlp.TokenClassifier(args.d_model,
num_classes=vocab_size,
num_layers=1,
log_softmax=True)
beam_translator = nemo_nlp.BeamSearchTranslatorNM(
decoder=decoder,
log_softmax=t_log_softmax,
max_seq_length=args.max_seq_length,
beam_size=args.beam_size,
length_penalty=args.len_pen,
bos_token=tokenizer.bos_id(),
pad_token=tokenizer.pad_id(),
eos_token=tokenizer.eos_id())
loss = nemo_nlp.PaddedSmoothedCrossEntropyLossNM(pad_id=0, smoothing=0.1)
loss_eval = nemo_nlp.PaddedSmoothedCrossEntropyLossNM(pad_id=0, smoothing=0.0)
pretrained_bert_model = nemo_nlp.huggingface.BERT(
pretrained_model_name=args.pretrained_bert_model, factory=nf)
else:
""" Use this if you're using a BERT model that you pre-trained yourself.
Replace BERT-STEP-150000.pt with the path to your checkpoint.
"""
tokenizer = SentencePieceTokenizer(model_path=tokenizer_model)
tokenizer.add_special_tokens(["[MASK]", "[CLS]", "[SEP]"])
bert_model = nemo_nlp.huggingface.BERT(config_filename=args.bert_config,
factory=nf)
pretrained_bert_model.restore_from(args.bert_checkpoint)
hidden_size = pretrained_bert_model.local_parameters["hidden_size"]
ner_classifier = nemo_nlp.TokenClassifier(hidden_size=hidden_size,
num_classes=args.num_classes,
dropout=args.fc_dropout)
ner_loss = nemo_nlp.TokenClassificationLoss(num_classes=args.num_classes)
def create_pipeline(input_file,
max_seq_length=args.max_seq_length,
batch_size=args.batch_size,
local_rank=args.local_rank,
num_gpus=args.num_gpus):
data_layer = nemo_nlp.BertTokenClassificationDataLayer(
tokenizer=tokenizer,
input_file=input_file,
max_seq_length=max_seq_length,
dataset_type=args.dataset_type,
batch_size=batch_size,
"""
pretrained_bert_model = nemo_nlp.huggingface.BERT(
pretrained_model_name=args.pretrained_bert_model)
hidden_size = pretrained_bert_model.local_parameters["hidden_size"]
tokenizer = NemoBertTokenizer(args.pretrained_bert_model)
data_layer = nemo_nlp.BertTokenClassificationInferDataLayer(
queries=args.queries,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
batch_size=1)
punct_classifier = \
nemo_nlp.TokenClassifier(hidden_size=hidden_size,
num_classes=len(punct_labels_dict),
dropout=args.fc_dropout,
num_layers=args.punct_num_fc_layers,
name='Punctuation')
capit_classifier = nemo_nlp.TokenClassifier(hidden_size=hidden_size,
num_classes=len(capit_labels_dict),
dropout=args.fc_dropout,
name='Capitalization')
input_ids, input_type_ids, input_mask, loss_mask, subtokens_mask = data_layer()
hidden_states = pretrained_bert_model(input_ids=input_ids,
token_type_ids=input_type_ids,
attention_mask=input_mask)
punct_logits = punct_classifier(hidden_states=hidden_states)
See the list of pretrained models, call:
nemo_nlp.huggingface.BERT.list_pretrained_models()
"""
pretrained_bert_model = nemo_nlp.huggingface.BERT(
pretrained_model_name=args.pretrained_bert_model)
hidden_size = pretrained_bert_model.local_parameters["hidden_size"]
tokenizer = NemoBertTokenizer(args.pretrained_bert_model)
data_layer = nemo_nlp.BertTokenClassificationInferDataLayer(
queries=args.queries,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
batch_size=1)
classifier = nemo_nlp.TokenClassifier(hidden_size=hidden_size,
num_classes=len(labels_dict),
dropout=args.fc_dropout)
input_ids, input_type_ids, input_mask, _, subtokens_mask = data_layer()
hidden_states = pretrained_bert_model(input_ids=input_ids,
token_type_ids=input_type_ids,
attention_mask=input_mask)
logits = classifier(hidden_states=hidden_states)
###########################################################################
# Instantiate an optimizer to perform `infer` action
evaluated_tensors = nf.infer(
tensors=[logits, subtokens_mask],
checkpoint_dir=args.work_dir,
if args.bert_config is not None:
with open(args.bert_config) as json_file:
config = json.load(json_file)
model = nemo_nlp.huggingface.BERT(**config)
else:
""" Use this if you're using a standard BERT model.
To see the list of pretrained models, call:
nemo_nlp.huggingface.BERT.list_pretrained_models()
"""
model = nemo_nlp.huggingface.BERT(
pretrained_model_name=args.pretrained_bert_model)
hidden_size = model.local_parameters["hidden_size"]
qa_head = nemo_nlp.TokenClassifier(hidden_size=hidden_size,
num_classes=2,
num_layers=1,
log_softmax=False)
squad_loss = nemo_nlp.QuestionAnsweringLoss()
if args.bert_checkpoint is not None:
model.restore_from(args.bert_checkpoint)
if not args.evaluation_only:
train_loss, train_steps_per_epoch, _, _ = \
create_pipeline(
data_dir=args.data_dir,
model=model,
head=qa_head,
loss_fn=squad_loss,
max_query_length=args.max_query_length,
max_seq_length=args.max_seq_length,
doc_stride=args.doc_stride,