args.data_dir,
args.vocab_size,
args.sample_size,
special_tokens,
'train.txt',
)
if args.tokenizer == "sentence-piece":
nemo.logging.info("To use SentencePieceTokenizer.")
tokenizer = nemo_nlp.SentencePieceTokenizer(
model_path=data_desc.tokenizer_model)
tokenizer.add_special_tokens(special_tokens)
elif args.tokenizer == "nemo-bert":
nemo.logging.info("To use NemoBertTokenizer.")
vocab_file = os.path.join(args.data_dir, 'vocab.txt')
# To train on a Chinese dataset, use NemoBertTokenizer
tokenizer = nemo_nlp.NemoBertTokenizer(vocab_file=vocab_file)
else:
raise ValueError("Please add your tokenizer "
"or use sentence-piece or nemo-bert.")
args.vocab_size = tokenizer.vocab_size
print(vars(args))
bert_model = nemo_nlp.huggingface.BERT(
vocab_size=args.vocab_size,
num_hidden_layers=args.num_hidden_layers,
hidden_size=args.hidden_size,
num_attention_heads=args.num_attention_heads,
intermediate_size=args.intermediate_size,
max_position_embeddings=args.max_seq_length,
hidden_act=args.hidden_act,
)
def test_squad_v1(self):
version_2_with_negative = False
pretrained_bert_model = 'bert-base-uncased'
batch_size = 3
data_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../data/nlp/squad/v1.1'))
max_query_length = 64
max_seq_length = 384
doc_stride = 128
max_steps = 100
lr_warmup_proportion = 0
eval_step_freq = 50
lr = 3e-6
do_lower_case = True
n_best_size = 5
max_answer_length = 20
null_score_diff_threshold = 0.0
tokenizer = nemo_nlp.NemoBertTokenizer(pretrained_bert_model)
neural_factory = nemo.core.NeuralModuleFactory(
backend=nemo.core.Backend.PyTorch,
local_rank=None,
create_tb_writer=False,
)
model = nemo_nlp.huggingface.BERT(
pretrained_model_name=pretrained_bert_model)
hidden_size = model.local_parameters["hidden_size"]
qa_head = nemo_nlp.TokenClassifier(
hidden_size=hidden_size,
num_classes=2,
num_layers=1,
log_softmax=False,
)
squad_loss = nemo_nlp.QuestionAnsweringLoss()
data_layer = nemo_nlp.BertQuestionAnsweringDataLayer(
mode='train',
version_2_with_negative=version_2_with_negative,
batch_size=batch_size,
tokenizer=tokenizer,
data_dir=data_dir,
max_query_length=max_query_length,
max_seq_length=max_seq_length,
doc_stride=doc_stride,
)
(
input_ids,
input_type_ids,
input_mask,
start_positions,
end_positions,
_,
) = data_layer()
hidden_states = model(
input_ids=input_ids,
token_type_ids=input_type_ids,
attention_mask=input_mask,
)
qa_output = qa_head(hidden_states=hidden_states)
loss, _, _ = squad_loss(
logits=qa_output,
start_positions=start_positions,
end_positions=end_positions,
)
data_layer_eval = nemo_nlp.BertQuestionAnsweringDataLayer(
mode='dev',
version_2_with_negative=version_2_with_negative,
batch_size=batch_size,
tokenizer=tokenizer,
data_dir=data_dir,
max_query_length=max_query_length,
max_seq_length=max_seq_length,
doc_stride=doc_stride,
)
(
input_ids_eval,
input_type_ids_eval,
input_mask_eval,
start_positions_eval,
end_positions_eval,
unique_ids_eval,
) = data_layer_eval()
hidden_states_eval = model(
input_ids=input_ids_eval,
token_type_ids=input_type_ids_eval,
attention_mask=input_mask_eval,
)
qa_output_eval = qa_head(hidden_states=hidden_states_eval)
_, start_logits_eval, end_logits_eval = squad_loss(
logits=qa_output_eval,
start_positions=start_positions_eval,
end_positions=end_positions_eval,
)
eval_output = [start_logits_eval, end_logits_eval, unique_ids_eval]
callback_train = nemo.core.SimpleLossLoggerCallback(
tensors=[loss],
print_func=lambda x: print("Loss: {:.3f}".format(x[0].item())),
get_tb_values=lambda x: [["loss", x[0]]],
step_freq=10,
tb_writer=neural_factory.tb_writer,
)
callbacks_eval = nemo.core.EvaluatorCallback(
eval_tensors=eval_output,
user_iter_callback=lambda x, y: eval_iter_callback(x, y),
user_epochs_done_callback=lambda x: eval_epochs_done_callback(
x,
eval_data_layer=data_layer_eval,
do_lower_case=do_lower_case,
n_best_size=n_best_size,
max_answer_length=max_answer_length,
version_2_with_negative=version_2_with_negative,
null_score_diff_threshold=null_score_diff_threshold,
),
tb_writer=neural_factory.tb_writer,
eval_step=eval_step_freq,
)
lr_policy_fn = get_lr_policy(
'WarmupAnnealing',
total_steps=max_steps,
warmup_ratio=lr_warmup_proportion,
)
neural_factory.train(
tensors_to_optimize=[loss],
callbacks=[callback_train, callbacks_eval],
lr_policy=lr_policy_fn,
optimizer='adam_w',
optimization_params={
"max_steps": max_steps,
"lr": lr
},
)
log_dir=args.work_dir,
create_tb_writer=True,
files_to_copy=[__file__],
add_time_to_log_dir=True,
)
if args.tokenizer == "sentencepiece":
try:
tokenizer = nemo_nlp.SentencePieceTokenizer(
model_path=args.tokenizer_model)
except Exception:
raise ValueError("Using --tokenizer=sentencepiece \
requires valid --tokenizer_model")
tokenizer.add_special_tokens(["[CLS]", "[SEP]"])
elif args.tokenizer == "nemobert":
tokenizer = nemo_nlp.NemoBertTokenizer(args.pretrained_bert_model)
else:
raise ValueError(f"received unexpected tokenizer '{args.tokenizer}'")
if args.bert_config is not None:
with open(args.bert_config) as json_file:
config = json.load(json_file)
model = nemo_nlp.huggingface.BERT(**config)
else:
""" Use this if you're using a standard BERT model.
To see the list of pretrained models, call:
nemo_nlp.huggingface.BERT.list_pretrained_models()
"""
model = nemo_nlp.huggingface.BERT(
pretrained_model_name=args.pretrained_bert_model)