args.data_dir, args.vocab_size, args.sample_size, special_tokens, 'train.txt', ) if args.tokenizer == "sentence-piece": nemo.logging.info("To use SentencePieceTokenizer.") tokenizer = nemo_nlp.SentencePieceTokenizer( model_path=data_desc.tokenizer_model) tokenizer.add_special_tokens(special_tokens) elif args.tokenizer == "nemo-bert": nemo.logging.info("To use NemoBertTokenizer.") vocab_file = os.path.join(args.data_dir, 'vocab.txt') # To train on a Chinese dataset, use NemoBertTokenizer tokenizer = nemo_nlp.NemoBertTokenizer(vocab_file=vocab_file) else: raise ValueError("Please add your tokenizer " "or use sentence-piece or nemo-bert.") args.vocab_size = tokenizer.vocab_size print(vars(args)) bert_model = nemo_nlp.huggingface.BERT( vocab_size=args.vocab_size, num_hidden_layers=args.num_hidden_layers, hidden_size=args.hidden_size, num_attention_heads=args.num_attention_heads, intermediate_size=args.intermediate_size, max_position_embeddings=args.max_seq_length, hidden_act=args.hidden_act, )
def test_squad_v1(self): version_2_with_negative = False pretrained_bert_model = 'bert-base-uncased' batch_size = 3 data_dir = os.path.abspath( os.path.join(os.path.dirname(__file__), '../data/nlp/squad/v1.1')) max_query_length = 64 max_seq_length = 384 doc_stride = 128 max_steps = 100 lr_warmup_proportion = 0 eval_step_freq = 50 lr = 3e-6 do_lower_case = True n_best_size = 5 max_answer_length = 20 null_score_diff_threshold = 0.0 tokenizer = nemo_nlp.NemoBertTokenizer(pretrained_bert_model) neural_factory = nemo.core.NeuralModuleFactory( backend=nemo.core.Backend.PyTorch, local_rank=None, create_tb_writer=False, ) model = nemo_nlp.huggingface.BERT( pretrained_model_name=pretrained_bert_model) hidden_size = model.local_parameters["hidden_size"] qa_head = nemo_nlp.TokenClassifier( hidden_size=hidden_size, num_classes=2, num_layers=1, log_softmax=False, ) squad_loss = nemo_nlp.QuestionAnsweringLoss() data_layer = nemo_nlp.BertQuestionAnsweringDataLayer( mode='train', version_2_with_negative=version_2_with_negative, batch_size=batch_size, tokenizer=tokenizer, data_dir=data_dir, max_query_length=max_query_length, max_seq_length=max_seq_length, doc_stride=doc_stride, ) ( input_ids, input_type_ids, input_mask, start_positions, end_positions, _, ) = data_layer() hidden_states = model( input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask, ) qa_output = qa_head(hidden_states=hidden_states) loss, _, _ = squad_loss( logits=qa_output, start_positions=start_positions, end_positions=end_positions, ) data_layer_eval = nemo_nlp.BertQuestionAnsweringDataLayer( mode='dev', version_2_with_negative=version_2_with_negative, batch_size=batch_size, tokenizer=tokenizer, data_dir=data_dir, max_query_length=max_query_length, max_seq_length=max_seq_length, doc_stride=doc_stride, ) ( input_ids_eval, input_type_ids_eval, input_mask_eval, start_positions_eval, end_positions_eval, unique_ids_eval, ) = data_layer_eval() hidden_states_eval = model( input_ids=input_ids_eval, token_type_ids=input_type_ids_eval, attention_mask=input_mask_eval, ) qa_output_eval = qa_head(hidden_states=hidden_states_eval) _, start_logits_eval, end_logits_eval = squad_loss( logits=qa_output_eval, start_positions=start_positions_eval, end_positions=end_positions_eval, ) eval_output = [start_logits_eval, end_logits_eval, unique_ids_eval] callback_train = nemo.core.SimpleLossLoggerCallback( tensors=[loss], print_func=lambda x: print("Loss: {:.3f}".format(x[0].item())), get_tb_values=lambda x: [["loss", x[0]]], step_freq=10, tb_writer=neural_factory.tb_writer, ) callbacks_eval = nemo.core.EvaluatorCallback( eval_tensors=eval_output, user_iter_callback=lambda x, y: eval_iter_callback(x, y), user_epochs_done_callback=lambda x: eval_epochs_done_callback( x, eval_data_layer=data_layer_eval, do_lower_case=do_lower_case, n_best_size=n_best_size, max_answer_length=max_answer_length, version_2_with_negative=version_2_with_negative, null_score_diff_threshold=null_score_diff_threshold, ), tb_writer=neural_factory.tb_writer, eval_step=eval_step_freq, ) lr_policy_fn = get_lr_policy( 'WarmupAnnealing', total_steps=max_steps, warmup_ratio=lr_warmup_proportion, ) neural_factory.train( tensors_to_optimize=[loss], callbacks=[callback_train, callbacks_eval], lr_policy=lr_policy_fn, optimizer='adam_w', optimization_params={ "max_steps": max_steps, "lr": lr }, )
log_dir=args.work_dir, create_tb_writer=True, files_to_copy=[__file__], add_time_to_log_dir=True, ) if args.tokenizer == "sentencepiece": try: tokenizer = nemo_nlp.SentencePieceTokenizer( model_path=args.tokenizer_model) except Exception: raise ValueError("Using --tokenizer=sentencepiece \ requires valid --tokenizer_model") tokenizer.add_special_tokens(["[CLS]", "[SEP]"]) elif args.tokenizer == "nemobert": tokenizer = nemo_nlp.NemoBertTokenizer(args.pretrained_bert_model) else: raise ValueError(f"received unexpected tokenizer '{args.tokenizer}'") if args.bert_config is not None: with open(args.bert_config) as json_file: config = json.load(json_file) model = nemo_nlp.huggingface.BERT(**config) else: """ Use this if you're using a standard BERT model. To see the list of pretrained models, call: nemo_nlp.huggingface.BERT.list_pretrained_models() """ model = nemo_nlp.huggingface.BERT( pretrained_model_name=args.pretrained_bert_model)