d_model=args.d_model, d_inner=args.d_inner, num_layers=args.num_layers, num_attn_heads=args.num_heads, ffn_dropout=args.ffn_dropout, vocab_size=vocab_size, max_seq_length=args.max_seq_length, embedding_dropout=args.embedding_dropout, learn_positional_encodings=True, hidden_act="gelu", **dec_first_sublayer_params) decoder.restore_from(args.restore_from, local_rank=args.local_rank) t_log_softmax = nemo_nlp.TokenClassifier(args.d_model, num_classes=vocab_size, num_layers=1, log_softmax=True) beam_translator = nemo_nlp.BeamSearchTranslatorNM( decoder=decoder, log_softmax=t_log_softmax, max_seq_length=args.max_seq_length, beam_size=args.beam_size, length_penalty=args.len_pen, bos_token=tokenizer.bos_id(), pad_token=tokenizer.pad_id(), eos_token=tokenizer.eos_id()) loss = nemo_nlp.PaddedSmoothedCrossEntropyLossNM(pad_id=0, smoothing=0.1) loss_eval = nemo_nlp.PaddedSmoothedCrossEntropyLossNM(pad_id=0, smoothing=0.0)
pretrained_bert_model = nemo_nlp.huggingface.BERT( pretrained_model_name=args.pretrained_bert_model, factory=nf) else: """ Use this if you're using a BERT model that you pre-trained yourself. Replace BERT-STEP-150000.pt with the path to your checkpoint. """ tokenizer = SentencePieceTokenizer(model_path=tokenizer_model) tokenizer.add_special_tokens(["[MASK]", "[CLS]", "[SEP]"]) bert_model = nemo_nlp.huggingface.BERT(config_filename=args.bert_config, factory=nf) pretrained_bert_model.restore_from(args.bert_checkpoint) hidden_size = pretrained_bert_model.local_parameters["hidden_size"] ner_classifier = nemo_nlp.TokenClassifier(hidden_size=hidden_size, num_classes=args.num_classes, dropout=args.fc_dropout) ner_loss = nemo_nlp.TokenClassificationLoss(num_classes=args.num_classes) def create_pipeline(input_file, max_seq_length=args.max_seq_length, batch_size=args.batch_size, local_rank=args.local_rank, num_gpus=args.num_gpus): data_layer = nemo_nlp.BertTokenClassificationDataLayer( tokenizer=tokenizer, input_file=input_file, max_seq_length=max_seq_length, dataset_type=args.dataset_type, batch_size=batch_size,
""" pretrained_bert_model = nemo_nlp.huggingface.BERT( pretrained_model_name=args.pretrained_bert_model) hidden_size = pretrained_bert_model.local_parameters["hidden_size"] tokenizer = NemoBertTokenizer(args.pretrained_bert_model) data_layer = nemo_nlp.BertTokenClassificationInferDataLayer( queries=args.queries, tokenizer=tokenizer, max_seq_length=args.max_seq_length, batch_size=1) punct_classifier = \ nemo_nlp.TokenClassifier(hidden_size=hidden_size, num_classes=len(punct_labels_dict), dropout=args.fc_dropout, num_layers=args.punct_num_fc_layers, name='Punctuation') capit_classifier = nemo_nlp.TokenClassifier(hidden_size=hidden_size, num_classes=len(capit_labels_dict), dropout=args.fc_dropout, name='Capitalization') input_ids, input_type_ids, input_mask, loss_mask, subtokens_mask = data_layer() hidden_states = pretrained_bert_model(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) punct_logits = punct_classifier(hidden_states=hidden_states)
See the list of pretrained models, call: nemo_nlp.huggingface.BERT.list_pretrained_models() """ pretrained_bert_model = nemo_nlp.huggingface.BERT( pretrained_model_name=args.pretrained_bert_model) hidden_size = pretrained_bert_model.local_parameters["hidden_size"] tokenizer = NemoBertTokenizer(args.pretrained_bert_model) data_layer = nemo_nlp.BertTokenClassificationInferDataLayer( queries=args.queries, tokenizer=tokenizer, max_seq_length=args.max_seq_length, batch_size=1) classifier = nemo_nlp.TokenClassifier(hidden_size=hidden_size, num_classes=len(labels_dict), dropout=args.fc_dropout) input_ids, input_type_ids, input_mask, _, subtokens_mask = data_layer() hidden_states = pretrained_bert_model(input_ids=input_ids, token_type_ids=input_type_ids, attention_mask=input_mask) logits = classifier(hidden_states=hidden_states) ########################################################################### # Instantiate an optimizer to perform `infer` action evaluated_tensors = nf.infer( tensors=[logits, subtokens_mask], checkpoint_dir=args.work_dir,
if args.bert_config is not None: with open(args.bert_config) as json_file: config = json.load(json_file) model = nemo_nlp.huggingface.BERT(**config) else: """ Use this if you're using a standard BERT model. To see the list of pretrained models, call: nemo_nlp.huggingface.BERT.list_pretrained_models() """ model = nemo_nlp.huggingface.BERT( pretrained_model_name=args.pretrained_bert_model) hidden_size = model.local_parameters["hidden_size"] qa_head = nemo_nlp.TokenClassifier(hidden_size=hidden_size, num_classes=2, num_layers=1, log_softmax=False) squad_loss = nemo_nlp.QuestionAnsweringLoss() if args.bert_checkpoint is not None: model.restore_from(args.bert_checkpoint) if not args.evaluation_only: train_loss, train_steps_per_epoch, _, _ = \ create_pipeline( data_dir=args.data_dir, model=model, head=qa_head, loss_fn=squad_loss, max_query_length=args.max_query_length, max_seq_length=args.max_seq_length, doc_stride=args.doc_stride,