args = parser.parse_args()
nf = nemo.core.NeuralModuleFactory(backend=nemo.core.Backend.PyTorch,
optimization_level=args.amp_opt_level,
log_dir=None)
""" Load the pretrained BERT parameters
See the list of pretrained models, call:
nemo_nlp.BERT.list_pretrained_models()
"""
pretrained_bert_model = nemo_nlp.nm.trainables.huggingface.BERT(
pretrained_model_name=args.pretrained_bert_model)
tokenizer = BertTokenizer.from_pretrained(args.pretrained_bert_model)
hidden_size = pretrained_bert_model.hidden_size
data_desc = JointIntentSlotDataDesc(args.data_dir, args.do_lower_case,
args.dataset_name)
query = args.query
if args.do_lower_case:
query = query.lower()
data_layer = nemo.collections.nlp.nm.data_layers.joint_intent_slot_datalayer.BertJointIntentSlotInferDataLayer(
queries=[query],
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
batch_size=1)
# Create sentence classification loss on top
classifier = nemo.collections.nlp.nm.trainables.joint_intent_slot.joint_intent_slot_nm.JointIntentSlotClassifier(
hidden_size=hidden_size,
num_intents=data_desc.num_intents,
)
pretrained_bert_model = nemo_nlp.nm.trainables.get_huggingface_model(
bert_config=args.bert_config,
pretrained_model_name=args.pretrained_model_name)
tokenizer = nemo_nlp.data.NemoBertTokenizer(
pretrained_model=args.pretrained_model_name)
if args.bert_checkpoint:
pretrained_bert_model.restore_from(args.bert_checkpoint)
logging.info(f"Model restored from {args.bert_checkpoint}")
hidden_size = pretrained_bert_model.hidden_size
data_desc = JointIntentSlotDataDesc(data_dir=args.data_dir,
none_slot_label=args.none_slot_label,
pad_label=args.pad_label)
# Create sentence classification loss on top
classifier = JointIntentSlotClassifier(hidden_size=hidden_size,
num_intents=data_desc.num_intents,
num_slots=data_desc.num_slots,
dropout=args.fc_dropout)
if args.class_balancing == 'weighted_loss':
# To tackle imbalanced classes, you may use weighted loss
intent_loss_fn = CrossEntropyLossNM(logits_ndim=2,
weight=data_desc.intent_weights)
slot_loss_fn = CrossEntropyLossNM(logits_ndim=3,
weight=data_desc.slot_weights)
else:
pretrained_bert_model = nemo_nlp.nm.trainables.get_pretrained_lm_model(
pretrained_model_name=args.pretrained_model_name,
config=args.bert_config,
vocab=args.vocab_file)
tokenizer = nemo.collections.nlp.data.tokenizers.get_tokenizer(
tokenizer_name=args.tokenizer,
pretrained_model_name=args.pretrained_model_name,
tokenizer_model=args.tokenizer_model,
vocab_file=args.vocab_file,
do_lower_case=args.do_lower_case,
)
hidden_size = pretrained_bert_model.hidden_size
data_desc = JointIntentSlotDataDesc(data_dir=args.data_dir)
query = args.query
if args.do_lower_case:
query = query.lower()
data_layer = BertJointIntentSlotInferDataLayer(
queries=[query],
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
batch_size=1)
# Create sentence classification loss on top
classifier = JointIntentSlotClassifier(hidden_size=hidden_size,
num_intents=data_desc.num_intents,
num_slots=data_desc.num_slots,
tokenizer = BertTokenizer.from_pretrained(args.pretrained_bert_model)
""" Load the pretrained BERT parameters
See the list of pretrained models, call:
nemo_nlp.huggingface.BERT.list_pretrained_models()
"""
if args.bert_checkpoint and args.bert_config:
pretrained_bert_model = BERT(config_filename=args.bert_config)
pretrained_bert_model.restore_from(args.bert_checkpoint)
else:
pretrained_bert_model = BERT(
pretrained_model_name=args.pretrained_bert_model)
hidden_size = pretrained_bert_model.hidden_size
data_desc = JointIntentSlotDataDesc(args.data_dir, args.do_lower_case,
args.dataset_name, args.none_slot_label,
args.pad_label)
# Create sentence classification loss on top
classifier = JointIntentSlotClassifier(hidden_size=hidden_size,
num_intents=data_desc.num_intents,
num_slots=data_desc.num_slots,
dropout=args.fc_dropout)
if args.class_balancing == 'weighted_loss':
# To tackle imbalanced classes, you may use weighted loss
intent_loss_fn = CrossEntropyLossNM(logits_dim=2,
weight=data_desc.intent_weights)
slot_loss_fn = CrossEntropyLossNM(logits_dim=3,
weight=data_desc.slot_weights)