def create_pipeline(num_samples=-1,
batch_size=32,
num_gpus=1,
local_rank=0,
mode='train'):
nemo.logging.info(f"Loading {mode} data...")
data_file = f'{data_desc.data_dir}/{mode}.tsv'
slot_file = f'{data_desc.data_dir}/{mode}_slots.tsv'
shuffle = args.shuffle_data if mode == 'train' else False
data_layer = nemo_nlp.BertJointIntentSlotDataLayer(
input_file=data_file,
slot_file=slot_file,
pad_label=data_desc.pad_label,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
num_samples=num_samples,
shuffle=shuffle,
batch_size=batch_size,
num_workers=0,
local_rank=local_rank,
ignore_extra_tokens=args.ignore_extra_tokens,
ignore_start_end=args.ignore_start_end
)
ids, type_ids, input_mask, loss_mask, \
subtokens_mask, intents, slots = data_layer()
data_size = len(data_layer)
print(f'The length of data layer is {data_size}')
if data_size < batch_size:
nemo.logging.warning("Batch_size is larger than the dataset size")
nemo.logging.warning("Reducing batch_size to dataset size")
batch_size = data_size
steps_per_epoch = math.ceil(data_size / (batch_size * num_gpus))
nemo.logging.info(f"Steps_per_epoch = {steps_per_epoch}")
hidden_states = pretrained_bert_model(input_ids=ids,
token_type_ids=type_ids,
attention_mask=input_mask)
intent_logits, slot_logits = classifier(hidden_states=hidden_states)
loss = loss_fn(intent_logits=intent_logits,
slot_logits=slot_logits,
loss_mask=loss_mask,
intents=intents,
slots=slots)
if mode == 'train':
tensors_to_evaluate = [loss, intent_logits, slot_logits]
else:
tensors_to_evaluate = [intent_logits, slot_logits, intents,
slots, subtokens_mask]
return tensors_to_evaluate, loss, steps_per_epoch, data_layer
def create_pipeline(dataset,
batch_size=32,
num_gpus=1,
local_rank=0,
mode='train'):
data_layer = nemo_nlp.BertJointIntentSlotDataLayer(dataset,
batch_size=batch_size,
num_workers=0,
local_rank=local_rank)
ids, type_ids, input_mask, slot_mask, intents, slots = data_layer()
data_size = len(data_layer)
if data_size < batch_size:
nf.logger.warning("Batch_size is larger than the dataset size")
nf.logger.warning("Reducing batch_size to dataset size")
batch_size = data_size
steps_per_epoch = math.ceil(data_size / (batch_size * num_gpus))
nf.logger.info(f"Steps_per_epoch = {steps_per_epoch}")
hidden_states = pretrained_bert_model(input_ids=ids,
token_type_ids=type_ids,
attention_mask=input_mask)
intent_logits, slot_logits = classifier(hidden_states=hidden_states)
loss = loss_fn(intent_logits=intent_logits,
slot_logits=slot_logits,
input_mask=input_mask,
intents=intents,
slots=slots)
if mode == 'train':
tensors_to_evaluate = [loss, intent_logits, slot_logits]
else:
tensors_to_evaluate = [intent_logits, slot_logits, intents, slots]
return tensors_to_evaluate, loss, steps_per_epoch, data_layer
"""
pretrained_bert_model = nemo_nlp.huggingface.BERT(
pretrained_model_name=args.pretrained_bert_model)
hidden_size = pretrained_bert_model.local_parameters["hidden_size"]
tokenizer = BertTokenizer.from_pretrained(args.pretrained_bert_model)
data_desc = JointIntentSlotDataDesc(args.data_dir, args.do_lower_case,
args.dataset_name)
# Evaluation pipeline
nf.logger.info("Loading eval data...")
data_layer = nemo_nlp.BertJointIntentSlotDataLayer(
input_file=f'{data_desc.data_dir}/{args.eval_file_prefix}.tsv',
slot_file=f'{data_desc.data_dir}/{args.eval_file_prefix}_slots.tsv',
pad_label=data_desc.pad_label,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
shuffle=False,
batch_size=args.batch_size,
num_workers=0,
local_rank=args.local_rank)
classifier = nemo_nlp.JointIntentSlotClassifier(
hidden_size=hidden_size,
num_intents=data_desc.num_intents,
num_slots=data_desc.num_slots)
ids, type_ids, \
input_mask, loss_mask, subtokens_mask, \
intents, slots = data_layer()
hidden_states = pretrained_bert_model(input_ids=ids,
def create_pipeline(data_file,
slot_file,
max_seq_length,
batch_size=32,
num_samples=-1,
shuffle=True,
num_gpus=1,
local_rank=0,
mode='train'):
nf.logger.info(f"Loading {mode} data...")
data_layer = nemo_nlp.BertJointIntentSlotDataLayer(
path_to_data=data_file,
path_to_slot=slot_file,
pad_label=pad_label,
tokenizer=tokenizer,
mode=mode,
max_seq_length=max_seq_length,
num_samples=num_samples,
batch_size=batch_size,
shuffle=shuffle,
num_workers=0,
local_rank=local_rank)
ids, type_ids, input_mask, slot_mask, intents, slots = data_layer()
data_size = len(data_layer)
if data_size < batch_size:
nf.logger.warning("Batch_size is larger than the dataset size")
nf.logger.warning("Reducing batch_size to dataset size")
batch_size = data_size
steps_per_epoch = int(data_size / (batch_size * num_gpus))
nf.logger.info(f"Steps_per_epoch = {steps_per_epoch}")
hidden_states = pretrained_bert_model(input_ids=ids,
token_type_ids=type_ids,
attention_mask=input_mask)
intent_logits, slot_logits = classifier(hidden_states=hidden_states)
loss = loss_fn(intent_logits=intent_logits,
slot_logits=slot_logits,
input_mask=input_mask,
intents=intents,
slots=slots)
# Create trainer and execute training action
if mode == 'train':
callback_fn = nemo.core.SimpleLossLoggerCallback(
tensors=[loss, intent_logits, slot_logits],
print_func=lambda x: str(np.round(x[0].item(), 3)),
tb_writer=nf.tb_writer,
get_tb_values=lambda x: [["loss", x[0]]],
step_freq=100)
elif mode == 'eval':
callback_fn = nemo.core.EvaluatorCallback(
eval_tensors=[intent_logits, slot_logits, intents, slots],
user_iter_callback=lambda x, y: eval_iter_callback(
x, y, data_layer),
user_epochs_done_callback=lambda x: eval_epochs_done_callback(
x, f'{nf.work_dir}/graphs'),
tb_writer=nf.tb_writer,
eval_step=steps_per_epoch)
return loss, callback_fn, steps_per_epoch
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
shuffle=False)
classifier = nemo_nlp.JointIntentSlotClassifier(
hidden_size=hidden_size,
num_intents=data_desc.num_intents,
num_slots=data_desc.num_slots,
dropout=args.fc_dropout)
loss_fn = nemo_nlp.JointIntentSlotLoss(num_slots=data_desc.num_slots)
# Evaluation pipeline
nf.logger.info("Loading eval data...")
data_layer = nemo_nlp.BertJointIntentSlotDataLayer(dataset,
batch_size=args.batch_size,
num_workers=0,
local_rank=args.local_rank)
ids, type_ids, input_mask, slot_mask, intents, slots = data_layer()
hidden_states = pretrained_bert_model(input_ids=ids,
token_type_ids=type_ids,
attention_mask=input_mask)
intent_logits, slot_logits = classifier(hidden_states=hidden_states)
###########################################################################
# Instantiate an optimizer to perform `infer` action
evaluated_tensors = nf.infer(
tensors=[intent_logits, slot_logits, slot_mask, intents, slots],
classifier = nemo_nlp.JointIntentSlotClassifier(hidden_size=hidden_size,
num_intents=num_intents,
num_slots=num_slots,
dropout=args.fc_dropout)
loss_fn = nemo_nlp.JointIntentSlotLoss(num_slots=num_slots)
# Evaluation pipeline
logger.info("Loading eval data...")
data_layer = nemo_nlp.BertJointIntentSlotDataLayer(
path_to_data=data_dir + '/test.tsv',
path_to_slot=data_dir + '/test_slots.tsv',
pad_label=num_slots - 1,
tokenizer=tokenizer,
mode='eval',
max_seq_length=args.max_seq_length,
batch_size=args.batch_size,
shuffle=False,
num_workers=0,
local_rank=args.local_rank)
ids, type_ids, input_mask, slot_mask, intents, slots = data_layer()
hidden_states = pretrained_bert_model(input_ids=ids,
token_type_ids=type_ids,
attention_mask=input_mask)
intent_logits, slot_logits = classifier(hidden_states=hidden_states)
###########################################################################
