def create_loss_module(self):
# create the loss module if it is not yet created by the training data loader
if not hasattr(self, 'loss'):
if hasattr(self, 'class_weights') and self.class_weights:
# You may need to increase the number of epochs for convergence when using weighted_loss
self.loss = CrossEntropyLoss(weight=self.class_weights)
else:
self.loss = CrossEntropyLoss()
def __init__(self, cfg: DictConfig, trainer: Trainer = None):
""" Initializes BERT Joint Intent and Slot model.
"""
self.data_dir = cfg.data_dir
self.max_seq_length = cfg.language_model.max_seq_length
self.data_desc = IntentSlotDataDesc(
data_dir=cfg.data_dir,
modes=[cfg.train_ds.prefix, cfg.validation_ds.prefix])
self._setup_tokenizer(cfg.tokenizer)
# init superclass
super().__init__(cfg=cfg, trainer=trainer)
# initialize Bert model
self.bert_model = get_lm_model(
pretrained_model_name=cfg.language_model.pretrained_model_name,
config_file=cfg.language_model.config_file,
config_dict=OmegaConf.to_container(cfg.language_model.config)
if cfg.language_model.config else None,
checkpoint_file=cfg.language_model.lm_checkpoint,
)
self.classifier = SequenceTokenClassifier(
hidden_size=self.bert_model.config.hidden_size,
num_intents=self.data_desc.num_intents,
num_slots=self.data_desc.num_slots,
dropout=cfg.head.fc_dropout,
num_layers=cfg.head.num_output_layers,
log_softmax=False,
)
# define losses
if cfg.class_balancing == 'weighted_loss':
# You may need to increase the number of epochs for convergence when using weighted_loss
self.intent_loss = CrossEntropyLoss(
logits_ndim=2, weight=self.data_desc.intent_weights)
self.slot_loss = CrossEntropyLoss(
logits_ndim=3, weight=self.data_desc.slot_weights)
else:
self.intent_loss = CrossEntropyLoss(logits_ndim=2)
self.slot_loss = CrossEntropyLoss(logits_ndim=3)
self.total_loss = AggregatorLoss(
num_inputs=2,
weights=[cfg.intent_loss_weight, 1.0 - cfg.intent_loss_weight])
# setup to track metrics
self.intent_classification_report = ClassificationReport(
self.data_desc.num_intents, self.data_desc.intents_label_ids)
self.slot_classification_report = ClassificationReport(
self.data_desc.num_slots, self.data_desc.slots_label_ids)
# Optimizer setup needs to happen after all model weights are ready
self.setup_optimization(cfg.optim)
def __init__(self, cfg: DictConfig, trainer: Trainer = None):
"""Initializes the BERTTextClassifier model."""
# shared params for dataset and data loaders
self.dataset_cfg = cfg.dataset
# tokenizer needs to get initialized before the super.__init__()
# as dataloaders and datasets need it to process the data
self.setup_tokenizer(cfg.tokenizer)
super().__init__(cfg=cfg, trainer=trainer)
self.bert_model = get_lm_model(
pretrained_model_name=cfg.language_model.pretrained_model_name,
config_file=cfg.language_model.config_file,
config_dict=cfg.language_model.config,
checkpoint_file=cfg.language_model.lm_checkpoint,
)
self.classifier = SequenceClassifier(
hidden_size=self.bert_model.config.hidden_size,
num_classes=cfg.dataset.num_classes,
num_layers=cfg.classifier_head.num_output_layers,
activation='relu',
log_softmax=False,
dropout=cfg.classifier_head.fc_dropout,
use_transformer_init=True,
idx_conditioned_on=0,
)
class_weights = None
if cfg.dataset.class_balancing == 'weighted_loss':
if cfg.train_ds.file_path:
class_weights = calc_class_weights(cfg.train_ds.file_path,
cfg.dataset.num_classes)
else:
logging.info(
'Class_balancing feature is enabled but no train file is given. Calculating the class weights is skipped.'
)
if class_weights:
# You may need to increase the number of epochs for convergence when using weighted_loss
self.loss = CrossEntropyLoss(weight=class_weights)
else:
self.loss = CrossEntropyLoss()
# setup to track metrics
self.classification_report = ClassificationReport(
num_classes=cfg.dataset.num_classes,
mode='micro',
dist_sync_on_step=True)
# register the file containing the labels into the artifacts to get stored in the '.nemo' file later
if 'class_labels' in cfg and 'class_labels_file' in cfg.class_labels and cfg.class_labels.class_labels_file:
self.register_artifact('class_labels',
cfg.class_labels.class_labels_file)
def setup_loss(self, class_balancing: str = None):
"""Setup loss
Setup or update loss.
Args:
class_balancing: whether to use class weights during training
"""
if class_balancing == 'weighted_loss' and self.class_weights:
# you may need to increase the number of epochs for convergence when using weighted_loss
loss = CrossEntropyLoss(logits_ndim=3, weight=self.class_weights)
else:
loss = CrossEntropyLoss(logits_ndim=3)
return loss
def setup_loss(self, class_balancing: str = None):
"""Setup loss
Call this method only after update_data_dir() so that self.data_desc has class weights stats
Args:
class_balancing: whether to use class weights during training
"""
if class_balancing == 'weighted_loss' and self.data_desc:
# you may need to increase the number of epochs for convergence when using weighted_loss
loss = CrossEntropyLoss(logits_ndim=3,
weight=self.data_desc.class_weights)
else:
loss = CrossEntropyLoss(logits_ndim=3)
return loss
def __init__(self, cfg: DictConfig, trainer: Trainer = None):
super().__init__(cfg=cfg, trainer=trainer)
self._update_decoder_config(self.cfg.decoder)
self.preprocessor = EncDecClassificationModel.from_config_dict(
self._cfg.preprocessor)
self.encoder = EncDecClassificationModel.from_config_dict(
self._cfg.encoder)
self.decoder = EncDecClassificationModel.from_config_dict(
self._cfg.decoder)
self.loss = CrossEntropyLoss()
if hasattr(self._cfg,
'spec_augment') and self._cfg.spec_augment is not None:
self.spec_augmentation = EncDecClassificationModel.from_config_dict(
self._cfg.spec_augment)
else:
self.spec_augmentation = None
if hasattr(self._cfg, 'crop_or_pad_augment'
) and self._cfg.crop_or_pad_augment is not None:
self.crop_or_pad = EncDecClassificationModel.from_config_dict(
self._cfg.crop_or_pad_augment)
else:
self.crop_or_pad = None
# Setup metric objects
self._accuracy = TopKClassificationAccuracy(dist_sync_on_step=True)
def __init__(self, cfg: DictConfig, trainer: Trainer = None):
# Get global rank and total number of GPU workers for IterableDataset partitioning, if applicable
self.global_rank = 0
self.world_size = 1
self.local_rank = 0
if trainer is not None:
self.global_rank = (trainer.node_rank * trainer.num_gpus) + trainer.local_rank
self.world_size = trainer.num_nodes * trainer.num_gpus
self.local_rank = trainer.local_rank
super().__init__(cfg=cfg, trainer=trainer)
self._update_decoder_config(self._cfg.decoder)
self.preprocessor = EncDecClassificationModel.from_config_dict(self._cfg.preprocessor)
self.encoder = EncDecClassificationModel.from_config_dict(self._cfg.encoder)
self.decoder = EncDecClassificationModel.from_config_dict(self._cfg.decoder)
self.loss = CrossEntropyLoss()
if hasattr(self._cfg, 'spec_augment') and self._cfg.spec_augment is not None:
self.spec_augmentation = EncDecClassificationModel.from_config_dict(self._cfg.spec_augment)
else:
self.spec_augmentation = None
if hasattr(self._cfg, 'crop_or_pad_augment') and self._cfg.crop_or_pad_augment is not None:
self.crop_or_pad = EncDecClassificationModel.from_config_dict(self._cfg.crop_or_pad_augment)
else:
self.crop_or_pad = None
# Setup metric objects
self._accuracy = TopKClassificationAccuracy(dist_sync_on_step=True)
def __init__(self, cfg: DictConfig, trainer: Trainer = None):
if cfg.tokenizer is not None:
self._setup_tokenizer(cfg.tokenizer)
else:
self.tokenizer = None
super().__init__(cfg=cfg, trainer=trainer)
self.bert_model = get_lm_model(
pretrained_model_name=cfg.language_model.pretrained_model_name,
config_file=cfg.language_model.config_file,
config_dict=OmegaConf.to_container(cfg.language_model.config)
if cfg.language_model.config else None,
checkpoint_file=cfg.language_model.lm_checkpoint,
vocab_file=cfg.tokenizer.get('vocab_file')
if cfg.tokenizer is not None else None,
)
self.hidden_size = self.bert_model.config.hidden_size
self.vocab_size = self.bert_model.config.vocab_size
self.only_mlm_loss = cfg.only_mlm_loss
self.mlm_classifier = BertPretrainingTokenClassifier(
hidden_size=self.hidden_size,
num_classes=self.vocab_size,
num_layers=cfg.num_tok_classification_layers,
activation="gelu",
log_softmax=True,
use_transformer_init=True,
)
self.mlm_loss = SmoothedCrossEntropyLoss()
if not self.only_mlm_loss:
self.nsp_classifier = SequenceClassifier(
hidden_size=self.hidden_size,
num_classes=2,
num_layers=cfg.num_seq_classification_layers,
log_softmax=False,
activation="tanh",
use_transformer_init=True,
)
self.nsp_loss = CrossEntropyLoss()
self.agg_loss = AggregatorLoss(num_inputs=2)
# # tie weights of MLM softmax layer and embedding layer of the encoder
if (self.mlm_classifier.mlp.last_linear_layer.weight.shape !=
self.bert_model.embeddings.word_embeddings.weight.shape):
raise ValueError(
"Final classification layer does not match embedding layer.")
self.mlm_classifier.mlp.last_linear_layer.weight = self.bert_model.embeddings.word_embeddings.weight
# create extra bias
# setup to track metrics
self.validation_perplexity = Perplexity(compute_on_step=False)
self.setup_optimization(cfg.optim)
def setup_loss(self, class_balancing: str = None):
"""Setup loss
Setup or update loss.
Args:
class_balancing: whether to use class weights during training
"""
if class_balancing not in ['weighted_loss', None]:
raise ValueError(f'Class balancing {class_balancing} is not supported. Choose from: [null, weighted_loss]')
if class_balancing == 'weighted_loss' and self.class_weights:
# you may need to increase the number of epochs for convergence when using weighted_loss
loss = CrossEntropyLoss(logits_ndim=3, weight=self.class_weights)
logging.debug(f'Using {class_balancing} class balancing.')
else:
loss = CrossEntropyLoss(logits_ndim=3)
logging.debug(f'Using CrossEntropyLoss class balancing.')
return loss
def __init__(self, cfg: DictConfig, trainer: Trainer = None):
"""
Initializes model to use BERT model for GLUE tasks.
"""
self.data_dir = cfg.dataset.data_dir
if not os.path.exists(self.data_dir):
raise FileNotFoundError(
"GLUE datasets not found. For more details on how to get the data, see: "
"https://gist.github.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e"
)
if cfg.task_name not in cfg.supported_tasks:
raise ValueError(
f'{cfg.task_name} not in supported task. Choose from {cfg.supported_tasks}'
)
self.task_name = cfg.task_name
# MNLI task has two separate dev sets: matched and mismatched
cfg.train_ds.file_name = os.path.join(self.data_dir,
cfg.train_ds.file_name)
if self.task_name == "mnli":
cfg.validation_ds.file_name = [
os.path.join(self.data_dir, 'dev_matched.tsv'),
os.path.join(self.data_dir, 'dev_mismatched.tsv'),
]
else:
cfg.validation_ds.file_name = os.path.join(
self.data_dir, cfg.validation_ds.file_name)
logging.info(
f'Using {cfg.validation_ds.file_name} for model evaluation.')
self._setup_tokenizer(cfg.tokenizer)
super().__init__(cfg=cfg, trainer=trainer)
num_labels = GLUE_TASKS_NUM_LABELS[self.task_name]
self.bert_model = get_lm_model(
pretrained_model_name=cfg.language_model.pretrained_model_name,
config_file=cfg.language_model.config_file,
config_dict=OmegaConf.to_container(cfg.language_model.config)
if cfg.language_model.config else None,
checkpoint_file=cfg.language_model.lm_checkpoint,
)
# uses [CLS] token for classification (the first token)
if self.task_name == "sts-b":
self.pooler = SequenceRegression(
hidden_size=self.bert_model.config.hidden_size)
self.loss = MSELoss()
else:
self.pooler = SequenceClassifier(
hidden_size=self.bert_model.config.hidden_size,
num_classes=num_labels,
log_softmax=False)
self.loss = CrossEntropyLoss()
# Optimizer setup needs to happen after all model weights are ready
self.setup_optimization(cfg.optim)
def _reconfigure_classifier(self) -> None:
""" Method reconfigures the classifier depending on the settings of model cfg.data_desc """
self.classifier = SequenceTokenClassifier(
hidden_size=self.bert_model.config.hidden_size,
num_intents=len(self.cfg.data_desc.intent_labels),
num_slots=len(self.cfg.data_desc.slot_labels),
dropout=self.cfg.head.fc_dropout,
num_layers=self.cfg.head.num_output_layers,
log_softmax=False,
)
# define losses
if self.cfg.class_balancing == "weighted_loss":
# You may need to increase the number of epochs for convergence when using weighted_loss
self.intent_loss = BCEWithLogitsLoss(
logits_ndim=2, pos_weight=self.cfg.data_desc.intent_weights)
self.slot_loss = CrossEntropyLoss(
logits_ndim=3, weight=self.cfg.data_desc.slot_weights)
else:
self.intent_loss = BCEWithLogitsLoss(logits_ndim=2)
self.slot_loss = CrossEntropyLoss(logits_ndim=3)
self.total_loss = AggregatorLoss(
num_inputs=2,
weights=[
self.cfg.intent_loss_weight, 1.0 - self.cfg.intent_loss_weight
],
)
# setup to track metrics
self.intent_classification_report = MultiLabelClassificationReport(
num_classes=len(self.cfg.data_desc.intent_labels),
label_ids=self.cfg.data_desc.intent_label_ids,
dist_sync_on_step=True,
mode="micro",
)
self.slot_classification_report = ClassificationReport(
num_classes=len(self.cfg.data_desc.slot_labels),
label_ids=self.cfg.data_desc.slot_label_ids,
dist_sync_on_step=True,
mode="micro",
)
def __init__(self, cfg: DictConfig, trainer: Trainer = None):
"""
Initializes BERT Punctuation and Capitalization model.
"""
self.setup_tokenizer(cfg.tokenizer)
super().__init__(cfg=cfg, trainer=trainer)
self.bert_model = get_lm_model(
pretrained_model_name=cfg.language_model.pretrained_model_name,
config_file=self.register_artifact('language_model.config_file',
cfg.language_model.config_file),
config_dict=OmegaConf.to_container(cfg.language_model.config)
if cfg.language_model.config else None,
checkpoint_file=cfg.language_model.lm_checkpoint,
vocab_file=self.register_artifact('tokenizer.vocab_file',
cfg.tokenizer.vocab_file),
)
self.punct_classifier = TokenClassifier(
hidden_size=self.bert_model.config.hidden_size,
num_classes=len(self._cfg.punct_label_ids),
activation=cfg.punct_head.activation,
log_softmax=False,
dropout=cfg.punct_head.fc_dropout,
num_layers=cfg.punct_head.punct_num_fc_layers,
use_transformer_init=cfg.punct_head.use_transformer_init,
)
self.capit_classifier = TokenClassifier(
hidden_size=self.bert_model.config.hidden_size,
num_classes=len(self._cfg.capit_label_ids),
activation=cfg.capit_head.activation,
log_softmax=False,
dropout=cfg.capit_head.fc_dropout,
num_layers=cfg.capit_head.capit_num_fc_layers,
use_transformer_init=cfg.capit_head.use_transformer_init,
)
self.loss = CrossEntropyLoss(logits_ndim=3)
self.agg_loss = AggregatorLoss(num_inputs=2)
# setup to track metrics
self.punct_class_report = ClassificationReport(
num_classes=len(self._cfg.punct_label_ids),
label_ids=self._cfg.punct_label_ids,
mode='macro',
dist_sync_on_step=True,
)
self.capit_class_report = ClassificationReport(
num_classes=len(self._cfg.capit_label_ids),
label_ids=self._cfg.capit_label_ids,
mode='macro',
dist_sync_on_step=True,
)
def __init__(self, cfg: DictConfig, trainer: Trainer = None):
"""
Initializes model to use BERT model for GLUE tasks.
"""
if cfg.task_name not in cfg.supported_tasks:
raise ValueError(
f'{cfg.task_name} not in supported task. Choose from {cfg.supported_tasks}'
)
self.task_name = cfg.task_name
# needed to setup validation on multiple datasets
# MNLI task has two separate dev sets: matched and mismatched
if not self._is_model_being_restored():
if self.task_name == "mnli":
cfg.validation_ds.ds_item = [
os.path.join(cfg.dataset.data_dir, 'dev_matched.tsv'),
os.path.join(cfg.dataset.data_dir, 'dev_mismatched.tsv'),
]
else:
cfg.validation_ds.ds_item = os.path.join(
cfg.dataset.data_dir, cfg.validation_ds.ds_item)
cfg.train_ds.ds_item = os.path.join(cfg.dataset.data_dir,
cfg.train_ds.ds_item)
logging.info(
f'Using {cfg.validation_ds.ds_item} for model evaluation.')
self.setup_tokenizer(cfg.tokenizer)
super().__init__(cfg=cfg, trainer=trainer)
num_labels = GLUE_TASKS_NUM_LABELS[self.task_name]
self.bert_model = get_lm_model(
pretrained_model_name=cfg.language_model.pretrained_model_name,
config_file=self.register_artifact('language_model.config_file',
cfg.language_model.config_file),
config_dict=OmegaConf.to_container(cfg.language_model.config)
if cfg.language_model.config else None,
checkpoint_file=cfg.language_model.lm_checkpoint,
vocab_file=self.register_artifact('tokenizer.vocab_file',
cfg.tokenizer.vocab_file),
)
# uses [CLS] token for classification (the first token)
if self.task_name == "sts-b":
self.pooler = SequenceRegression(
hidden_size=self.bert_model.config.hidden_size)
self.loss = MSELoss()
else:
self.pooler = SequenceClassifier(
hidden_size=self.bert_model.config.hidden_size,
num_classes=num_labels,
log_softmax=False)
self.loss = CrossEntropyLoss()
def _setup_loss(self):
return CrossEntropyLoss()
def __init__(self, cfg: DictConfig, trainer: Trainer = None) -> None:
super().__init__(cfg=cfg, trainer=trainer)
label_map_file = self.register_artifact("label_map",
cfg.label_map,
verify_src_exists=True)
semiotic_classes_file = self.register_artifact("semiotic_classes",
cfg.semiotic_classes,
verify_src_exists=True)
self.label_map = read_label_map(label_map_file)
self.semiotic_classes = read_semiotic_classes(semiotic_classes_file)
self.num_labels = len(self.label_map)
self.num_semiotic_labels = len(self.semiotic_classes)
self.id_2_tag = {
tag_id: tagging.Tag(tag)
for tag, tag_id in self.label_map.items()
}
self.id_2_semiotic = {
semiotic_id: semiotic
for semiotic, semiotic_id in self.semiotic_classes.items()
}
self.max_sequence_len = cfg.get(
'max_sequence_len', self.tokenizer.tokenizer.model_max_length)
# setup to track metrics
# we will have (len(self.semiotic_classes) + 1) labels
# last one stands for WRONG (span in which the predicted tags don't match the labels)
# this is needed to feed the sequence of classes to classification_report during validation
label_ids = self.semiotic_classes.copy()
label_ids["WRONG"] = len(self.semiotic_classes)
self.tag_classification_report = ClassificationReport(
len(self.semiotic_classes) + 1,
label_ids=label_ids,
mode='micro',
dist_sync_on_step=True)
self.tag_multiword_classification_report = ClassificationReport(
len(self.semiotic_classes) + 1,
label_ids=label_ids,
mode='micro',
dist_sync_on_step=True)
self.semiotic_classification_report = ClassificationReport(
len(self.semiotic_classes) + 1,
label_ids=label_ids,
mode='micro',
dist_sync_on_step=True)
self.hidden_size = cfg.hidden_size
self.logits = TokenClassifier(self.hidden_size,
num_classes=self.num_labels,
num_layers=1,
log_softmax=False,
dropout=0.1)
self.semiotic_logits = TokenClassifier(
self.hidden_size,
num_classes=self.num_semiotic_labels,
num_layers=1,
log_softmax=False,
dropout=0.1)
self.loss_fn = CrossEntropyLoss(logits_ndim=3)
self.builder = bert_example.BertExampleBuilder(
self.label_map, self.semiotic_classes, self.tokenizer.tokenizer,
self.max_sequence_len)
