def on_vocab_update(self):
num_words = vocabulary.words_vocab_size(self.backbone.vocab)
if len(self._loss.softmax_b) != num_words:
self._loss = SoftmaxLoss(
num_words=num_words,
embedding_dim=self._forward_dim,
)
def __init__(
self,
backbone: ModelBackbone,
dropout: float = None,
bidirectional: bool = False,
) -> None:
super(LanguageModelling, self).__init__(backbone)
self._empty_prediction = LanguageModellingPrediction(
lm_embeddings=numpy.array([]), mask=numpy.array([]))
self.bidirectional = bidirectional
if not backbone.featurizer.has_word_features:
raise ConfigurationError(
"`LanguageModelling` defines a word-level next token language model. "
"Please check your `features` configuration to enable at least `words` features."
)
if backbone.encoder.is_bidirectional() is not bidirectional:
raise ConfigurationError(
"Bidirectionality of contextualizer must match bidirectionality of "
"language model. "
f"Contextualizer bidirectional: {backbone.encoder.is_bidirectional()}, "
f"language model bidirectional: {bidirectional}")
if self.bidirectional:
self._forward_dim = backbone.encoder.get_output_dim() // 2
else:
self._forward_dim = backbone.encoder.get_output_dim()
if dropout:
self._dropout = torch.nn.Dropout(dropout)
else:
self._dropout = lambda x: x
self._metrics = Metrics(perplexity={"type": "perplexity"})
self._loss = SoftmaxLoss(
num_words=vocabulary.words_vocab_size(self.backbone.vocab),
embedding_dim=self._forward_dim,
)
def __init__(self, backbone: ModelBackbone, dropout: float = None) -> None:
super(LanguageModelling, self).__init__(backbone)
if not backbone.featurizer.has_word_features:
raise ConfigurationError(
"`LanguageModelling` defines a word-level next token language model. "
"Please check your `features` configuration to enable at least `words` features."
)
self._forward_dim = backbone.encoder.get_output_dim()
if dropout:
self._dropout = torch.nn.Dropout(dropout)
else:
self._dropout = lambda x: x
self.metrics = {"perplexity": Perplexity()}
self._loss = SoftmaxLoss(
num_words=vocabulary.words_vocab_size(self.backbone.vocab),
embedding_dim=self.backbone.encoder.get_output_dim(),
)
def on_vocab_update(self):
self._loss = SoftmaxLoss(
num_words=vocabulary.words_vocab_size(self.backbone.vocab),
embedding_dim=self._forward_dim,
)
def on_vocab_update(self):
self._loss = SoftmaxLoss(
num_words=vocabulary.words_vocab_size(self.backbone.vocab),
embedding_dim=self.backbone.encoder.get_output_dim(),
)