def on_vocab_update(self): num_words = vocabulary.words_vocab_size(self.backbone.vocab) if len(self._loss.softmax_b) != num_words: self._loss = SoftmaxLoss( num_words=num_words, embedding_dim=self._forward_dim, )
def __init__( self, backbone: ModelBackbone, dropout: float = None, bidirectional: bool = False, ) -> None: super(LanguageModelling, self).__init__(backbone) self._empty_prediction = LanguageModellingPrediction( lm_embeddings=numpy.array([]), mask=numpy.array([])) self.bidirectional = bidirectional if not backbone.featurizer.has_word_features: raise ConfigurationError( "`LanguageModelling` defines a word-level next token language model. " "Please check your `features` configuration to enable at least `words` features." ) if backbone.encoder.is_bidirectional() is not bidirectional: raise ConfigurationError( "Bidirectionality of contextualizer must match bidirectionality of " "language model. " f"Contextualizer bidirectional: {backbone.encoder.is_bidirectional()}, " f"language model bidirectional: {bidirectional}") if self.bidirectional: self._forward_dim = backbone.encoder.get_output_dim() // 2 else: self._forward_dim = backbone.encoder.get_output_dim() if dropout: self._dropout = torch.nn.Dropout(dropout) else: self._dropout = lambda x: x self._metrics = Metrics(perplexity={"type": "perplexity"}) self._loss = SoftmaxLoss( num_words=vocabulary.words_vocab_size(self.backbone.vocab), embedding_dim=self._forward_dim, )
def __init__(self, backbone: ModelBackbone, dropout: float = None) -> None: super(LanguageModelling, self).__init__(backbone) if not backbone.featurizer.has_word_features: raise ConfigurationError( "`LanguageModelling` defines a word-level next token language model. " "Please check your `features` configuration to enable at least `words` features." ) self._forward_dim = backbone.encoder.get_output_dim() if dropout: self._dropout = torch.nn.Dropout(dropout) else: self._dropout = lambda x: x self.metrics = {"perplexity": Perplexity()} self._loss = SoftmaxLoss( num_words=vocabulary.words_vocab_size(self.backbone.vocab), embedding_dim=self.backbone.encoder.get_output_dim(), )
def on_vocab_update(self): self._loss = SoftmaxLoss( num_words=vocabulary.words_vocab_size(self.backbone.vocab), embedding_dim=self._forward_dim, )
def on_vocab_update(self): self._loss = SoftmaxLoss( num_words=vocabulary.words_vocab_size(self.backbone.vocab), embedding_dim=self.backbone.encoder.get_output_dim(), )