def _prepare_layers(self) -> None: self._tf_layers[f"loss.{LABEL}"] = layers.DotProductLoss( self.config[NUM_NEG], self.config[LOSS_TYPE], self.config[MAX_POS_SIM], self.config[MAX_NEG_SIM], self.config[USE_MAX_NEG_SIM], self.config[NEGATIVE_MARGIN_SCALE], self.config[SCALE_LOSS], # set to 1 to get deterministic behaviour parallel_iterations=1 if self.random_seed is not None else 1000, ) self._tf_layers[f"ffnn.{DIALOGUE}"] = layers.Ffnn( self.config[HIDDEN_LAYERS_SIZES][DIALOGUE], self.config[DROP_RATE_DIALOGUE], self.config[REGULARIZATION_CONSTANT], self.config[WEIGHT_SPARSITY], layer_name_suffix=DIALOGUE, ) self._tf_layers[f"ffnn.{LABEL}"] = layers.Ffnn( self.config[HIDDEN_LAYERS_SIZES][LABEL], self.config[DROP_RATE_LABEL], self.config[REGULARIZATION_CONSTANT], self.config[WEIGHT_SPARSITY], layer_name_suffix=LABEL, ) self._tf_layers["transformer"] = TransformerEncoder( self.config[NUM_TRANSFORMER_LAYERS], self.config[TRANSFORMER_SIZE], self.config[NUM_HEADS], self.config[TRANSFORMER_SIZE] * 4, self.config[REGULARIZATION_CONSTANT], dropout_rate=self.config[DROP_RATE_DIALOGUE], attention_dropout_rate=self.config[DROP_RATE_ATTENTION], sparsity=self.config[WEIGHT_SPARSITY], unidirectional=True, use_key_relative_position=self.config[KEY_RELATIVE_ATTENTION], use_value_relative_position=self.config[VALUE_RELATIVE_ATTENTION], max_relative_position=self.config[MAX_RELATIVE_POSITION], name=DIALOGUE + "_encoder", ) self._tf_layers[f"embed.{DIALOGUE}"] = layers.Embed( self.config[EMBEDDING_DIMENSION], self.config[REGULARIZATION_CONSTANT], DIALOGUE, self.config[SIMILARITY_TYPE], ) self._tf_layers[f"embed.{LABEL}"] = layers.Embed( self.config[EMBEDDING_DIMENSION], self.config[REGULARIZATION_CONSTANT], LABEL, self.config[SIMILARITY_TYPE], )
def _prepare_entity_recognition_layers(self) -> None: for tag_spec in self._entity_tag_specs: name = tag_spec.tag_name num_tags = tag_spec.num_tags self._tf_layers[f"embed.{name}.logits"] = layers.Embed( num_tags, self.config[REGULARIZATION_CONSTANT], f"logits.{name}") self._tf_layers[f"crf.{name}"] = layers.CRF( num_tags, self.config[REGULARIZATION_CONSTANT], self.config[SCALE_LOSS]) self._tf_layers[f"embed.{name}.tags"] = layers.Embed( self.config[EMBEDDING_DIMENSION], self.config[REGULARIZATION_CONSTANT], f"tags.{name}", )
def _prepare_embed_layers(self, name: Text, prefix: Text = "embed") -> None: self._tf_layers[f"{prefix}.{name}"] = layers.Embed( self.config[EMBEDDING_DIMENSION], self.config[REGULARIZATION_CONSTANT], name, self.config[SIMILARITY_TYPE], )