def _prepare_layers(self) -> None:
self._tf_layers[f"loss.{LABEL}"] = layers.DotProductLoss(
self.config[NUM_NEG],
self.config[LOSS_TYPE],
self.config[MAX_POS_SIM],
self.config[MAX_NEG_SIM],
self.config[USE_MAX_NEG_SIM],
self.config[NEGATIVE_MARGIN_SCALE],
self.config[SCALE_LOSS],
# set to 1 to get deterministic behaviour
parallel_iterations=1 if self.random_seed is not None else 1000,
)
self._tf_layers[f"ffnn.{DIALOGUE}"] = layers.Ffnn(
self.config[HIDDEN_LAYERS_SIZES][DIALOGUE],
self.config[DROP_RATE_DIALOGUE],
self.config[REGULARIZATION_CONSTANT],
self.config[WEIGHT_SPARSITY],
layer_name_suffix=DIALOGUE,
)
self._tf_layers[f"ffnn.{LABEL}"] = layers.Ffnn(
self.config[HIDDEN_LAYERS_SIZES][LABEL],
self.config[DROP_RATE_LABEL],
self.config[REGULARIZATION_CONSTANT],
self.config[WEIGHT_SPARSITY],
layer_name_suffix=LABEL,
)
self._tf_layers["transformer"] = TransformerEncoder(
self.config[NUM_TRANSFORMER_LAYERS],
self.config[TRANSFORMER_SIZE],
self.config[NUM_HEADS],
self.config[TRANSFORMER_SIZE] * 4,
self.config[REGULARIZATION_CONSTANT],
dropout_rate=self.config[DROP_RATE_DIALOGUE],
attention_dropout_rate=self.config[DROP_RATE_ATTENTION],
sparsity=self.config[WEIGHT_SPARSITY],
unidirectional=True,
use_key_relative_position=self.config[KEY_RELATIVE_ATTENTION],
use_value_relative_position=self.config[VALUE_RELATIVE_ATTENTION],
max_relative_position=self.config[MAX_RELATIVE_POSITION],
name=DIALOGUE + "_encoder",
)
self._tf_layers[f"embed.{DIALOGUE}"] = layers.Embed(
self.config[EMBEDDING_DIMENSION],
self.config[REGULARIZATION_CONSTANT],
DIALOGUE,
self.config[SIMILARITY_TYPE],
)
self._tf_layers[f"embed.{LABEL}"] = layers.Embed(
self.config[EMBEDDING_DIMENSION],
self.config[REGULARIZATION_CONSTANT],
LABEL,
self.config[SIMILARITY_TYPE],
)
def _prepare_entity_recognition_layers(self) -> None:
for tag_spec in self._entity_tag_specs:
name = tag_spec.tag_name
num_tags = tag_spec.num_tags
self._tf_layers[f"embed.{name}.logits"] = layers.Embed(
num_tags, self.config[REGULARIZATION_CONSTANT],
f"logits.{name}")
self._tf_layers[f"crf.{name}"] = layers.CRF(
num_tags, self.config[REGULARIZATION_CONSTANT],
self.config[SCALE_LOSS])
self._tf_layers[f"embed.{name}.tags"] = layers.Embed(
self.config[EMBEDDING_DIMENSION],
self.config[REGULARIZATION_CONSTANT],
f"tags.{name}",
)
def _prepare_embed_layers(self, name: Text, prefix: Text = "embed") -> None:
self._tf_layers[f"{prefix}.{name}"] = layers.Embed(
self.config[EMBEDDING_DIMENSION],
self.config[REGULARIZATION_CONSTANT],
name,
self.config[SIMILARITY_TYPE],
)
