def __init__( self, attribute: Text, feature_type: Text, feature_type_signature: List[FeatureSignature], config: Dict[Text, Any], ) -> None: """Creates a new `ConcatenateSparseDenseFeatures` object.""" if not feature_type_signature: raise TFLayerConfigException( "The feature type signature must contain some feature signatures." ) super().__init__( name=f"concatenate_sparse_dense_features_{attribute}_{feature_type}" ) self._check_sparse_input_units(feature_type_signature) self.output_units = self._calculate_output_units( attribute, feature_type_signature, config) # Prepare dropout and sparse-to-dense layers if any sparse tensors are expected self._tf_layers: Dict[Text, tf.keras.layers.Layer] = {} if any([signature.is_sparse for signature in feature_type_signature]): self._prepare_layers_for_sparse_tensors(attribute, feature_type, config)
def __init__( self, attribute: Text, attribute_signature: Dict[Text, List[FeatureSignature]], config: Dict[Text, Any], ) -> None: """Creates a new `RasaFeatureCombiningLayer` object.""" if not attribute_signature or not (attribute_signature.get( SENTENCE, []) or attribute_signature.get(SEQUENCE, [])): raise TFLayerConfigException( "The attribute signature must contain some feature signatures." ) super().__init__(name=f"rasa_feature_combining_layer_{attribute}") self._tf_layers: Dict[Text, tf.keras.layers.Layer] = {} # Prepare sparse-dense combining layers for each present feature type self._feature_types_present = self._get_present_feature_types( attribute_signature) self._prepare_sparse_dense_concat_layers(attribute, attribute_signature, config) # Prepare components for combining sequence- and sentence-level features self._prepare_sequence_sentence_concat(attribute, config) self.output_units = self._calculate_output_units(attribute, config)
def __init__(self, density: float = 0.2, **kwargs: Any) -> None: """Declares instance variables with default values. Args: density: Float between 0 and 1. Approximate fraction of trainable weights. units: Positive integer, dimensionality of the output space. activation: Activation function to use. If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). use_bias: Boolean, whether the layer uses a bias vector. kernel_initializer: Initializer for the `kernel` weights matrix. bias_initializer: Initializer for the bias vector. kernel_regularizer: Regularizer function applied to the `kernel` weights matrix. bias_regularizer: Regularizer function applied to the bias vector. activity_regularizer: Regularizer function applied to the output of the layer (its "activation").. kernel_constraint: Constraint function applied to the `kernel` weights matrix. bias_constraint: Constraint function applied to the bias vector. """ super().__init__(**kwargs) if density < 0.0 or density > 1.0: raise TFLayerConfigException("Layer density must be in [0, 1].") self.density = density
def _chosen_loss(self) -> Callable: """Use loss depending on given option.""" if self.loss_type == MARGIN: return self._loss_margin elif self.loss_type == CROSS_ENTROPY: return self._loss_cross_entropy else: raise TFLayerConfigException( f"Wrong loss type '{self.loss_type}', " f"should be '{MARGIN}' or '{CROSS_ENTROPY}'")
def _check_sparse_input_units( self, feature_type_signature: List[FeatureSignature]) -> None: """Checks that all sparse features have the same last dimension size.""" sparse_units = [ feature_sig.units for feature_sig in feature_type_signature if feature_sig.is_sparse ] if len(set(sparse_units)) > 1: raise TFLayerConfigException( f"All sparse features must have the same last dimension size but found " f"different sizes: {set(sparse_units)}.")
def __init__( self, attribute: Text, attribute_signature: Dict[Text, List[FeatureSignature]], config: Dict[Text, Any], ) -> None: """Creates a new `RasaSequenceLayer` object.""" if not attribute_signature or not attribute_signature.get(SEQUENCE, []): raise TFLayerConfigException( "The attribute signature must contain some sequence-level feature" "signatures but none were found." ) super().__init__(name=f"rasa_sequence_layer_{attribute}") self._tf_layers: Dict[Text, Any] = { self.FEATURE_COMBINING: RasaFeatureCombiningLayer( attribute, attribute_signature, config ), self.FFNN: layers.Ffnn( config[HIDDEN_LAYERS_SIZES][attribute], config[DROP_RATE], config[REGULARIZATION_CONSTANT], config[WEIGHT_SPARSITY], layer_name_suffix=attribute, ), } self._enables_mlm = False # Note: Within TED, masked language modeling becomes just input dropout, # since there is no loss term associated with predicting the masked tokens. self._prepare_masked_language_modeling(attribute, attribute_signature, config) transformer_layers, transformer_units = self._prepare_transformer( attribute, config ) self._has_transformer = transformer_layers > 0 self.output_units = self._calculate_output_units( attribute, transformer_layers, transformer_units, config )
def __init__( self, num_neg: int, loss_type: Text, mu_pos: float, mu_neg: float, use_max_sim_neg: bool, neg_lambda: float, scale_loss: bool, similarity_type: Text, name: Optional[Text] = None, same_sampling: bool = False, constrain_similarities: bool = True, model_confidence: Text = SOFTMAX, ) -> None: """Declare instance variables with default values. Args: num_neg: Positive integer, the number of incorrect labels; the algorithm will minimize their similarity to the input. loss_type: The type of the loss function, either 'cross_entropy' or 'margin'. mu_pos: Float, indicates how similar the algorithm should try to make embedding vectors for correct labels; should be 0.0 < ... < 1.0 for 'cosine' similarity type. mu_neg: Float, maximum negative similarity for incorrect labels, should be -1.0 < ... < 1.0 for 'cosine' similarity type. use_max_sim_neg: Boolean, if 'True' the algorithm only minimizes maximum similarity over incorrect intent labels, used only if 'loss_type' is set to 'margin'. neg_lambda: Float, the scale of how important is to minimize the maximum similarity between embeddings of different labels, used only if 'loss_type' is set to 'margin'. scale_loss: Boolean, if 'True' scale loss inverse proportionally to the confidence of the correct prediction. similarity_type: Similarity measure to use, either 'cosine' or 'inner'. name: Optional name of the layer. same_sampling: Boolean, if 'True' sample same negative labels for the whole batch. constrain_similarities: Boolean, if 'True' applies sigmoid on all similarity terms and adds to the loss function to ensure that similarity values are approximately bounded. Used inside _loss_cross_entropy() only. model_confidence: Model confidence to be returned during inference. Possible values - 'softmax', 'cosine' and 'inner'. Raises: LayerConfigException: When `similarity_type` is not one of 'cosine' or 'inner'. """ super().__init__(name=name) self.num_neg = num_neg self.loss_type = loss_type self.mu_pos = mu_pos self.mu_neg = mu_neg self.use_max_sim_neg = use_max_sim_neg self.neg_lambda = neg_lambda self.scale_loss = scale_loss self.same_sampling = same_sampling self.constrain_similarities = constrain_similarities self.model_confidence = model_confidence self.similarity_type = similarity_type if self.similarity_type not in {COSINE, INNER}: raise TFLayerConfigException( f"Wrong similarity type '{self.similarity_type}', " f"should be '{COSINE}' or '{INNER}'.")