def __init__( self, vocab, embedding_size=50, embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=False, dropout_rate=0.0, initializer=None, regularizer=None, **kwargs ): super(CategoricalSparseEncoder, self).__init__() logger.debug(' {}'.format(self.name)) logger.debug(' Embed') self.embed = Embed( vocab=vocab, embedding_size=embedding_size, representation='sparse', embeddings_trainable=embeddings_trainable, pretrained_embeddings=pretrained_embeddings, embeddings_on_cpu=embeddings_on_cpu, dropout_rate=dropout_rate, initializer=initializer, regularizer=regularizer )
def __init__( self, vocab: List[str], embedding_size: int = 50, embeddings_trainable: bool = True, pretrained_embeddings: Optional[str] = None, embeddings_on_cpu: bool = False, dropout: float = 0.0, embedding_initializer: Optional[Union[str, Dict]] = None, **kwargs, ): super().__init__() logger.debug(f" {self.name}") logger.debug(" Embed") self.embed = Embed( vocab=vocab, embedding_size=embedding_size, representation="sparse", embeddings_trainable=embeddings_trainable, pretrained_embeddings=pretrained_embeddings, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=embedding_initializer, ) self.embedding_size = self.embed.embedding_size
def __init__( self, vocab, embedding_size=50, embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=False, dropout=0.0, embedding_initializer=None, embedding_regularizer=None, **kwargs ): super().__init__() logger.debug(' {}'.format(self.name)) logger.debug(' Embed') self.embed = Embed( vocab=vocab, embedding_size=embedding_size, representation='sparse', embeddings_trainable=embeddings_trainable, pretrained_embeddings=pretrained_embeddings, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=embedding_initializer, embedding_regularizer=embedding_regularizer ) self.embedding_size = self.embed.embedding_size
def test_embed( vocab: List[str], embedding_size: int, representation: str, ): embed = Embed( vocab=vocab, embedding_size=embedding_size, representation=representation, ).to(DEVICE) inputs = torch.randint(0, 2, size=(2, 1)).bool().to(DEVICE) outputs = embed(inputs) assert outputs.shape[1:] == embed.output_shape
def __init__( self, input_features: Dict[str, "InputFeature"] = None, config: TabTransformerCombinerConfig = None, **kwargs ): super().__init__(input_features) self.name = "TabTransformerCombiner" logger.debug(f"Initializing {self.name}") if config.reduce_output is None: raise ValueError("TabTransformer requires the `reduce_output` " "parameter") self.reduce_output = config.reduce_output self.reduce_sequence = SequenceReducer(reduce_mode=config.reduce_output) self.supports_masking = True self.embed_input_feature_name = config.embed_input_feature_name if self.embed_input_feature_name: vocab = [ i_f for i_f in input_features if input_features[i_f].type != NUMERICAL or input_features[i_f].type != BINARY ] if self.embed_input_feature_name == "add": self.embed_i_f_name_layer = Embed(vocab, config.hidden_size, force_embedding_size=True) projector_size = config.hidden_size elif isinstance(self.embed_input_feature_name, int): if self.embed_input_feature_name > config.hidden_size: raise ValueError( "TabTransformer parameter " "`embed_input_feature_name` " "specified integer value ({}) " "needs to be smaller than " "`hidden_size` ({}).".format(self.embed_input_feature_name, config.hidden_size) ) self.embed_i_f_name_layer = Embed( vocab, self.embed_input_feature_name, force_embedding_size=True, ) projector_size = config.hidden_size - self.embed_input_feature_name else: raise ValueError( "TabTransformer parameter " "`embed_input_feature_name` " "should be either None, an integer or `add`, " "the current value is " "{}".format(self.embed_input_feature_name) ) else: projector_size = config.hidden_size logger.debug(" Projectors") self.unembeddable_features = [] self.embeddable_features = [] for i_f in input_features: if input_features[i_f].type in {NUMERICAL, BINARY}: self.unembeddable_features.append(input_features[i_f].name) else: self.embeddable_features.append(input_features[i_f].name) self.projectors = ModuleList() for i_f in self.embeddable_features: flatten_size = self.get_flatten_size(input_features[i_f].output_shape) self.projectors.append(Linear(flatten_size[0], projector_size)) # input to layer_norm are the encoder outputs for unembeddable features, # which are numerical or binary features. These should be 2-dim # tensors. Size should be concatenation of these tensors. concatenated_unembeddable_encoders_size = 0 for i_f in self.unembeddable_features: concatenated_unembeddable_encoders_size += input_features[i_f].output_shape[0] self.layer_norm = torch.nn.LayerNorm(concatenated_unembeddable_encoders_size) logger.debug(" TransformerStack") self.transformer_stack = TransformerStack( input_size=config.hidden_size, sequence_size=len(self.embeddable_features), hidden_size=config.hidden_size, # todo: can we just use projector_size? # hidden_size, num_heads=config.num_heads, fc_size=config.transformer_fc_size, num_layers=config.num_layers, dropout=config.dropout, ) logger.debug(" FCStack") transformer_hidden_size = self.transformer_stack.layers[-1].output_shape[-1] # determine input size to fully connected layer based on reducer if config.reduce_output == "concat": num_embeddable_features = len(self.embeddable_features) fc_input_size = num_embeddable_features * transformer_hidden_size else: fc_input_size = transformer_hidden_size if len(self.embeddable_features) > 0 else 0 self.fc_stack = FCStack( fc_input_size + concatenated_unembeddable_encoders_size, layers=config.fc_layers, num_layers=config.num_fc_layers, default_fc_size=config.fc_size, default_use_bias=config.use_bias, default_weights_initializer=config.weights_initializer, default_bias_initializer=config.bias_initializer, default_norm=config.norm, default_norm_params=config.norm_params, default_activation=config.fc_activation, default_dropout=config.fc_dropout, fc_residual=config.fc_residual, ) # Create empty tensor of shape [1, 0] to use as hidden in case there are no category or numeric/binary features. self.register_buffer("empty_hidden", torch.empty([1, 0])) self.register_buffer("embeddable_features_indices", torch.arange(0, len(self.embeddable_features)))
def __init__( self, embedding_size=10, embeddings_on_cpu=False, fc_layers=None, num_fc_layers=0, fc_size=10, use_bias=True, weights_initializer='glorot_uniform', bias_initializer='zeros', weights_regularizer=None, bias_regularizer=None, activity_regularizer=None, # weights_constraint=None, # bias_constraint=None, norm=None, norm_params=None, activation='relu', dropout=0, **kwargs): """ :param embedding_size: it is the maximum embedding size, the actual size will be `min(vocaularyb_size, embedding_size)` for `dense` representations and exacly `vocaularyb_size` for the `sparse` encoding, where `vocabulary_size` is the number of different strings appearing in the training set in the column the feature is named after (plus 1 for `<UNK>`). :type embedding_size: Integer :param embeddings_on_cpu: by default embedings matrices are stored on GPU memory if a GPU is used, as it allows for faster access, but in some cases the embedding matrix may be really big and this parameter forces the placement of the embedding matrix in regular memroy and the CPU is used to resolve them, slightly slowing down the process as a result of data transfer between CPU and GPU memory. :param fc_layers: list of dictionaries containing the parameters of all the fully connected layers :type fc_layers: List :param num_fc_layers: Number of stacked fully connected layers :type num_fc_layers: Integer :param fc_size: Size of each layer :type fc_size: Integer :param use_bias: bool determines where to use a bias vector :type use_bias: bool :param weights_initializer: Initializer for the weights (aka kernel) matrix :type weights_initializer: string :param bias_initializer: Initializer for the bias vector :type bias_initializer: string :param weights_regularizer: regularizer applied to the weights (kernal) matrix :type weights_regularizer: string :param bias_regularizer: reguralizer function applied to biase vector. :type bias_regularizer: string :param activity_regularizer: Regularizer applied to the output of the layer (activation) :type activity_regularizer: string :param norm: type of normalization to use 'batch' or 'layer' :type norm: string, default None :param norm_params: parameters to pass to normalization function :type norm_params: dictionary :param activation: Activation function to use. :type activation: string :param dropout: determines if there should be a dropout layer before returning the encoder output. :type dropout: float """ super().__init__() logger.debug(' {}'.format(self.name)) logger.debug(' year FCStack') self.year_fc = FCStack( num_layers=1, default_fc_size=1, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=weights_constraint, # default_bias_constraint=bias_constraint, default_norm=None, default_norm_params=None, default_activation=None, default_dropout=dropout, ) logger.debug(' month Embed') self.embed_month = Embed([str(i) for i in range(12)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' day Embed') self.embed_day = Embed([str(i) for i in range(31)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' weekday Embed') self.embed_weekday = Embed([str(i) for i in range(7)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' yearday Embed') self.embed_yearday = Embed([str(i) for i in range(366)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' hour Embed') self.embed_hour = Embed([str(i) for i in range(24)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' minute Embed') self.embed_minute = Embed([str(i) for i in range(60)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' second Embed') self.embed_second = Embed([str(i) for i in range(60)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' FCStack') self.fc_stack = FCStack( layers=fc_layers, num_layers=num_fc_layers, default_fc_size=fc_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=weights_constraint, # default_bias_constraint=bias_constraint, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, )
def __init__( self, embedding_size: int = 10, embeddings_on_cpu: bool = False, fc_layers: Optional[List[Dict]] = None, num_fc_layers: int = 0, output_size: int = 10, use_bias: bool = True, weights_initializer: str = "xavier_uniform", bias_initializer: str = "zeros", norm: Optional[str] = None, norm_params: Optional[Dict] = None, activation: str = "relu", dropout: float = 0, **kwargs, ): """ :param embedding_size: The maximum embedding size, the actual size will be `min(vocabulary_size, embedding_size)` for `dense` representations and exactly `vocabulary_size` for the `sparse` encoding, where `vocabulary_size` is the number of different strings appearing in the training set in the column the feature is named after (plus 1 for `<UNK>`). :type embedding_size: Integer :param embeddings_on_cpu: by default embeddings matrices are stored on GPU memory if a GPU is used, as it allows for faster access, but in some cases the embedding matrix may be really big and this parameter forces the placement of the embedding matrix in regular memory and the CPU is used to resolve them, slightly slowing down the process as a result of data transfer between CPU and GPU memory. :param fc_layers: list of dictionaries containing the parameters of all the fully connected layers. :type fc_layers: List :param num_fc_layers: Number of stacked fully connected layers. :type num_fc_layers: Integer :param output_size: Size of each layer. :type output_size: Integer :param use_bias: bool determines where to use a bias vector. :type use_bias: bool :param weights_initializer: Initializer for the weights (aka kernel) matrix. :type weights_initializer: string :param bias_initializer: Initializer for the bias vector. :type bias_initializer: string :param norm: type of normalization to use 'batch' or 'layer'. :type norm: string, default None :param norm_params: parameters to pass to normalization function. :type norm_params: dictionary :param activation: Activation function to use. :type activation: string :param dropout: determines if there should be a dropout layer before returning the encoder output. :type dropout: float """ super().__init__() logger.debug(f" {self.name}") logger.debug(" year FCStack") self.year_fc = FCStack( first_layer_input_size=1, num_layers=1, default_output_size=1, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_norm=None, default_norm_params=None, default_activation=None, default_dropout=dropout, ) logger.debug(" month Embed") self.embed_month = Embed( [str(i) for i in range(12)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" day Embed") self.embed_day = Embed( [str(i) for i in range(31)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" weekday Embed") self.embed_weekday = Embed( [str(i) for i in range(7)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" yearday Embed") self.embed_yearday = Embed( [str(i) for i in range(366)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" hour Embed") self.embed_hour = Embed( [str(i) for i in range(24)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" minute Embed") self.embed_minute = Embed( [str(i) for i in range(60)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" second Embed") self.embed_second = Embed( [str(i) for i in range(60)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) # Summed sizes of all of the embeddings. fc_layer_input_size = ( self.year_fc.output_shape[0] + self.embed_month.output_shape[0] + self.embed_day.output_shape[0] + self.embed_weekday.output_shape[0] + self.embed_yearday.output_shape[0] + self.embed_hour.output_shape[0] + self.embed_minute.output_shape[0] + self.embed_second.output_shape[0] + 1 # for periodic_second_of_day. ) logger.debug(" FCStack") self.fc_stack = FCStack( first_layer_input_size=fc_layer_input_size, layers=fc_layers, num_layers=num_fc_layers, default_output_size=output_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, )
def __init__( self, embedding_size=10, embeddings_on_cpu=False, fc_layers=None, num_fc_layers=0, fc_size=10, use_bias=True, weights_initializer='glorot_uniform', bias_initializer='zeros', weights_regularizer=None, bias_regularizer=None, activity_regularizer=None, # weights_constraint=None, # bias_constraint=None, norm=None, norm_params=None, activation='relu', dropout=0, reduce_output='sum', **kwargs): """ :param embedding_size: it is the maximum embedding size, the actual size will be `min(vocaularyb_size, embedding_size)` for `dense` representations and exacly `vocaularyb_size` for the `sparse` encoding, where `vocabulary_size` is the number of different strings appearing in the training set in the column the feature is named after (plus 1 for `<UNK>`). :type embedding_size: Integer :param embeddings_on_cpu: by default embedings matrices are stored on GPU memory if a GPU is used, as it allows for faster access, but in some cases the embedding matrix may be really big and this parameter forces the placement of the embedding matrix in regular memroy and the CPU is used to resolve them, slightly slowing down the process as a result of data transfer between CPU and GPU memory. :param dropout: determines if there should be a dropout layer before returning the encoder output. :type dropout: Boolean :param initializer: the initializer to use. If `None`, the default initialized of each variable is used (`glorot_uniform` in most cases). Options are: `constant`, `identity`, `zeros`, `ones`, `orthogonal`, `normal`, `uniform`, `truncated_normal`, `variance_scaling`, `glorot_normal`, `glorot_uniform`, `xavier_normal`, `xavier_uniform`, `he_normal`, `he_uniform`, `lecun_normal`, `lecun_uniform`. Alternatively it is possible to specify a dictionary with a key `type` that identifies the type of initialzier and other keys for its parameters, e.g. `{type: normal, mean: 0, stddev: 0}`. To know the parameters of each initializer, please refer to TensorFlow's documentation. :type initializer: str :param regularize: if `True` the embedding wieghts are added to the set of weights that get reularized by a regularization loss (if the `regularization_lambda` in `training` is greater than 0). :type regularize: Boolean """ super(H3Embed, self).__init__() logger.debug(' {}'.format(self.name)) self.embedding_size = embedding_size self.reduce_output = reduce_output self.reduce_sequence = SequenceReducer(reduce_mode=reduce_output) logger.debug(' mode Embed') self.embed_mode = Embed([str(i) for i in range(3)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, force_embedding_size=True, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' edge Embed') self.embed_edge = Embed([str(i) for i in range(7)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, force_embedding_size=True, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' resolution Embed') self.embed_resolution = Embed( [str(i) for i in range(16)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, force_embedding_size=True, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' base cell Embed') self.embed_base_cell = Embed([str(i) for i in range(122)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, force_embedding_size=True, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' cells Embed') self.embed_cells = Embed([str(i) for i in range(8)], embedding_size, representation='dense', embeddings_trainable=True, pretrained_embeddings=None, force_embedding_size=True, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, embedding_regularizer=weights_regularizer) logger.debug(' FCStack') self.fc_stack = FCStack( layers=fc_layers, num_layers=num_fc_layers, default_fc_size=fc_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=weights_constraint, # default_bias_constraint=bias_constraint, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, )
def __init__( self, input_features=None, embed_input_feature_name=None, # None or embedding size or "add" num_layers=1, hidden_size=256, num_heads=8, transformer_fc_size=256, dropout=0.1, fc_layers=None, num_fc_layers=0, fc_size=256, use_bias=True, weights_initializer='glorot_uniform', bias_initializer='zeros', weights_regularizer=None, bias_regularizer=None, activity_regularizer=None, # weights_constraint=None, # bias_constraint=None, norm=None, norm_params=None, fc_activation='relu', fc_dropout=0, fc_residual=False, reduce_output='concat', **kwargs ): super().__init__() logger.debug(' {}'.format(self.name)) if reduce_output is None: raise ValueError("TabTransformer requires the `resude_output` " "parametr") self.reduce_output = reduce_output self.reduce_sequence = SequenceReducer(reduce_mode=reduce_output) self.supports_masking = True self.layer_norm = LayerNormalization() self.embed_input_feature_name = embed_input_feature_name if self.embed_input_feature_name: vocab = [i_f for i_f in input_features if i_f[TYPE] != NUMERICAL or i_f[TYPE] != BINARY] if self.embed_input_feature_name == 'add': self.embed_i_f_name_layer = Embed(vocab, hidden_size, force_embedding_size=True) projector_size = hidden_size elif isinstance(self.embed_input_feature_name, int): if self.embed_input_feature_name > hidden_size: raise ValueError( "TabTransformer parameter " "`embed_input_feature_name` " "specified integer value ({}) " "needs to be smaller than " "`hidden_size` ({}).".format( self.embed_input_feature_name, hidden_size )) self.embed_i_f_name_layer = Embed( vocab, self.embed_input_feature_name, force_embedding_size=True, ) projector_size = hidden_size - self.embed_input_feature_name else: raise ValueError("TabTransformer parameter " "`embed_input_feature_name` " "should be either None, an integer or `add`, " "the current value is " "{}".format(self.embed_input_feature_name)) else: projector_size = hidden_size logger.debug(' Projectors') self.projectors = [Dense(projector_size) for i_f in input_features if i_f[TYPE] != NUMERICAL and i_f[TYPE] != BINARY] self.skip_features = [i_f[NAME] for i_f in input_features if i_f[TYPE] == NUMERICAL or i_f[TYPE] == BINARY] logger.debug(' TransformerStack') self.transformer_stack = TransformerStack( hidden_size=hidden_size, num_heads=num_heads, fc_size=transformer_fc_size, num_layers=num_layers, dropout=dropout ) logger.debug(' FCStack') self.fc_stack = FCStack( layers=fc_layers, num_layers=num_fc_layers, default_fc_size=fc_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_weights_regularizer=weights_regularizer, default_bias_regularizer=bias_regularizer, default_activity_regularizer=activity_regularizer, # default_weights_constraint=weights_constraint, # default_bias_constraint=bias_constraint, default_norm=norm, default_norm_params=norm_params, default_activation=fc_activation, default_dropout=fc_dropout, fc_residual=fc_residual, )
def __init__( self, embedding_size: int = 10, embeddings_on_cpu: bool = False, fc_layers: Optional[List] = None, num_fc_layers: int = 0, output_size: int = 10, use_bias: bool = True, weights_initializer: str = "xavier_uniform", bias_initializer: str = "zeros", norm: str = None, norm_params: Dict = None, activation: str = "relu", dropout: float = 0, reduce_output: str = "sum", **kwargs, ): """ :param embedding_size: it is the maximum embedding size, the actual size will be `min(vocabulary_size, embedding_size)` for `dense` representations and exactly `vocabulary_size` for the `sparse` encoding, where `vocabulary_size` is the number of different strings appearing in the training set in the column the feature is named after (plus 1 for `<UNK>`). :type embedding_size: Integer :param embeddings_on_cpu: by default embeddings matrices are stored on GPU memory if a GPU is used, as it allows for faster access, but in some cases the embedding matrix may be really big and this parameter forces the placement of the embedding matrix in regular memory and the CPU is used to resolve them, slightly slowing down the process as a result of data transfer between CPU and GPU memory. :param dropout: determines if there should be a dropout layer before returning the encoder output. :type dropout: Boolean """ super().__init__() logger.debug(f" {self.name}") self.embedding_size = embedding_size self.reduce_output = reduce_output self.reduce_sequence = SequenceReducer(reduce_mode=reduce_output) logger.debug(" mode Embed") self.embed_mode = Embed( [str(i) for i in range(3)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, force_embedding_size=True, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" edge Embed") self.embed_edge = Embed( [str(i) for i in range(7)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, force_embedding_size=True, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" resolution Embed") self.embed_resolution = Embed( [str(i) for i in range(16)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, force_embedding_size=True, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" base cell Embed") self.embed_base_cell = Embed( [str(i) for i in range(122)], embedding_size, representation="dense", embeddings_trainable=True, pretrained_embeddings=None, force_embedding_size=True, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" cells Embed") self.embed_cells = EmbedSequence( [str(i) for i in range(8)], embedding_size, max_sequence_length=(H3_INPUT_SIZE - 4), representation="dense", embeddings_trainable=True, pretrained_embeddings=None, force_embedding_size=True, embeddings_on_cpu=embeddings_on_cpu, dropout=dropout, embedding_initializer=weights_initializer, ) logger.debug(" FCStack") self.fc_stack = FCStack( first_layer_input_size=embedding_size, layers=fc_layers, num_layers=num_fc_layers, default_output_size=output_size, default_use_bias=use_bias, default_weights_initializer=weights_initializer, default_bias_initializer=bias_initializer, default_norm=norm, default_norm_params=norm_params, default_activation=activation, default_dropout=dropout, )