def __init__(
self,
vocab,
embedding_size=50,
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=False,
dropout_rate=0.0,
initializer=None,
regularizer=None,
**kwargs
):
super(CategoricalSparseEncoder, self).__init__()
logger.debug(' {}'.format(self.name))
logger.debug(' Embed')
self.embed = Embed(
vocab=vocab,
embedding_size=embedding_size,
representation='sparse',
embeddings_trainable=embeddings_trainable,
pretrained_embeddings=pretrained_embeddings,
embeddings_on_cpu=embeddings_on_cpu,
dropout_rate=dropout_rate,
initializer=initializer,
regularizer=regularizer
)
def __init__(
self,
vocab: List[str],
embedding_size: int = 50,
embeddings_trainable: bool = True,
pretrained_embeddings: Optional[str] = None,
embeddings_on_cpu: bool = False,
dropout: float = 0.0,
embedding_initializer: Optional[Union[str, Dict]] = None,
**kwargs,
):
super().__init__()
logger.debug(f" {self.name}")
logger.debug(" Embed")
self.embed = Embed(
vocab=vocab,
embedding_size=embedding_size,
representation="sparse",
embeddings_trainable=embeddings_trainable,
pretrained_embeddings=pretrained_embeddings,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=embedding_initializer,
)
self.embedding_size = self.embed.embedding_size
def __init__(
self,
vocab,
embedding_size=50,
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=False,
dropout=0.0,
embedding_initializer=None,
embedding_regularizer=None,
**kwargs
):
super().__init__()
logger.debug(' {}'.format(self.name))
logger.debug(' Embed')
self.embed = Embed(
vocab=vocab,
embedding_size=embedding_size,
representation='sparse',
embeddings_trainable=embeddings_trainable,
pretrained_embeddings=pretrained_embeddings,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=embedding_initializer,
embedding_regularizer=embedding_regularizer
)
self.embedding_size = self.embed.embedding_size
def test_embed(
vocab: List[str],
embedding_size: int,
representation: str,
):
embed = Embed(
vocab=vocab,
embedding_size=embedding_size,
representation=representation,
).to(DEVICE)
inputs = torch.randint(0, 2, size=(2, 1)).bool().to(DEVICE)
outputs = embed(inputs)
assert outputs.shape[1:] == embed.output_shape
def __init__(
self, input_features: Dict[str, "InputFeature"] = None, config: TabTransformerCombinerConfig = None, **kwargs
):
super().__init__(input_features)
self.name = "TabTransformerCombiner"
logger.debug(f"Initializing {self.name}")
if config.reduce_output is None:
raise ValueError("TabTransformer requires the `reduce_output` " "parameter")
self.reduce_output = config.reduce_output
self.reduce_sequence = SequenceReducer(reduce_mode=config.reduce_output)
self.supports_masking = True
self.embed_input_feature_name = config.embed_input_feature_name
if self.embed_input_feature_name:
vocab = [
i_f
for i_f in input_features
if input_features[i_f].type != NUMERICAL or input_features[i_f].type != BINARY
]
if self.embed_input_feature_name == "add":
self.embed_i_f_name_layer = Embed(vocab, config.hidden_size, force_embedding_size=True)
projector_size = config.hidden_size
elif isinstance(self.embed_input_feature_name, int):
if self.embed_input_feature_name > config.hidden_size:
raise ValueError(
"TabTransformer parameter "
"`embed_input_feature_name` "
"specified integer value ({}) "
"needs to be smaller than "
"`hidden_size` ({}).".format(self.embed_input_feature_name, config.hidden_size)
)
self.embed_i_f_name_layer = Embed(
vocab,
self.embed_input_feature_name,
force_embedding_size=True,
)
projector_size = config.hidden_size - self.embed_input_feature_name
else:
raise ValueError(
"TabTransformer parameter "
"`embed_input_feature_name` "
"should be either None, an integer or `add`, "
"the current value is "
"{}".format(self.embed_input_feature_name)
)
else:
projector_size = config.hidden_size
logger.debug(" Projectors")
self.unembeddable_features = []
self.embeddable_features = []
for i_f in input_features:
if input_features[i_f].type in {NUMERICAL, BINARY}:
self.unembeddable_features.append(input_features[i_f].name)
else:
self.embeddable_features.append(input_features[i_f].name)
self.projectors = ModuleList()
for i_f in self.embeddable_features:
flatten_size = self.get_flatten_size(input_features[i_f].output_shape)
self.projectors.append(Linear(flatten_size[0], projector_size))
# input to layer_norm are the encoder outputs for unembeddable features,
# which are numerical or binary features. These should be 2-dim
# tensors. Size should be concatenation of these tensors.
concatenated_unembeddable_encoders_size = 0
for i_f in self.unembeddable_features:
concatenated_unembeddable_encoders_size += input_features[i_f].output_shape[0]
self.layer_norm = torch.nn.LayerNorm(concatenated_unembeddable_encoders_size)
logger.debug(" TransformerStack")
self.transformer_stack = TransformerStack(
input_size=config.hidden_size,
sequence_size=len(self.embeddable_features),
hidden_size=config.hidden_size,
# todo: can we just use projector_size? # hidden_size,
num_heads=config.num_heads,
fc_size=config.transformer_fc_size,
num_layers=config.num_layers,
dropout=config.dropout,
)
logger.debug(" FCStack")
transformer_hidden_size = self.transformer_stack.layers[-1].output_shape[-1]
# determine input size to fully connected layer based on reducer
if config.reduce_output == "concat":
num_embeddable_features = len(self.embeddable_features)
fc_input_size = num_embeddable_features * transformer_hidden_size
else:
fc_input_size = transformer_hidden_size if len(self.embeddable_features) > 0 else 0
self.fc_stack = FCStack(
fc_input_size + concatenated_unembeddable_encoders_size,
layers=config.fc_layers,
num_layers=config.num_fc_layers,
default_fc_size=config.fc_size,
default_use_bias=config.use_bias,
default_weights_initializer=config.weights_initializer,
default_bias_initializer=config.bias_initializer,
default_norm=config.norm,
default_norm_params=config.norm_params,
default_activation=config.fc_activation,
default_dropout=config.fc_dropout,
fc_residual=config.fc_residual,
)
# Create empty tensor of shape [1, 0] to use as hidden in case there are no category or numeric/binary features.
self.register_buffer("empty_hidden", torch.empty([1, 0]))
self.register_buffer("embeddable_features_indices", torch.arange(0, len(self.embeddable_features)))
def __init__(
self,
embedding_size=10,
embeddings_on_cpu=False,
fc_layers=None,
num_fc_layers=0,
fc_size=10,
use_bias=True,
weights_initializer='glorot_uniform',
bias_initializer='zeros',
weights_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
# weights_constraint=None,
# bias_constraint=None,
norm=None,
norm_params=None,
activation='relu',
dropout=0,
**kwargs):
"""
:param embedding_size: it is the maximum embedding size, the actual
size will be `min(vocaularyb_size, embedding_size)`
for `dense` representations and exacly `vocaularyb_size`
for the `sparse` encoding, where `vocabulary_size` is
the number of different strings appearing in the training set
in the column the feature is named after (plus 1 for `<UNK>`).
:type embedding_size: Integer
:param embeddings_on_cpu: by default embedings matrices are stored
on GPU memory if a GPU is used, as it allows
for faster access, but in some cases the embedding matrix
may be really big and this parameter forces the placement
of the embedding matrix in regular memroy and the CPU is used
to resolve them, slightly slowing down the process
as a result of data transfer between CPU and GPU memory.
:param fc_layers: list of dictionaries containing the parameters of
all the fully connected layers
:type fc_layers: List
:param num_fc_layers: Number of stacked fully connected layers
:type num_fc_layers: Integer
:param fc_size: Size of each layer
:type fc_size: Integer
:param use_bias: bool determines where to use a bias vector
:type use_bias: bool
:param weights_initializer: Initializer for the weights (aka kernel)
matrix
:type weights_initializer: string
:param bias_initializer: Initializer for the bias vector
:type bias_initializer: string
:param weights_regularizer: regularizer applied to the weights
(kernal) matrix
:type weights_regularizer: string
:param bias_regularizer: reguralizer function applied to biase vector.
:type bias_regularizer: string
:param activity_regularizer: Regularizer applied to the output of the
layer (activation)
:type activity_regularizer: string
:param norm: type of normalization to use 'batch' or 'layer'
:type norm: string, default None
:param norm_params: parameters to pass to normalization function
:type norm_params: dictionary
:param activation: Activation function to use.
:type activation: string
:param dropout: determines if there should be a dropout layer before
returning the encoder output.
:type dropout: float
"""
super().__init__()
logger.debug(' {}'.format(self.name))
logger.debug(' year FCStack')
self.year_fc = FCStack(
num_layers=1,
default_fc_size=1,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_weights_regularizer=weights_regularizer,
default_bias_regularizer=bias_regularizer,
default_activity_regularizer=activity_regularizer,
# default_weights_constraint=weights_constraint,
# default_bias_constraint=bias_constraint,
default_norm=None,
default_norm_params=None,
default_activation=None,
default_dropout=dropout,
)
logger.debug(' month Embed')
self.embed_month = Embed([str(i) for i in range(12)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' day Embed')
self.embed_day = Embed([str(i) for i in range(31)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' weekday Embed')
self.embed_weekday = Embed([str(i) for i in range(7)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' yearday Embed')
self.embed_yearday = Embed([str(i) for i in range(366)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' hour Embed')
self.embed_hour = Embed([str(i) for i in range(24)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' minute Embed')
self.embed_minute = Embed([str(i) for i in range(60)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' second Embed')
self.embed_second = Embed([str(i) for i in range(60)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' FCStack')
self.fc_stack = FCStack(
layers=fc_layers,
num_layers=num_fc_layers,
default_fc_size=fc_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_weights_regularizer=weights_regularizer,
default_bias_regularizer=bias_regularizer,
default_activity_regularizer=activity_regularizer,
# default_weights_constraint=weights_constraint,
# default_bias_constraint=bias_constraint,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout=dropout,
)
def __init__(
self,
embedding_size: int = 10,
embeddings_on_cpu: bool = False,
fc_layers: Optional[List[Dict]] = None,
num_fc_layers: int = 0,
output_size: int = 10,
use_bias: bool = True,
weights_initializer: str = "xavier_uniform",
bias_initializer: str = "zeros",
norm: Optional[str] = None,
norm_params: Optional[Dict] = None,
activation: str = "relu",
dropout: float = 0,
**kwargs,
):
"""
:param embedding_size: The maximum embedding size, the actual size
will be `min(vocabulary_size, embedding_size)` for `dense`
representations and exactly `vocabulary_size` for the `sparse`
encoding, where `vocabulary_size` is the number of different
strings appearing in the training set in the column the feature
is named after (plus 1 for `<UNK>`).
:type embedding_size: Integer
:param embeddings_on_cpu: by default embeddings matrices are stored
on GPU memory if a GPU is used, as it allows for faster access,
but in some cases the embedding matrix may be really big and
this parameter forces the placement of the embedding matrix in
regular memory and the CPU is used to resolve them, slightly
slowing down the process as a result of data transfer between
CPU and GPU memory.
:param fc_layers: list of dictionaries containing the parameters of
all the fully connected layers.
:type fc_layers: List
:param num_fc_layers: Number of stacked fully connected layers.
:type num_fc_layers: Integer
:param output_size: Size of each layer.
:type output_size: Integer
:param use_bias: bool determines where to use a bias vector.
:type use_bias: bool
:param weights_initializer: Initializer for the weights (aka kernel)
matrix.
:type weights_initializer: string
:param bias_initializer: Initializer for the bias vector.
:type bias_initializer: string
:param norm: type of normalization to use 'batch' or 'layer'.
:type norm: string, default None
:param norm_params: parameters to pass to normalization function.
:type norm_params: dictionary
:param activation: Activation function to use.
:type activation: string
:param dropout: determines if there should be a dropout layer before
returning the encoder output.
:type dropout: float
"""
super().__init__()
logger.debug(f" {self.name}")
logger.debug(" year FCStack")
self.year_fc = FCStack(
first_layer_input_size=1,
num_layers=1,
default_output_size=1,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_norm=None,
default_norm_params=None,
default_activation=None,
default_dropout=dropout,
)
logger.debug(" month Embed")
self.embed_month = Embed(
[str(i) for i in range(12)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" day Embed")
self.embed_day = Embed(
[str(i) for i in range(31)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" weekday Embed")
self.embed_weekday = Embed(
[str(i) for i in range(7)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" yearday Embed")
self.embed_yearday = Embed(
[str(i) for i in range(366)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" hour Embed")
self.embed_hour = Embed(
[str(i) for i in range(24)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" minute Embed")
self.embed_minute = Embed(
[str(i) for i in range(60)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" second Embed")
self.embed_second = Embed(
[str(i) for i in range(60)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
# Summed sizes of all of the embeddings.
fc_layer_input_size = (
self.year_fc.output_shape[0] + self.embed_month.output_shape[0] +
self.embed_day.output_shape[0] +
self.embed_weekday.output_shape[0] +
self.embed_yearday.output_shape[0] +
self.embed_hour.output_shape[0] +
self.embed_minute.output_shape[0] +
self.embed_second.output_shape[0] +
1 # for periodic_second_of_day.
)
logger.debug(" FCStack")
self.fc_stack = FCStack(
first_layer_input_size=fc_layer_input_size,
layers=fc_layers,
num_layers=num_fc_layers,
default_output_size=output_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout=dropout,
)
def __init__(
self,
embedding_size=10,
embeddings_on_cpu=False,
fc_layers=None,
num_fc_layers=0,
fc_size=10,
use_bias=True,
weights_initializer='glorot_uniform',
bias_initializer='zeros',
weights_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
# weights_constraint=None,
# bias_constraint=None,
norm=None,
norm_params=None,
activation='relu',
dropout=0,
reduce_output='sum',
**kwargs):
"""
:param embedding_size: it is the maximum embedding size, the actual
size will be `min(vocaularyb_size, embedding_size)`
for `dense` representations and exacly `vocaularyb_size`
for the `sparse` encoding, where `vocabulary_size` is
the number of different strings appearing in the training set
in the column the feature is named after (plus 1 for `<UNK>`).
:type embedding_size: Integer
:param embeddings_on_cpu: by default embedings matrices are stored
on GPU memory if a GPU is used, as it allows
for faster access, but in some cases the embedding matrix
may be really big and this parameter forces the placement
of the embedding matrix in regular memroy and the CPU is used
to resolve them, slightly slowing down the process
as a result of data transfer between CPU and GPU memory.
:param dropout: determines if there should be a dropout layer before
returning the encoder output.
:type dropout: Boolean
:param initializer: the initializer to use. If `None`, the default
initialized of each variable is used (`glorot_uniform`
in most cases). Options are: `constant`, `identity`, `zeros`,
`ones`, `orthogonal`, `normal`, `uniform`,
`truncated_normal`, `variance_scaling`, `glorot_normal`,
`glorot_uniform`, `xavier_normal`, `xavier_uniform`,
`he_normal`, `he_uniform`, `lecun_normal`, `lecun_uniform`.
Alternatively it is possible to specify a dictionary with
a key `type` that identifies the type of initialzier and
other keys for its parameters, e.g.
`{type: normal, mean: 0, stddev: 0}`.
To know the parameters of each initializer, please refer to
TensorFlow's documentation.
:type initializer: str
:param regularize: if `True` the embedding wieghts are added to
the set of weights that get reularized by a regularization
loss (if the `regularization_lambda` in `training`
is greater than 0).
:type regularize: Boolean
"""
super(H3Embed, self).__init__()
logger.debug(' {}'.format(self.name))
self.embedding_size = embedding_size
self.reduce_output = reduce_output
self.reduce_sequence = SequenceReducer(reduce_mode=reduce_output)
logger.debug(' mode Embed')
self.embed_mode = Embed([str(i) for i in range(3)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
force_embedding_size=True,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' edge Embed')
self.embed_edge = Embed([str(i) for i in range(7)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
force_embedding_size=True,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' resolution Embed')
self.embed_resolution = Embed(
[str(i) for i in range(16)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
force_embedding_size=True,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' base cell Embed')
self.embed_base_cell = Embed([str(i) for i in range(122)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
force_embedding_size=True,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' cells Embed')
self.embed_cells = Embed([str(i) for i in range(8)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
force_embedding_size=True,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' FCStack')
self.fc_stack = FCStack(
layers=fc_layers,
num_layers=num_fc_layers,
default_fc_size=fc_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_weights_regularizer=weights_regularizer,
default_bias_regularizer=bias_regularizer,
default_activity_regularizer=activity_regularizer,
# default_weights_constraint=weights_constraint,
# default_bias_constraint=bias_constraint,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout=dropout,
)
def __init__(
self,
input_features=None,
embed_input_feature_name=None, # None or embedding size or "add"
num_layers=1,
hidden_size=256,
num_heads=8,
transformer_fc_size=256,
dropout=0.1,
fc_layers=None,
num_fc_layers=0,
fc_size=256,
use_bias=True,
weights_initializer='glorot_uniform',
bias_initializer='zeros',
weights_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
# weights_constraint=None,
# bias_constraint=None,
norm=None,
norm_params=None,
fc_activation='relu',
fc_dropout=0,
fc_residual=False,
reduce_output='concat',
**kwargs
):
super().__init__()
logger.debug(' {}'.format(self.name))
if reduce_output is None:
raise ValueError("TabTransformer requires the `resude_output` "
"parametr")
self.reduce_output = reduce_output
self.reduce_sequence = SequenceReducer(reduce_mode=reduce_output)
self.supports_masking = True
self.layer_norm = LayerNormalization()
self.embed_input_feature_name = embed_input_feature_name
if self.embed_input_feature_name:
vocab = [i_f for i_f in input_features
if i_f[TYPE] != NUMERICAL or i_f[TYPE] != BINARY]
if self.embed_input_feature_name == 'add':
self.embed_i_f_name_layer = Embed(vocab, hidden_size,
force_embedding_size=True)
projector_size = hidden_size
elif isinstance(self.embed_input_feature_name, int):
if self.embed_input_feature_name > hidden_size:
raise ValueError(
"TabTransformer parameter "
"`embed_input_feature_name` "
"specified integer value ({}) "
"needs to be smaller than "
"`hidden_size` ({}).".format(
self.embed_input_feature_name, hidden_size
))
self.embed_i_f_name_layer = Embed(
vocab,
self.embed_input_feature_name,
force_embedding_size=True,
)
projector_size = hidden_size - self.embed_input_feature_name
else:
raise ValueError("TabTransformer parameter "
"`embed_input_feature_name` "
"should be either None, an integer or `add`, "
"the current value is "
"{}".format(self.embed_input_feature_name))
else:
projector_size = hidden_size
logger.debug(' Projectors')
self.projectors = [Dense(projector_size) for i_f in input_features
if i_f[TYPE] != NUMERICAL and i_f[TYPE] != BINARY]
self.skip_features = [i_f[NAME] for i_f in input_features
if i_f[TYPE] == NUMERICAL or i_f[TYPE] == BINARY]
logger.debug(' TransformerStack')
self.transformer_stack = TransformerStack(
hidden_size=hidden_size,
num_heads=num_heads,
fc_size=transformer_fc_size,
num_layers=num_layers,
dropout=dropout
)
logger.debug(' FCStack')
self.fc_stack = FCStack(
layers=fc_layers,
num_layers=num_fc_layers,
default_fc_size=fc_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_weights_regularizer=weights_regularizer,
default_bias_regularizer=bias_regularizer,
default_activity_regularizer=activity_regularizer,
# default_weights_constraint=weights_constraint,
# default_bias_constraint=bias_constraint,
default_norm=norm,
default_norm_params=norm_params,
default_activation=fc_activation,
default_dropout=fc_dropout,
fc_residual=fc_residual,
)
def __init__(
self,
embedding_size: int = 10,
embeddings_on_cpu: bool = False,
fc_layers: Optional[List] = None,
num_fc_layers: int = 0,
output_size: int = 10,
use_bias: bool = True,
weights_initializer: str = "xavier_uniform",
bias_initializer: str = "zeros",
norm: str = None,
norm_params: Dict = None,
activation: str = "relu",
dropout: float = 0,
reduce_output: str = "sum",
**kwargs,
):
"""
:param embedding_size: it is the maximum embedding size, the actual
size will be `min(vocabulary_size, embedding_size)`
for `dense` representations and exactly `vocabulary_size`
for the `sparse` encoding, where `vocabulary_size` is
the number of different strings appearing in the training set
in the column the feature is named after (plus 1 for
`<UNK>`).
:type embedding_size: Integer
:param embeddings_on_cpu: by default embeddings matrices are stored
on GPU memory if a GPU is used, as it allows
for faster access, but in some cases the embedding matrix
may be really big and this parameter forces the placement
of the embedding matrix in regular memory and the CPU is used
to resolve them, slightly slowing down the process
as a result of data transfer between CPU and GPU memory.
:param dropout: determines if there should be a dropout layer before
returning the encoder output.
:type dropout: Boolean
"""
super().__init__()
logger.debug(f" {self.name}")
self.embedding_size = embedding_size
self.reduce_output = reduce_output
self.reduce_sequence = SequenceReducer(reduce_mode=reduce_output)
logger.debug(" mode Embed")
self.embed_mode = Embed(
[str(i) for i in range(3)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
force_embedding_size=True,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" edge Embed")
self.embed_edge = Embed(
[str(i) for i in range(7)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
force_embedding_size=True,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" resolution Embed")
self.embed_resolution = Embed(
[str(i) for i in range(16)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
force_embedding_size=True,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" base cell Embed")
self.embed_base_cell = Embed(
[str(i) for i in range(122)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
force_embedding_size=True,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" cells Embed")
self.embed_cells = EmbedSequence(
[str(i) for i in range(8)],
embedding_size,
max_sequence_length=(H3_INPUT_SIZE - 4),
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
force_embedding_size=True,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" FCStack")
self.fc_stack = FCStack(
first_layer_input_size=embedding_size,
layers=fc_layers,
num_layers=num_fc_layers,
default_output_size=output_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout=dropout,
)
