def __init__(
self,
input_features: Dict[str, "InputFeature"],
config: ComparatorCombinerConfig = None,
**kwargs,
):
super().__init__(input_features)
self.name = "ComparatorCombiner"
logger.debug(f"Entering {self.name}")
self.entity_1 = config.entity_1
self.entity_2 = config.entity_2
self.required_inputs = set(config.entity_1 + config.entity_2)
self.fc_size = config.fc_size
self.fc_stack = None
# todo future: this may be redundant, check
fc_layers = config.fc_layers
if fc_layers is None and config.num_fc_layers is not None:
fc_layers = []
for i in range(config.num_fc_layers):
fc_layers.append({"fc_size": config.fc_size})
if fc_layers is not None:
logger.debug("Setting up FCStack")
self.e1_fc_stack = FCStack(
self.get_entity_shape(config.entity_1)[-1],
layers=fc_layers,
num_layers=config.num_fc_layers,
default_fc_size=config.fc_size,
default_use_bias=config.use_bias,
default_weights_initializer=config.weights_initializer,
default_bias_initializer=config.bias_initializer,
default_norm=config.norm,
default_norm_params=config.norm_params,
default_activation=config.activation,
default_dropout=config.dropout,
)
self.e2_fc_stack = FCStack(
self.get_entity_shape(config.entity_2)[-1],
layers=fc_layers,
num_layers=config.num_fc_layers,
default_fc_size=config.fc_size,
default_use_bias=config.use_bias,
default_weights_initializer=config.weights_initializer,
default_bias_initializer=config.bias_initializer,
default_norm=config.norm,
default_norm_params=config.norm_params,
default_activation=config.activation,
default_dropout=config.dropout,
)
self.last_fc_layer_fc_size = fc_layers[-1]["fc_size"]
# todo: set initializer and regularization
self.register_buffer(
"bilinear_weights",
torch.randn([self.last_fc_layer_fc_size, self.last_fc_layer_fc_size], dtype=torch.float32),
)
def test_fc_stack(first_layer_input_size: Optional[int], layers: Optional[List], num_layers: Optional[int]):
if first_layer_input_size is None:
first_layer_input_size = layers[0]["input_size"]
fc_stack = FCStack(first_layer_input_size=first_layer_input_size, layers=layers, num_layers=num_layers).to(DEVICE)
input_tensor = torch.randn(BATCH_SIZE, first_layer_input_size, device=DEVICE)
output_tensor = fc_stack(input_tensor)
assert output_tensor.shape[1:] == fc_stack.output_shape
def __init__(
self,
vocab,
representation='dense',
embedding_size=50,
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=False,
fc_layers=None,
num_fc_layers=0,
fc_size=10,
use_bias=True,
weights_initializer='glorot_uniform',
bias_initializer='zeros',
weights_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
# weights_constraint=None,
# bias_constraint=None,
norm=None,
norm_params=None,
activation='relu',
dropout_rate=0.0,
reduce_output='sum',
**kwargs):
super(SetSparseEncoder, self).__init__()
logger.debug(' {}'.format(self.name))
logger.debug(' EmbedSparse')
self.embed_sparse = EmbedSparse(
vocab,
embedding_size,
representation=representation,
embeddings_trainable=embeddings_trainable,
pretrained_embeddings=pretrained_embeddings,
embeddings_on_cpu=embeddings_on_cpu,
dropout_rate=dropout_rate,
initializer=weights_initializer,
regularizer=weights_regularizer,
reduce_output=reduce_output,
)
logger.debug(' FCStack')
self.fc_stack = FCStack(
layers=fc_layers,
num_layers=num_fc_layers,
default_fc_size=fc_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_weights_regularizer=weights_regularizer,
default_bias_regularizer=bias_regularizer,
default_activity_regularizer=activity_regularizer,
# default_weights_constraint=weights_constraint,
# default_bias_constraint=bias_constraint,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout_rate=dropout_rate,
)
def __init__(self, input_features: Dict[str, "InputFeature"] = None, config: ConcatCombinerConfig = None, **kwargs):
super().__init__(input_features)
self.name = "ConcatCombiner"
logger.debug(f" {self.name}")
self.flatten_inputs = config.flatten_inputs
self.fc_stack = None
# todo future: this may be redundant, check
fc_layers = config.fc_layers
if fc_layers is None and config.num_fc_layers is not None:
fc_layers = []
for i in range(config.num_fc_layers):
fc_layers.append({"fc_size": config.fc_size})
self.fc_layers = fc_layers
if self.fc_layers is not None:
logger.debug(" FCStack")
self.fc_stack = FCStack(
first_layer_input_size=self.concatenated_shape[-1],
layers=config.fc_layers,
num_layers=config.num_fc_layers,
default_fc_size=config.fc_size,
default_use_bias=config.use_bias,
default_weights_initializer=config.weights_initializer,
default_bias_initializer=config.bias_initializer,
default_norm=config.norm,
default_norm_params=config.norm_params,
default_activation=config.activation,
default_dropout=config.dropout,
residual=config.residual,
)
if input_features and len(input_features) == 1 and self.fc_layers is None:
self.supports_masking = True
def __init__(
self,
input_size,
layers=None,
num_layers=1,
fc_size=256,
use_bias=True,
weights_initializer="xavier_uniform",
bias_initializer="zeros",
norm=None,
norm_params=None,
activation="relu",
dropout=0,
**kwargs,
):
super().__init__()
logger.debug(f" {self.name}")
self.input_size = input_size
logger.debug(" FCStack")
self.fc_stack = FCStack(
first_layer_input_size=input_size,
layers=layers,
num_layers=num_layers,
default_fc_size=fc_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout=dropout,
)
def __init__(self,
input_features: Dict[str, "InputFeature"] = None,
config: TransformerCombinerConfig = None,
**kwargs):
super().__init__(input_features)
self.name = "TransformerCombiner"
logger.debug(f" {self.name}")
self.reduce_output = config.reduce_output
self.reduce_sequence = SequenceReducer(
reduce_mode=config.reduce_output,
max_sequence_length=len(self.input_features),
encoding_size=config.hidden_size,
)
if self.reduce_output is None:
self.supports_masking = True
# sequence size for Transformer layer is number of input features
self.sequence_size = len(self.input_features)
logger.debug(" Projectors")
self.projectors = ModuleList(
# regardless of rank-2 or rank-3 input, torch.prod() calculates size
# after flattening the encoder output tensor
[
Linear(
torch.prod(
torch.Tensor([*input_features[inp].output_shape
])).type(torch.int32),
config.hidden_size) for inp in input_features
])
logger.debug(" TransformerStack")
self.transformer_stack = TransformerStack(
input_size=config.hidden_size,
sequence_size=self.sequence_size,
hidden_size=config.hidden_size,
num_heads=config.num_heads,
output_size=config.transformer_output_size,
num_layers=config.num_layers,
dropout=config.dropout,
)
if self.reduce_output is not None:
logger.debug(" FCStack")
self.fc_stack = FCStack(
self.transformer_stack.output_shape[-1],
layers=config.fc_layers,
num_layers=config.num_fc_layers,
default_output_size=config.output_size,
default_use_bias=config.use_bias,
default_weights_initializer=config.weights_initializer,
default_bias_initializer=config.bias_initializer,
default_norm=config.norm,
default_norm_params=config.norm_params,
default_activation=config.fc_activation,
default_dropout=config.fc_dropout,
fc_residual=config.fc_residual,
)
def __init__(
self,
input_features=None,
fc_layers=None,
num_fc_layers=None,
fc_size=256,
use_bias=True,
weights_initializer='glorot_uniform',
bias_initializer='zeros',
weights_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
# weights_constraint=None,
# bias_constraint=None,
norm=None,
norm_params=None,
activation='relu',
dropout=0,
flatten_inputs=False,
residual=False,
**kwargs
):
super().__init__()
logger.debug(' {}'.format(self.name))
self.flatten_inputs = flatten_inputs
self.fc_stack = None
# todo future: this may be redundant, check
if fc_layers is None and \
num_fc_layers is not None:
fc_layers = []
for i in range(num_fc_layers):
fc_layers.append({'fc_size': fc_size})
if fc_layers is not None:
logger.debug(' FCStack')
self.fc_stack = FCStack(
layers=fc_layers,
num_layers=num_fc_layers,
default_fc_size=fc_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_weights_regularizer=weights_regularizer,
default_bias_regularizer=bias_regularizer,
default_activity_regularizer=activity_regularizer,
# default_weights_constraint=weights_constraint,
# default_bias_constraint=bias_constraint,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout=dropout,
residual=residual,
)
if input_features and len(input_features) == 1 and fc_layers is None:
self.supports_masking = True
def __init__(self, feature, *args, **kwargs):
super().__init__(*args, feature=feature, **kwargs)
self.reduce_input = None
self.reduce_dependencies = None
self.dependencies = []
self.fc_layers = None
self.num_fc_layers = 0
self.fc_size = 256
self.use_bias = True
self.weights_initializer = 'glorot_uniform'
self.bias_initializer = 'zeros'
self.weights_regularizer = None
self.bias_regularizer = None
self.activity_regularizer = None
# self.weights_constraint=None
# self.bias_constraint=None
self.norm = None
self.norm_params = None
self.activation = 'relu'
self.dropout = 0
self.overwrite_defaults(feature)
logger.debug(' output feature fully connected layers')
logger.debug(' FCStack')
self.fc_stack = FCStack(
layers=self.fc_layers,
num_layers=self.num_fc_layers,
default_fc_size=self.fc_size,
default_use_bias=self.use_bias,
default_weights_initializer=self.weights_initializer,
default_bias_initializer=self.bias_initializer,
default_weights_regularizer=self.weights_regularizer,
default_bias_regularizer=self.bias_regularizer,
default_activity_regularizer=self.activity_regularizer,
# default_weights_constraint=self.weights_constraint,
# default_bias_constraint=self.bias_constraint,
default_norm=self.norm,
default_norm_params=self.norm_params,
default_activation=self.activation,
default_dropout=self.dropout,
)
# set up two sequence reducers, one for inputs and other for dependencies
self.reduce_sequence_input = SequenceReducer(
reduce_mode=self.reduce_input
)
if self.dependencies:
self.dependency_reducers = {}
for dependency in self.dependencies:
self.dependency_reducers[dependency] = SequenceReducer(
reduce_mode=self.reduce_dependencies
)
def __init__(
self,
vocab: List[str],
representation: str = "dense",
embedding_size: int = 50,
embeddings_trainable: bool = True,
pretrained_embeddings: Optional[str] = None,
embeddings_on_cpu: bool = False,
fc_layers=None,
num_fc_layers: int = 0,
output_size: int = 10,
use_bias: bool = True,
weights_initializer: str = "xavier_uniform",
bias_initializer: str = "zeros",
norm: Optional[str] = None,
norm_params: Optional[Dict[str, Any]] = None,
activation: str = "relu",
dropout: float = 0.0,
**kwargs,
):
super().__init__()
logger.debug(f" {self.name}")
self.vocab_size = len(vocab)
logger.debug(" Embed")
self.embed = EmbedSet(
vocab,
embedding_size,
representation=representation,
embeddings_trainable=embeddings_trainable,
pretrained_embeddings=pretrained_embeddings,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" FCStack")
# TODO(shreya): Make sure this is updated when FCStack is updated
self.fc_stack = FCStack(
first_layer_input_size=self.embed.output_shape[-1],
layers=fc_layers,
num_layers=num_fc_layers,
default_output_size=output_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout=dropout,
)
def __init__(self,
input_features: Dict[str, "InputFeature"] = None,
config: ProjectAggregateCombinerConfig = None,
**kwargs):
super().__init__(input_features)
self.name = "ProjectAggregateCombiner"
logger.debug(f" {self.name}")
logger.debug(" Projectors")
self.projectors = ModuleList(
# regardless of rank-2 or rank-3 input, torch.prod() calculates size
# after flattening the encoder output tensor
[
Linear(
torch.prod(
torch.Tensor([*input_features[inp].output_shape
])).type(torch.int32),
config.projection_size,
) for inp in input_features
])
self.fc_stack = None
# todo future: this may be redundant, check
fc_layers = config.fc_layers
if fc_layers is None and config.num_fc_layers is not None:
fc_layers = []
for i in range(config.num_fc_layers):
fc_layers.append({"output_size": config.output_size})
self.fc_layers = fc_layers
if self.fc_layers is not None:
logger.debug(" FCStack")
self.fc_stack = FCStack(
first_layer_input_size=config.projection_size,
layers=config.fc_layers,
num_layers=config.num_fc_layers,
default_output_size=config.output_size,
default_use_bias=config.use_bias,
default_weights_initializer=config.weights_initializer,
default_bias_initializer=config.bias_initializer,
default_norm=config.norm,
default_norm_params=config.norm_params,
default_activation=config.activation,
default_dropout=config.dropout,
residual=config.residual,
)
if input_features and len(
input_features) == 1 and self.fc_layers is None:
self.supports_masking = True
def test_fc_stack(
first_layer_input_size: Optional[int],
layers: Optional[List],
num_layers: Optional[int],
):
set_random_seed(RANDOM_SEED)
fc_stack = FCStack(first_layer_input_size=first_layer_input_size,
layers=layers,
num_layers=num_layers).to(DEVICE)
input_tensor = torch.randn(BATCH_SIZE,
first_layer_input_size,
device=DEVICE)
output_tensor = fc_stack(input_tensor)
assert output_tensor.shape[1:] == fc_stack.output_shape
def test_fc_stack_input_size_mismatch_fails():
first_layer_input_size = 10
layers = [{"input_size": 2, "output_size": 4}, {"output_size": 8}]
fc_stack = FCStack(
first_layer_input_size=first_layer_input_size,
layers=layers,
).to(DEVICE)
input_tensor = torch.randn(BATCH_SIZE,
first_layer_input_size,
device=DEVICE)
with pytest.raises(RuntimeError):
fc_stack(input_tensor)
def __init__(self, feature, *args, **kwargs):
super().__init__(*args, feature=feature, **kwargs)
self.loss = None
self.train_loss_function = None
self.eval_loss_function = None
self.metric_functions = OrderedDict()
self.reduce_input = None
self.reduce_dependencies = None
self.dependencies = []
self.fc_layers = None
self.num_fc_layers = 0
self.fc_size = 256
self.use_bias = True
self.weights_initializer = 'glorot_uniform'
self.bias_initializer = 'zeros'
self.weights_regularizer = None
self.bias_regularizer = None
self.activity_regularizer = None
# self.weights_constraint=None
# self.bias_constraint=None
self.norm = None
self.norm_params = None
self.activation = 'relu'
self.dropout = 0
self.overwrite_defaults(feature)
logger.debug(' output feature fully connected layers')
logger.debug(' FCStack')
self.fc_stack = FCStack(
layers=self.fc_layers,
num_layers=self.num_fc_layers,
default_fc_size=self.fc_size,
default_use_bias=self.use_bias,
default_weights_initializer=self.weights_initializer,
default_bias_initializer=self.bias_initializer,
default_weights_regularizer=self.weights_regularizer,
default_bias_regularizer=self.bias_regularizer,
default_activity_regularizer=self.activity_regularizer,
# default_weights_constraint=self.weights_constraint,
# default_bias_constraint=self.bias_constraint,
default_norm=self.norm,
default_norm_params=self.norm_params,
default_activation=self.activation,
default_dropout=self.dropout,
)
def __init__(self, feature, *args, **kwargs):
super().__init__(*args, feature=feature, **kwargs)
self.reduce_input = None
self.reduce_dependencies = None
self.dependencies = []
self.fc_layers = None
self.num_fc_layers = 0
self.fc_size = 256
self.use_bias = True
self.weights_initializer = "xavier_uniform"
self.bias_initializer = "zeros"
self.norm = None
self.norm_params = None
self.activation = "relu"
self.dropout = 0
self.input_size = None
self.overwrite_defaults(feature)
logger.debug(" output feature fully connected layers")
logger.debug(" FCStack")
self.fc_stack = FCStack(
first_layer_input_size=self.input_size,
layers=self.fc_layers,
num_layers=self.num_fc_layers,
default_fc_size=self.fc_size,
default_use_bias=self.use_bias,
default_weights_initializer=self.weights_initializer,
default_bias_initializer=self.bias_initializer,
default_norm=self.norm,
default_norm_params=self.norm_params,
default_activation=self.activation,
default_dropout=self.dropout,
)
# set up two sequence reducers, one for inputs and other for dependencies
self.reduce_sequence_input = SequenceReducer(
reduce_mode=self.reduce_input)
if self.dependencies:
self.dependency_reducers = torch.nn.ModuleDict()
# todo: re-evaluate need for separate handling of `attention` reducer
# currently this code does not support `attention`
for dependency in self.dependencies:
self.dependency_reducers[dependency] = SequenceReducer(
reduce_mode=self.reduce_dependencies)
def test_fc_stack_no_layers_behaves_like_passthrough():
first_layer_input_size = 10
layers = None
num_layers = 0
output_size = 15
fc_stack = FCStack(
first_layer_input_size=first_layer_input_size,
layers=layers,
num_layers=num_layers,
default_output_size=output_size,
).to(DEVICE)
input_tensor = torch.randn(BATCH_SIZE,
first_layer_input_size,
device=DEVICE)
output_tensor = fc_stack(input_tensor)
assert list(output_tensor.shape[1:]) == [first_layer_input_size]
assert output_tensor.shape[1:] == fc_stack.output_shape
assert np.all(np.isclose(input_tensor, output_tensor))
def __init__(
self,
layers=None,
num_layers=1,
fc_size=256,
use_bias=True,
weights_initializer='glorot_uniform',
bias_initializer='zeros',
weights_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
# weights_constraint=None,
# bias_constraint=None,
norm=None,
norm_params=None,
activation='relu',
dropout_rate=0,
**kwargs):
super(DenseEncoder, self).__init__()
logger.debug(' {}'.format(self.name))
logger.debug(' FCStack')
self.fc_stack = FCStack(
layers=layers,
num_layers=num_layers,
default_fc_size=fc_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_weights_regularizer=weights_regularizer,
default_bias_regularizer=bias_regularizer,
default_activity_regularizer=activity_regularizer,
# default_weights_constraint=weights_constraint,
# default_bias_constraint=bias_constraint,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout_rate=dropout_rate,
)
def __init__(
self,
fc_layers=None,
num_fc_layers=0,
fc_size=10,
use_bias=True,
weights_initializer='glorot_uniform',
bias_initializer='zeros',
weights_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
# weights_constraint=None,
# bias_constraint=None,
norm=None,
norm_params=None,
activation='relu',
dropout=0,
**kwargs):
"""
:param fc_layers: list of dictionaries containing the parameters of
all the fully connected layers
:type fc_layers: List
:param num_fc_layers: Number of stacked fully connected layers
:type num_fc_layers: Integer
:param fc_size: Size of each layer
:type fc_size: Integer
:param use_bias: bool determines where to use a bias vector
:type use_bias: bool
:param weights_initializer: Initializer for the weights (aka kernel)
matrix
:type weights_initializer: string
:param bias_initializer: Initializer for the bias vector
:type bias_initializer: string
:param weights_regularizer: regularizer applied to the weights
(kernal) matrix
:type weights_regularizer: string
:param bias_regularizer: reguralizer function applied to biase vector.
:type bias_regularizer: string
:param activity_regularizer: Regularizer applied to the output of the
layer (activation)
:type activity_regularizer: string
:param norm: type of normalization to use 'batch' or 'layer'
:type norm: string, default None
:param norm_params: parameters to pass to normalization function
:type norm_params: dictionary
:param activation: Activation function to use.
:type activation: string
:param dropout: determines if there should be a dropout layer before
returning the encoder output.
:type dropout: float
"""
super().__init__()
logger.debug(' {}'.format(self.name))
logger.debug(' year FCStack')
self.year_fc = FCStack(
num_layers=1,
default_fc_size=1,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_weights_regularizer=weights_regularizer,
default_bias_regularizer=bias_regularizer,
default_activity_regularizer=activity_regularizer,
# default_weights_constraint=weights_constraint,
# default_bias_constraint=bias_constraint,
default_norm=None,
default_norm_params=None,
default_activation=None,
default_dropout=dropout,
)
logger.debug(' FCStack')
self.fc_stack = FCStack(
layers=fc_layers,
num_layers=num_fc_layers,
default_fc_size=fc_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_weights_regularizer=weights_regularizer,
default_bias_regularizer=bias_regularizer,
default_activity_regularizer=activity_regularizer,
# default_weights_constraint=weights_constraint,
# default_bias_constraint=bias_constraint,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout=dropout,
)
def __init__(
self,
embedding_size: int = 10,
embeddings_on_cpu: bool = False,
fc_layers: Optional[List[Dict]] = None,
num_fc_layers: int = 0,
output_size: int = 10,
use_bias: bool = True,
weights_initializer: str = "xavier_uniform",
bias_initializer: str = "zeros",
norm: Optional[str] = None,
norm_params: Optional[Dict] = None,
activation: str = "relu",
dropout: float = 0,
**kwargs,
):
"""
:param embedding_size: The maximum embedding size, the actual size
will be `min(vocabulary_size, embedding_size)` for `dense`
representations and exactly `vocabulary_size` for the `sparse`
encoding, where `vocabulary_size` is the number of different
strings appearing in the training set in the column the feature
is named after (plus 1 for `<UNK>`).
:type embedding_size: Integer
:param embeddings_on_cpu: by default embeddings matrices are stored
on GPU memory if a GPU is used, as it allows for faster access,
but in some cases the embedding matrix may be really big and
this parameter forces the placement of the embedding matrix in
regular memory and the CPU is used to resolve them, slightly
slowing down the process as a result of data transfer between
CPU and GPU memory.
:param fc_layers: list of dictionaries containing the parameters of
all the fully connected layers.
:type fc_layers: List
:param num_fc_layers: Number of stacked fully connected layers.
:type num_fc_layers: Integer
:param output_size: Size of each layer.
:type output_size: Integer
:param use_bias: bool determines where to use a bias vector.
:type use_bias: bool
:param weights_initializer: Initializer for the weights (aka kernel)
matrix.
:type weights_initializer: string
:param bias_initializer: Initializer for the bias vector.
:type bias_initializer: string
:param norm: type of normalization to use 'batch' or 'layer'.
:type norm: string, default None
:param norm_params: parameters to pass to normalization function.
:type norm_params: dictionary
:param activation: Activation function to use.
:type activation: string
:param dropout: determines if there should be a dropout layer before
returning the encoder output.
:type dropout: float
"""
super().__init__()
logger.debug(f" {self.name}")
logger.debug(" year FCStack")
self.year_fc = FCStack(
first_layer_input_size=1,
num_layers=1,
default_output_size=1,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_norm=None,
default_norm_params=None,
default_activation=None,
default_dropout=dropout,
)
logger.debug(" month Embed")
self.embed_month = Embed(
[str(i) for i in range(12)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" day Embed")
self.embed_day = Embed(
[str(i) for i in range(31)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" weekday Embed")
self.embed_weekday = Embed(
[str(i) for i in range(7)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" yearday Embed")
self.embed_yearday = Embed(
[str(i) for i in range(366)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" hour Embed")
self.embed_hour = Embed(
[str(i) for i in range(24)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" minute Embed")
self.embed_minute = Embed(
[str(i) for i in range(60)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
logger.debug(" second Embed")
self.embed_second = Embed(
[str(i) for i in range(60)],
embedding_size,
representation="dense",
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
)
# Summed sizes of all of the embeddings.
fc_layer_input_size = (
self.year_fc.output_shape[0] + self.embed_month.output_shape[0] +
self.embed_day.output_shape[0] +
self.embed_weekday.output_shape[0] +
self.embed_yearday.output_shape[0] +
self.embed_hour.output_shape[0] +
self.embed_minute.output_shape[0] +
self.embed_second.output_shape[0] +
1 # for periodic_second_of_day.
)
logger.debug(" FCStack")
self.fc_stack = FCStack(
first_layer_input_size=fc_layer_input_size,
layers=fc_layers,
num_layers=num_fc_layers,
default_output_size=output_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout=dropout,
)
def __init__(
self,
fc_layers: Optional[List[FCStack]] = None,
num_fc_layers: int = 1,
output_size: int = 10,
use_bias: bool = True,
weights_initializer: str = "xavier_uniform",
bias_initializer: str = "zeros",
norm: Optional[str] = None,
norm_params: Optional[Dict] = None,
activation: str = "relu",
dropout: int = 0,
**kwargs,
):
"""
:param fc_layers: list of dictionaries containing the parameters of
all the fully connected layers.
:type fc_layers: List
:param num_fc_layers: Number of stacked fully connected layers.
:type num_fc_layers: Integer
:param output_size: Size of each layer.
:type output_size: Integer
:param use_bias: bool determines where to use a bias vector.
:type use_bias: bool
:param weights_initializer: Initializer for the weights (aka kernel)
matrix.
:type weights_initializer: string
:param bias_initializer: Initializer for the bias vector.
:type bias_initializer: string
:param norm: type of normalization to use 'batch' or 'layer'.
:type norm: string, default None
:param norm_params: parameters to pass to normalization function.
:type norm_params: dictionary
:param activation: Activation function to use.
:type activation: string
:param dropout: determines if there should be a dropout layer before
returning the encoder output.
:type dropout: float
"""
super().__init__()
logger.debug(f" {self.name}")
logger.debug(" year FCStack")
self.year_fc = FCStack(
first_layer_input_size=1,
num_layers=1,
default_output_size=1,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_norm=None,
default_norm_params=None,
default_activation=None,
default_dropout=dropout,
)
# Summed sizes of all of the embeddings.
# Additional 8 for periodic_[month, day, ..., second_of_day].
fc_layer_input_size = self.year_fc.output_shape[0] + 8
logger.debug(" FCStack")
self.fc_stack = FCStack(
first_layer_input_size=fc_layer_input_size,
layers=fc_layers,
num_layers=num_fc_layers,
default_output_size=output_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout=dropout,
)
def __init__(
self,
embedding_size=10,
embeddings_on_cpu=False,
fc_layers=None,
num_fc_layers=0,
fc_size=10,
use_bias=True,
weights_initializer='glorot_uniform',
bias_initializer='zeros',
weights_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
# weights_constraint=None,
# bias_constraint=None,
norm=None,
norm_params=None,
activation='relu',
dropout=0,
reduce_output='sum',
**kwargs):
"""
:param embedding_size: it is the maximum embedding size, the actual
size will be `min(vocaularyb_size, embedding_size)`
for `dense` representations and exacly `vocaularyb_size`
for the `sparse` encoding, where `vocabulary_size` is
the number of different strings appearing in the training set
in the column the feature is named after (plus 1 for `<UNK>`).
:type embedding_size: Integer
:param embeddings_on_cpu: by default embedings matrices are stored
on GPU memory if a GPU is used, as it allows
for faster access, but in some cases the embedding matrix
may be really big and this parameter forces the placement
of the embedding matrix in regular memroy and the CPU is used
to resolve them, slightly slowing down the process
as a result of data transfer between CPU and GPU memory.
:param dropout: determines if there should be a dropout layer before
returning the encoder output.
:type dropout: Boolean
:param initializer: the initializer to use. If `None`, the default
initialized of each variable is used (`glorot_uniform`
in most cases). Options are: `constant`, `identity`, `zeros`,
`ones`, `orthogonal`, `normal`, `uniform`,
`truncated_normal`, `variance_scaling`, `glorot_normal`,
`glorot_uniform`, `xavier_normal`, `xavier_uniform`,
`he_normal`, `he_uniform`, `lecun_normal`, `lecun_uniform`.
Alternatively it is possible to specify a dictionary with
a key `type` that identifies the type of initialzier and
other keys for its parameters, e.g.
`{type: normal, mean: 0, stddev: 0}`.
To know the parameters of each initializer, please refer to
TensorFlow's documentation.
:type initializer: str
:param regularize: if `True` the embedding wieghts are added to
the set of weights that get reularized by a regularization
loss (if the `regularization_lambda` in `training`
is greater than 0).
:type regularize: Boolean
"""
super(H3Embed, self).__init__()
logger.debug(' {}'.format(self.name))
self.embedding_size = embedding_size
self.reduce_output = reduce_output
self.reduce_sequence = SequenceReducer(reduce_mode=reduce_output)
logger.debug(' mode Embed')
self.embed_mode = Embed([str(i) for i in range(3)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
force_embedding_size=True,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' edge Embed')
self.embed_edge = Embed([str(i) for i in range(7)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
force_embedding_size=True,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' resolution Embed')
self.embed_resolution = Embed(
[str(i) for i in range(16)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
force_embedding_size=True,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' base cell Embed')
self.embed_base_cell = Embed([str(i) for i in range(122)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
force_embedding_size=True,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' cells Embed')
self.embed_cells = Embed([str(i) for i in range(8)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
force_embedding_size=True,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' FCStack')
self.fc_stack = FCStack(
layers=fc_layers,
num_layers=num_fc_layers,
default_fc_size=fc_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_weights_regularizer=weights_regularizer,
default_bias_regularizer=bias_regularizer,
default_activity_regularizer=activity_regularizer,
# default_weights_constraint=weights_constraint,
# default_bias_constraint=bias_constraint,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout=dropout,
)
def __init__(
self,
height: int,
width: int,
conv_layers: Optional[List[Dict]] = None,
num_conv_layers: Optional[int] = None,
num_channels: int = None,
out_channels: int = 32,
kernel_size: Union[int, Tuple[int]] = 3,
stride: Union[int, Tuple[int]] = 1,
padding: Union[int, Tuple[int], str] = "valid",
dilation: Union[int, Tuple[int]] = 1,
conv_use_bias: bool = True,
padding_mode: str = "zeros",
conv_norm: Optional[str] = None,
conv_norm_params: Optional[Dict[str, Any]] = None,
conv_activation: str = "relu",
conv_dropout: int = 0,
pool_function: str = "max",
pool_kernel_size: Union[int, Tuple[int]] = 2,
pool_stride: Union[int, Tuple[int]] = None,
pool_padding: Union[int, Tuple[int]] = 0,
pool_dilation: Union[int, Tuple[int]] = 1,
groups: int = 1,
fc_layers: Optional[List[Dict]] = None,
num_fc_layers: Optional[int] = 1,
output_size: int = 128,
fc_use_bias: bool = True,
fc_weights_initializer: str = "xavier_uniform",
fc_bias_initializer: str = "zeros",
fc_norm: Optional[str] = None,
fc_norm_params: Optional[Dict[str, Any]] = None,
fc_activation: str = "relu",
fc_dropout: float = 0,
**kwargs,
):
super().__init__()
logger.debug(f" {self.name}")
# map parameter input feature config names to internal names
img_height = height
img_width = width
first_in_channels = num_channels
self._input_shape = (first_in_channels, img_height, img_width)
if first_in_channels is None:
raise ValueError("first_in_channels must not be None.")
logger.debug(" Conv2DStack")
self.conv_stack_2d = Conv2DStack(
img_height=img_height,
img_width=img_width,
layers=conv_layers,
num_layers=num_conv_layers,
first_in_channels=first_in_channels,
default_out_channels=out_channels,
default_kernel_size=kernel_size,
default_stride=stride,
default_padding=padding,
default_dilation=dilation,
default_groups=groups,
default_use_bias=conv_use_bias,
default_padding_mode=padding_mode,
default_norm=conv_norm,
default_norm_params=conv_norm_params,
default_activation=conv_activation,
default_dropout=conv_dropout,
default_pool_function=pool_function,
default_pool_kernel_size=pool_kernel_size,
default_pool_stride=pool_stride,
default_pool_padding=pool_padding,
default_pool_dilation=pool_dilation,
)
out_channels, img_height, img_width = self.conv_stack_2d.output_shape
first_fc_layer_input_size = out_channels * img_height * img_width
self.flatten = torch.nn.Flatten()
logger.debug(" FCStack")
self.fc_stack = FCStack(
first_layer_input_size=first_fc_layer_input_size,
layers=fc_layers,
num_layers=num_fc_layers,
default_output_size=output_size,
default_use_bias=fc_use_bias,
default_weights_initializer=fc_weights_initializer,
default_bias_initializer=fc_bias_initializer,
default_norm=fc_norm,
default_norm_params=fc_norm_params,
default_activation=fc_activation,
default_dropout=fc_dropout,
)
def __init__(
self, input_features: Dict[str, "InputFeature"] = None, config: TabTransformerCombinerConfig = None, **kwargs
):
super().__init__(input_features)
self.name = "TabTransformerCombiner"
logger.debug(f"Initializing {self.name}")
if config.reduce_output is None:
raise ValueError("TabTransformer requires the `reduce_output` " "parameter")
self.reduce_output = config.reduce_output
self.reduce_sequence = SequenceReducer(reduce_mode=config.reduce_output)
self.supports_masking = True
self.embed_input_feature_name = config.embed_input_feature_name
if self.embed_input_feature_name:
vocab = [
i_f
for i_f in input_features
if input_features[i_f].type != NUMERICAL or input_features[i_f].type != BINARY
]
if self.embed_input_feature_name == "add":
self.embed_i_f_name_layer = Embed(vocab, config.hidden_size, force_embedding_size=True)
projector_size = config.hidden_size
elif isinstance(self.embed_input_feature_name, int):
if self.embed_input_feature_name > config.hidden_size:
raise ValueError(
"TabTransformer parameter "
"`embed_input_feature_name` "
"specified integer value ({}) "
"needs to be smaller than "
"`hidden_size` ({}).".format(self.embed_input_feature_name, config.hidden_size)
)
self.embed_i_f_name_layer = Embed(
vocab,
self.embed_input_feature_name,
force_embedding_size=True,
)
projector_size = config.hidden_size - self.embed_input_feature_name
else:
raise ValueError(
"TabTransformer parameter "
"`embed_input_feature_name` "
"should be either None, an integer or `add`, "
"the current value is "
"{}".format(self.embed_input_feature_name)
)
else:
projector_size = config.hidden_size
logger.debug(" Projectors")
self.unembeddable_features = []
self.embeddable_features = []
for i_f in input_features:
if input_features[i_f].type in {NUMERICAL, BINARY}:
self.unembeddable_features.append(input_features[i_f].name)
else:
self.embeddable_features.append(input_features[i_f].name)
self.projectors = ModuleList()
for i_f in self.embeddable_features:
flatten_size = self.get_flatten_size(input_features[i_f].output_shape)
self.projectors.append(Linear(flatten_size[0], projector_size))
# input to layer_norm are the encoder outputs for unembeddable features,
# which are numerical or binary features. These should be 2-dim
# tensors. Size should be concatenation of these tensors.
concatenated_unembeddable_encoders_size = 0
for i_f in self.unembeddable_features:
concatenated_unembeddable_encoders_size += input_features[i_f].output_shape[0]
self.layer_norm = torch.nn.LayerNorm(concatenated_unembeddable_encoders_size)
logger.debug(" TransformerStack")
self.transformer_stack = TransformerStack(
input_size=config.hidden_size,
sequence_size=len(self.embeddable_features),
hidden_size=config.hidden_size,
# todo: can we just use projector_size? # hidden_size,
num_heads=config.num_heads,
fc_size=config.transformer_fc_size,
num_layers=config.num_layers,
dropout=config.dropout,
)
logger.debug(" FCStack")
transformer_hidden_size = self.transformer_stack.layers[-1].output_shape[-1]
# determine input size to fully connected layer based on reducer
if config.reduce_output == "concat":
num_embeddable_features = len(self.embeddable_features)
fc_input_size = num_embeddable_features * transformer_hidden_size
else:
fc_input_size = transformer_hidden_size if len(self.embeddable_features) > 0 else 0
self.fc_stack = FCStack(
fc_input_size + concatenated_unembeddable_encoders_size,
layers=config.fc_layers,
num_layers=config.num_fc_layers,
default_fc_size=config.fc_size,
default_use_bias=config.use_bias,
default_weights_initializer=config.weights_initializer,
default_bias_initializer=config.bias_initializer,
default_norm=config.norm,
default_norm_params=config.norm_params,
default_activation=config.fc_activation,
default_dropout=config.fc_dropout,
fc_residual=config.fc_residual,
)
# Create empty tensor of shape [1, 0] to use as hidden in case there are no category or numeric/binary features.
self.register_buffer("empty_hidden", torch.empty([1, 0]))
self.register_buffer("embeddable_features_indices", torch.arange(0, len(self.embeddable_features)))
def __init__(
self,
entity_1: List[str],
entity_2: List[str],
# fc_layers=None,
num_fc_layers=1,
fc_size=256,
use_bias=True,
weights_initializer="glorot_uniform",
bias_initializer="zeros",
weights_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
# weights_constraint=None,
# bias_constraint=None,
norm=None,
norm_params=None,
activation="relu",
dropout=0,
**kwargs,
):
super().__init__()
logger.debug(" {}".format(self.name))
self.fc_stack = None
# todo future: this may be redundant, check
# if fc_layers is None and num_fc_layers is not None:
fc_layers = []
for i in range(num_fc_layers):
fc_layers.append({"fc_size": fc_size})
if fc_layers is not None:
logger.debug(" FCStack")
self.e1_fc_stack = FCStack(
layers=fc_layers,
num_layers=num_fc_layers,
default_fc_size=fc_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_weights_regularizer=weights_regularizer,
default_bias_regularizer=bias_regularizer,
default_activity_regularizer=activity_regularizer,
# default_weights_constraint=weights_constraint,
# default_bias_constraint=bias_constraint,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout=dropout,
)
self.e2_fc_stack = FCStack(
layers=fc_layers,
num_layers=num_fc_layers,
default_fc_size=fc_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_weights_regularizer=weights_regularizer,
default_bias_regularizer=bias_regularizer,
default_activity_regularizer=activity_regularizer,
# default_weights_constraint=weights_constraint,
# default_bias_constraint=bias_constraint,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout=dropout,
)
# todo: this should actually be the size of the last fc layer,
# not just fc_size
# todo: set initializer and regularization
self.bilinear_weights = tf.random.normal([fc_size, fc_size],
dtype=tf.float32)
self.entity_1 = entity_1
self.entity_2 = entity_2
self.required_inputs = set(entity_1 + entity_2)
self.fc_size = fc_size
def __init__(
self,
input_features=None,
embed_input_feature_name=None, # None or embedding size or "add"
num_layers=1,
hidden_size=256,
num_heads=8,
transformer_fc_size=256,
dropout=0.1,
fc_layers=None,
num_fc_layers=0,
fc_size=256,
use_bias=True,
weights_initializer='glorot_uniform',
bias_initializer='zeros',
weights_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
# weights_constraint=None,
# bias_constraint=None,
norm=None,
norm_params=None,
fc_activation='relu',
fc_dropout=0,
fc_residual=False,
reduce_output='concat',
**kwargs
):
super().__init__()
logger.debug(' {}'.format(self.name))
if reduce_output is None:
raise ValueError("TabTransformer requires the `resude_output` "
"parametr")
self.reduce_output = reduce_output
self.reduce_sequence = SequenceReducer(reduce_mode=reduce_output)
self.supports_masking = True
self.layer_norm = LayerNormalization()
self.embed_input_feature_name = embed_input_feature_name
if self.embed_input_feature_name:
vocab = [i_f for i_f in input_features
if i_f[TYPE] != NUMERICAL or i_f[TYPE] != BINARY]
if self.embed_input_feature_name == 'add':
self.embed_i_f_name_layer = Embed(vocab, hidden_size,
force_embedding_size=True)
projector_size = hidden_size
elif isinstance(self.embed_input_feature_name, int):
if self.embed_input_feature_name > hidden_size:
raise ValueError(
"TabTransformer parameter "
"`embed_input_feature_name` "
"specified integer value ({}) "
"needs to be smaller than "
"`hidden_size` ({}).".format(
self.embed_input_feature_name, hidden_size
))
self.embed_i_f_name_layer = Embed(
vocab,
self.embed_input_feature_name,
force_embedding_size=True,
)
projector_size = hidden_size - self.embed_input_feature_name
else:
raise ValueError("TabTransformer parameter "
"`embed_input_feature_name` "
"should be either None, an integer or `add`, "
"the current value is "
"{}".format(self.embed_input_feature_name))
else:
projector_size = hidden_size
logger.debug(' Projectors')
self.projectors = [Dense(projector_size) for i_f in input_features
if i_f[TYPE] != NUMERICAL and i_f[TYPE] != BINARY]
self.skip_features = [i_f[NAME] for i_f in input_features
if i_f[TYPE] == NUMERICAL or i_f[TYPE] == BINARY]
logger.debug(' TransformerStack')
self.transformer_stack = TransformerStack(
hidden_size=hidden_size,
num_heads=num_heads,
fc_size=transformer_fc_size,
num_layers=num_layers,
dropout=dropout
)
logger.debug(' FCStack')
self.fc_stack = FCStack(
layers=fc_layers,
num_layers=num_fc_layers,
default_fc_size=fc_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_weights_regularizer=weights_regularizer,
default_bias_regularizer=bias_regularizer,
default_activity_regularizer=activity_regularizer,
# default_weights_constraint=weights_constraint,
# default_bias_constraint=bias_constraint,
default_norm=norm,
default_norm_params=norm_params,
default_activation=fc_activation,
default_dropout=fc_dropout,
fc_residual=fc_residual,
)
def __init__(self, feature: Dict[str, Any],
other_output_features: Dict[str, "OutputFeature"], *args,
**kwargs):
"""Defines defaults, overwrites them based on the feature dictionary, and sets up dependencies.
Any output feature can depend on one or more other output features. The `other_output_features` input dictionary
should contain entries for any dependent output features, which is accomplished by constructing output features
in topographically sorted order. Attributes of any dependent output features are used to properly initialize
this feature's sizes.
"""
super().__init__(*args, feature=feature, **kwargs)
self.reduce_input = None
self.reduce_dependencies = None
# List of feature names that this output feature is dependent on.
self.dependencies = []
self.fc_layers = None
self.num_fc_layers = 0
self.output_size = 256
self.use_bias = True
self.weights_initializer = "xavier_uniform"
self.bias_initializer = "zeros"
self.norm = None
self.norm_params = None
self.activation = "relu"
self.dropout = 0
self.input_size = None
self.overwrite_defaults(feature)
logger.debug(" output feature fully connected layers")
logger.debug(" FCStack")
self.input_size = get_input_size_with_dependencies(
self.input_size, self.dependencies, other_output_features)
feature["input_size"] = self.input_size # needed for future overrides
self.fc_stack = FCStack(
first_layer_input_size=self.input_size,
layers=self.fc_layers,
num_layers=self.num_fc_layers,
default_output_size=self.output_size,
default_use_bias=self.use_bias,
default_weights_initializer=self.weights_initializer,
default_bias_initializer=self.bias_initializer,
default_norm=self.norm,
default_norm_params=self.norm_params,
default_activation=self.activation,
default_dropout=self.dropout,
)
self._calibration_module = self.create_calibration_module(feature)
self._prediction_module = self.create_predict_module()
# set up two sequence reducers, one for inputs and other for dependencies
self.reduce_sequence_input = SequenceReducer(
reduce_mode=self.reduce_input)
if self.dependencies:
self.dependency_reducers = torch.nn.ModuleDict()
# todo: re-evaluate need for separate handling of `attention` reducer
# currently this code does not support `attention`
for dependency in self.dependencies:
self.dependency_reducers[dependency] = SequenceReducer(
reduce_mode=self.reduce_dependencies)
def __init__(
self,
embedding_size=10,
embeddings_on_cpu=False,
fc_layers=None,
num_fc_layers=0,
fc_size=10,
use_bias=True,
weights_initializer='glorot_uniform',
bias_initializer='zeros',
weights_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
# weights_constraint=None,
# bias_constraint=None,
norm=None,
norm_params=None,
activation='relu',
dropout=0,
**kwargs):
"""
:param embedding_size: it is the maximum embedding size, the actual
size will be `min(vocaularyb_size, embedding_size)`
for `dense` representations and exacly `vocaularyb_size`
for the `sparse` encoding, where `vocabulary_size` is
the number of different strings appearing in the training set
in the column the feature is named after (plus 1 for `<UNK>`).
:type embedding_size: Integer
:param embeddings_on_cpu: by default embedings matrices are stored
on GPU memory if a GPU is used, as it allows
for faster access, but in some cases the embedding matrix
may be really big and this parameter forces the placement
of the embedding matrix in regular memroy and the CPU is used
to resolve them, slightly slowing down the process
as a result of data transfer between CPU and GPU memory.
:param fc_layers: list of dictionaries containing the parameters of
all the fully connected layers
:type fc_layers: List
:param num_fc_layers: Number of stacked fully connected layers
:type num_fc_layers: Integer
:param fc_size: Size of each layer
:type fc_size: Integer
:param use_bias: bool determines where to use a bias vector
:type use_bias: bool
:param weights_initializer: Initializer for the weights (aka kernel)
matrix
:type weights_initializer: string
:param bias_initializer: Initializer for the bias vector
:type bias_initializer: string
:param weights_regularizer: regularizer applied to the weights
(kernal) matrix
:type weights_regularizer: string
:param bias_regularizer: reguralizer function applied to biase vector.
:type bias_regularizer: string
:param activity_regularizer: Regularizer applied to the output of the
layer (activation)
:type activity_regularizer: string
:param norm: type of normalization to use 'batch' or 'layer'
:type norm: string, default None
:param norm_params: parameters to pass to normalization function
:type norm_params: dictionary
:param activation: Activation function to use.
:type activation: string
:param dropout: determines if there should be a dropout layer before
returning the encoder output.
:type dropout: float
"""
super().__init__()
logger.debug(' {}'.format(self.name))
logger.debug(' year FCStack')
self.year_fc = FCStack(
num_layers=1,
default_fc_size=1,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_weights_regularizer=weights_regularizer,
default_bias_regularizer=bias_regularizer,
default_activity_regularizer=activity_regularizer,
# default_weights_constraint=weights_constraint,
# default_bias_constraint=bias_constraint,
default_norm=None,
default_norm_params=None,
default_activation=None,
default_dropout=dropout,
)
logger.debug(' month Embed')
self.embed_month = Embed([str(i) for i in range(12)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' day Embed')
self.embed_day = Embed([str(i) for i in range(31)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' weekday Embed')
self.embed_weekday = Embed([str(i) for i in range(7)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' yearday Embed')
self.embed_yearday = Embed([str(i) for i in range(366)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' hour Embed')
self.embed_hour = Embed([str(i) for i in range(24)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' minute Embed')
self.embed_minute = Embed([str(i) for i in range(60)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' second Embed')
self.embed_second = Embed([str(i) for i in range(60)],
embedding_size,
representation='dense',
embeddings_trainable=True,
pretrained_embeddings=None,
embeddings_on_cpu=embeddings_on_cpu,
dropout=dropout,
embedding_initializer=weights_initializer,
embedding_regularizer=weights_regularizer)
logger.debug(' FCStack')
self.fc_stack = FCStack(
layers=fc_layers,
num_layers=num_fc_layers,
default_fc_size=fc_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_weights_regularizer=weights_regularizer,
default_bias_regularizer=bias_regularizer,
default_activity_regularizer=activity_regularizer,
# default_weights_constraint=weights_constraint,
# default_bias_constraint=bias_constraint,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout=dropout,
)
def __init__(
self,
resnet_size=50,
num_filters=16,
kernel_size=3,
conv_stride=1,
first_pool_size=None,
first_pool_stride=None,
batch_norm_momentum=0.9,
batch_norm_epsilon=0.001,
fc_layers=None,
num_fc_layers=1,
fc_size=256,
use_bias=True,
weights_initializer='glorot_uniform',
bias_initializer='zeros',
weights_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
# weights_constraint=None,
# bias_constraint=None,
norm=None,
norm_params=None,
activation='relu',
dropout=0,
**kwargs):
super(ResNetEncoder, self).__init__()
logger.debug(' {}'.format(self.name))
if resnet_size < 50:
bottleneck = False
else:
bottleneck = True
block_sizes = get_resnet_block_sizes(resnet_size)
block_strides = [1, 2, 2, 2][:len(block_sizes)]
logger.debug(' ResNet2')
self.resnet = ResNet2(resnet_size, bottleneck, num_filters,
kernel_size, conv_stride, first_pool_size,
first_pool_stride, block_sizes, block_strides,
batch_norm_momentum, batch_norm_epsilon)
self.flatten = Flatten()
logger.debug(' FCStack')
self.fc_stack = FCStack(
layers=fc_layers,
num_layers=num_fc_layers,
default_fc_size=fc_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_weights_regularizer=weights_regularizer,
default_bias_regularizer=bias_regularizer,
default_activity_regularizer=activity_regularizer,
# default_weights_constraint=fc_weights_constraint,
# default_bias_constraint=fc_bias_constraint,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout=dropout,
)
def __init__(
self,
conv_layers=None,
num_conv_layers=None,
filter_size=3,
num_filters=32,
strides=(1, 1),
padding='valid',
dilation_rate=(1, 1),
conv_use_bias=True,
conv_weights_initializer='glorot_uniform',
conv_bias_initializer='zeros',
conv_weights_regularizer=None,
conv_bias_regularizer=None,
conv_activity_regularizer=None,
# conv_weights_constraint=None,
# conv_bias_constraint=None,
conv_norm=None,
conv_norm_params=None,
conv_activation='relu',
conv_dropout=0,
pool_size=(2, 2),
pool_strides=None,
fc_layers=None,
num_fc_layers=1,
fc_size=128,
fc_use_bias=True,
fc_weights_initializer='glorot_uniform',
fc_bias_initializer='zeros',
fc_weights_regularizer=None,
fc_bias_regularizer=None,
fc_activity_regularizer=None,
# fc_weights_constraint=None,
# fc_bias_constraint=None,
fc_norm=None,
fc_norm_params=None,
fc_activation='relu',
fc_dropout=0,
**kwargs):
super(Stacked2DCNN, self).__init__()
logger.debug(' {}'.format(self.name))
logger.debug(' Conv2DStack')
self.conv_stack_2d = Conv2DStack(
layers=conv_layers,
num_layers=num_conv_layers,
default_num_filters=num_filters,
default_filter_size=filter_size,
default_strides=strides,
default_padding=padding,
default_dilation_rate=dilation_rate,
default_use_bias=conv_use_bias,
default_weights_initializer=conv_weights_initializer,
defaultbias_initializer=conv_bias_initializer,
default_weights_regularizer=conv_weights_regularizer,
default_bias_regularizer=conv_bias_regularizer,
default_activity_regularizer=conv_activity_regularizer,
# default_weights_constraint=conv_weights_constraint,
# default_bias_constraint=conv_bias_constraint,
default_norm=conv_norm,
default_norm_params=conv_norm_params,
default_activation=conv_activation,
default_dropout=conv_dropout,
default_pool_size=pool_size,
default_pool_strides=pool_strides,
)
logger.debug(' FCStacl')
self.fc_stack = FCStack(
layers=fc_layers,
num_layers=num_fc_layers,
default_fc_size=fc_size,
default_use_bias=fc_use_bias,
default_weights_initializer=fc_weights_initializer,
default_bias_initializer=fc_bias_initializer,
default_weights_regularizer=fc_weights_regularizer,
default_bias_regularizer=fc_bias_regularizer,
default_activity_regularizer=fc_activity_regularizer,
# default_weights_constraint=fc_weights_constraint,
# default_bias_constraint=fc_bias_constraint,
default_norm=fc_norm,
default_norm_params=fc_norm_params,
default_activation=fc_activation,
default_dropout=fc_dropout,
)
def __init__(
self,
input_features=None,
num_layers=1,
hidden_size=256,
num_heads=8,
transformer_fc_size=256,
dropout=0.1,
fc_layers=None,
num_fc_layers=0,
fc_size=256,
use_bias=True,
weights_initializer='glorot_uniform',
bias_initializer='zeros',
weights_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
# weights_constraint=None,
# bias_constraint=None,
norm=None,
norm_params=None,
fc_activation='relu',
fc_dropout=0,
fc_residual=False,
reduce_output='mean',
**kwargs
):
super().__init__()
logger.debug(' {}'.format(self.name))
self.reduce_output = reduce_output
self.reduce_sequence = SequenceReducer(reduce_mode=reduce_output)
if self.reduce_output is None:
self.supports_masking = True
logger.debug(' Projectors')
self.projectors = [Dense(hidden_size) for _ in input_features]
logger.debug(' TransformerStack')
self.transformer_stack = TransformerStack(
hidden_size=hidden_size,
num_heads=num_heads,
fc_size=transformer_fc_size,
num_layers=num_layers,
dropout=dropout
)
if self.reduce_output is not None:
logger.debug(' FCStack')
self.fc_stack = FCStack(
layers=fc_layers,
num_layers=num_fc_layers,
default_fc_size=fc_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_weights_regularizer=weights_regularizer,
default_bias_regularizer=bias_regularizer,
default_activity_regularizer=activity_regularizer,
# default_weights_constraint=weights_constraint,
# default_bias_constraint=bias_constraint,
default_norm=norm,
default_norm_params=norm_params,
default_activation=fc_activation,
default_dropout=fc_dropout,
fc_residual=fc_residual,
)
def __init__(
self,
height: int,
width: int,
resnet_size: int = 50,
num_channels: int = 3,
out_channels: int = 16,
kernel_size: Union[int, Tuple[int]] = 3,
conv_stride: Union[int, Tuple[int]] = 1,
first_pool_kernel_size: Union[int, Tuple[int]] = None,
first_pool_stride: Union[int, Tuple[int]] = None,
batch_norm_momentum: float = 0.1,
batch_norm_epsilon: float = 0.001,
fc_layers: Optional[List[Dict]] = None,
num_fc_layers: Optional[int] = 1,
output_size: int = 256,
use_bias: bool = True,
weights_initializer: str = "xavier_uniform",
bias_initializer: str = "zeros",
norm: Optional[str] = None,
norm_params: Optional[Dict[str, Any]] = None,
activation: str = "relu",
dropout: float = 0,
**kwargs,
):
super().__init__()
logger.debug(f" {self.name}")
# map parameter input feature config names to internal names
img_height = height
img_width = width
first_in_channels = num_channels
self._input_shape = (first_in_channels, img_height, img_width)
logger.debug(" ResNet")
self.resnet = ResNet(
img_height=img_height,
img_width=img_width,
first_in_channels=first_in_channels,
out_channels=out_channels,
resnet_size=resnet_size,
kernel_size=kernel_size,
conv_stride=conv_stride,
first_pool_kernel_size=first_pool_kernel_size,
first_pool_stride=first_pool_stride,
batch_norm_momentum=batch_norm_momentum,
batch_norm_epsilon=batch_norm_epsilon,
)
first_fc_layer_input_size = self.resnet.output_shape[0]
logger.debug(" FCStack")
self.fc_stack = FCStack(
first_layer_input_size=first_fc_layer_input_size,
layers=fc_layers,
num_layers=num_fc_layers,
default_output_size=output_size,
default_use_bias=use_bias,
default_weights_initializer=weights_initializer,
default_bias_initializer=bias_initializer,
default_norm=norm,
default_norm_params=norm_params,
default_activation=activation,
default_dropout=dropout,
)
