def build(self, hp, inputs=None):
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
output_node = input_node
# Reduce the tensor to a vector.
if len(output_node.shape) > 2:
output_node = reduction.SpatialReduction().build(hp, output_node)
if self.dropout is not None:
dropout = self.dropout
else:
dropout = hp.Choice("dropout", [0.0, 0.25, 0.5], default=0)
if dropout > 0:
output_node = layers.Dropout(dropout)(output_node)
output_node = layers.Dense(self.shape[-1])(output_node)
if isinstance(self.loss, keras.losses.BinaryCrossentropy):
output_node = layers.Activation(activations.sigmoid, name=self.name)(
output_node
)
else:
output_node = layers.Softmax(name=self.name)(output_node)
return output_node
def build(self, hp, inputs=None):
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
if len(input_node.shape) > 2:
return layers.Flatten()(input_node)
return input_node
def build(self, hp, inputs=None):
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
output_node = input_node
output_node = reduction.Flatten().build(hp, output_node)
use_batchnorm = self.use_batchnorm
if use_batchnorm is None:
use_batchnorm = hp.Boolean("use_batchnorm", default=False)
if self.dropout is not None:
dropout = self.dropout
else:
dropout = hp.Choice("dropout", [0.0, 0.25, 0.5], default=0)
for i in range(utils.add_to_hp(self.num_layers, hp)):
units = utils.add_to_hp(self.num_units, hp,
"units_{i}".format(i=i))
output_node = layers.Dense(units)(output_node)
if use_batchnorm:
output_node = layers.BatchNormalization()(output_node)
output_node = layers.ReLU()(output_node)
if dropout > 0:
output_node = layers.Dropout(dropout)(output_node)
return output_node
def build(self, hp, inputs=None):
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
shape = input_node.shape.as_list()
if len(shape) != 3:
raise ValueError('Expect the input tensor to have '
'at least 3 dimensions for rnn models, '
'but got {shape}'.format(shape=input_node.shape))
feature_size = shape[-1]
output_node = input_node
bidirectional = self.bidirectional
if bidirectional is None:
bidirectional = hp.Boolean('bidirectional', default=True)
layer_type = self.layer_type or hp.Choice(
'layer_type', ['gru', 'lstm'], default='lstm')
num_layers = self.num_layers or hp.Choice('num_layers', [1, 2, 3],
default=2)
rnn_layers = {'gru': layers.GRU, 'lstm': layers.LSTM}
in_layer = rnn_layers[layer_type]
for i in range(num_layers):
return_sequences = True
if i == num_layers - 1:
return_sequences = self.return_sequences
if bidirectional:
output_node = layers.Bidirectional(
in_layer(feature_size,
return_sequences=return_sequences))(output_node)
else:
output_node = in_layer(
feature_size,
return_sequences=return_sequences)(output_node)
return output_node
def build(self, hp, inputs=None):
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
output_node = input_node
output_node = reduction.Flatten().build(hp, output_node)
num_layers = self.num_layers or hp.Choice("num_layers", [1, 2, 3],
default=2)
use_batchnorm = self.use_batchnorm
if use_batchnorm is None:
use_batchnorm = hp.Boolean("use_batchnorm", default=False)
if self.dropout is not None:
dropout = self.dropout
else:
dropout = hp.Choice("dropout", [0.0, 0.25, 0.5], default=0)
for i in range(num_layers):
units = hp.Choice(
"units_{i}".format(i=i),
[16, 32, 64, 128, 256, 512, 1024],
default=32,
)
output_node = layers.Dense(units)(output_node)
if use_batchnorm:
output_node = layers.BatchNormalization()(output_node)
output_node = layers.ReLU()(output_node)
if dropout > 0:
output_node = layers.Dropout(dropout)(output_node)
return output_node
def build(self, hp, inputs=None):
if self.num_classes:
expected = self.num_classes if self.num_classes > 2 else 1
if self.output_shape[-1] != expected:
raise ValueError('The data doesn\'t match the expected shape. '
'Expecting {} but got {}'.format(
expected, self.output_shape[-1]))
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
output_node = input_node
# Reduce the tensor to a vector.
if len(output_node.shape) > 2:
output_node = reduction.SpatialReduction().build(hp, output_node)
if self.dropout_rate is not None:
dropout_rate = self.dropout_rate
else:
dropout_rate = hp.Choice('dropout_rate', [0.0, 0.25, 0.5],
default=0)
if dropout_rate > 0:
output_node = layers.Dropout(dropout_rate)(output_node)
output_node = layers.Dense(self.output_shape[-1])(output_node)
if self.loss == 'binary_crossentropy':
output_node = layers.Activation(activations.sigmoid,
name=self.name)(output_node)
else:
output_node = layers.Softmax(name=self.name)(output_node)
return output_node
def build(self, hp, inputs=None):
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
shape = input_node.shape.as_list()
if len(shape) != 3:
raise ValueError(
"Expect the input tensor of RNNBlock to have dimensions of "
"[batch_size, time_steps, vec_len], "
"but got {shape}".format(shape=input_node.shape)
)
feature_size = shape[-1]
output_node = input_node
bidirectional = utils.add_to_hp(self.bidirectional, hp)
layer_type = utils.add_to_hp(self.layer_type, hp)
num_layers = utils.add_to_hp(self.num_layers, hp)
rnn_layers = {"gru": layers.GRU, "lstm": layers.LSTM}
in_layer = rnn_layers[layer_type]
for i in range(num_layers):
return_sequences = True
if i == num_layers - 1:
return_sequences = self.return_sequences
if bidirectional:
output_node = layers.Bidirectional(
in_layer(feature_size, return_sequences=return_sequences)
)(output_node)
else:
output_node = in_layer(
feature_size, return_sequences=return_sequences
)(output_node)
return output_node
def build(self, hp, inputs=None):
"""
# Arguments
hp: HyperParameters. The hyperparameters for building the model.
inputs: Tensor of Shape [batch_size, seq_len]
# Returns
Output Tensor of shape `[batch_size, seq_len, embedding_dim]`.
"""
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
pretraining = self.pretraining or hp.Choice(
'pretraining', ['random', 'glove', 'fasttext', 'word2vec', 'none'],
default='none')
embedding_dim = self.embedding_dim or hp.Choice(
'embedding_dim', [32, 64, 128, 256, 512], default=128)
num_heads = self.num_heads or hp.Choice('num_heads', [8, 16, 32],
default=8)
dense_dim = self.dense_dim or hp.Choice(
'dense_dim', [128, 256, 512, 1024, 2048], default=2048)
dropout_rate = self.dropout_rate or hp.Choice(
'dropout_rate', [0.0, 0.25, 0.5], default=0)
ffn = tf.keras.Sequential([
layers.Dense(dense_dim, activation="relu"),
layers.Dense(embedding_dim),
])
layernorm1 = layers.LayerNormalization(epsilon=1e-6)
layernorm2 = layers.LayerNormalization(epsilon=1e-6)
dropout1 = layers.Dropout(dropout_rate)
dropout2 = layers.Dropout(dropout_rate)
# Token and Position Embeddings
input_node = nest.flatten(inputs)[0]
token_embedding = Embedding(max_features=self.max_features,
pretraining=pretraining,
embedding_dim=embedding_dim,
dropout_rate=dropout_rate).build(
hp, input_node)
maxlen = input_node.shape[-1]
batch_size = tf.shape(input_node)[0]
positions = self.pos_array_funct(maxlen, batch_size)
position_embedding = Embedding(max_features=maxlen,
pretraining=pretraining,
embedding_dim=embedding_dim,
dropout_rate=dropout_rate).build(
hp, positions)
output_node = tf.keras.layers.Add()(
[token_embedding, position_embedding])
attn_output = MultiHeadSelfAttention(embedding_dim,
num_heads).build(hp, output_node)
attn_output = dropout1(attn_output)
add_inputs_1 = tf.keras.layers.Add()([output_node, attn_output])
out1 = layernorm1(add_inputs_1)
ffn_output = ffn(out1)
ffn_output = dropout2(ffn_output)
add_inputs_2 = tf.keras.layers.Add()([out1, ffn_output])
output = layernorm2(add_inputs_2)
return output
def build(self, hp, inputs=None):
"""
# Arguments
hp: HyperParameters. The hyperparameters for building the model.
inputs: Tensor of Shape [batch_size, seq_len]
# Returns
Output Tensor of shape `[batch_size, seq_len, embedding_dim]`.
"""
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
pretraining = utils.add_to_hp(self.pretraining, hp)
embedding_dim = utils.add_to_hp(self.embedding_dim, hp)
num_heads = utils.add_to_hp(self.num_heads, hp)
dense_dim = utils.add_to_hp(self.dense_dim, hp)
dropout = utils.add_to_hp(self.dropout, hp)
ffn = tf.keras.Sequential(
[
layers.Dense(dense_dim, activation="relu"),
layers.Dense(embedding_dim),
]
)
layernorm1 = layers.LayerNormalization(epsilon=1e-6)
layernorm2 = layers.LayerNormalization(epsilon=1e-6)
dropout1 = layers.Dropout(dropout)
dropout2 = layers.Dropout(dropout)
# Token and Position Embeddings
input_node = nest.flatten(inputs)[0]
token_embedding = Embedding(
max_features=self.max_features,
pretraining=pretraining,
embedding_dim=embedding_dim,
dropout=dropout,
).build(hp, input_node)
maxlen = input_node.shape[-1]
batch_size = tf.shape(input_node)[0]
positions = self.pos_array_funct(maxlen, batch_size)
position_embedding = Embedding(
max_features=maxlen,
pretraining=pretraining,
embedding_dim=embedding_dim,
dropout=dropout,
).build(hp, positions)
output_node = tf.keras.layers.Add()([token_embedding, position_embedding])
attn_output = MultiHeadSelfAttention(embedding_dim, num_heads).build(
hp, output_node
)
attn_output = dropout1(attn_output)
add_inputs_1 = tf.keras.layers.Add()([output_node, attn_output])
out1 = layernorm1(add_inputs_1)
ffn_output = ffn(out1)
ffn_output = dropout2(ffn_output)
add_inputs_2 = tf.keras.layers.Add()([out1, ffn_output])
return layernorm2(add_inputs_2)
def build(self, hp, inputs=None):
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
output_node = input_node
dropout = self.dropout or hp.Choice("dropout", [0.0, 0.25, 0.5], default=0)
if dropout > 0:
output_node = layers.Dropout(dropout)(output_node)
output_node = reduction.Flatten().build(hp, output_node)
output_node = layers.Dense(self.shape[-1], name=self.name)(output_node)
return output_node
def build(self, hp, inputs=None):
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
output_node = input_node
kernel_size = self.kernel_size or hp.Choice("kernel_size", [3, 5, 7],
default=3)
num_blocks = self.num_blocks or hp.Choice("num_blocks", [1, 2, 3],
default=2)
num_layers = self.num_layers or hp.Choice("num_layers", [1, 2],
default=2)
separable = self.separable
if separable is None:
separable = hp.Boolean("separable", default=False)
if separable:
conv = layer_utils.get_sep_conv(input_node.shape)
else:
conv = layer_utils.get_conv(input_node.shape)
max_pooling = self.max_pooling
if max_pooling is None:
max_pooling = hp.Boolean("max_pooling", default=True)
pool = layer_utils.get_max_pooling(input_node.shape)
if self.dropout is not None:
dropout = self.dropout
else:
dropout = hp.Choice("dropout", [0.0, 0.25, 0.5], default=0)
for i in range(num_blocks):
for j in range(num_layers):
output_node = conv(
hp.Choice(
"filters_{i}_{j}".format(i=i, j=j),
[16, 32, 64, 128, 256, 512],
default=32,
),
kernel_size,
padding=self._get_padding(kernel_size, output_node),
activation="relu",
)(output_node)
if max_pooling:
output_node = pool(
kernel_size - 1,
padding=self._get_padding(kernel_size - 1, output_node),
)(output_node)
if dropout > 0:
output_node = layers.Dropout(dropout)(output_node)
return output_node
def build(self, hp, inputs=None):
"""
# Arguments
hp: HyperParameters. The hyperparameters for building the model.
inputs: Tensor of Shape [batch_size, seq_len, embedding_dim]
# Returns
Self-Attention outputs of shape `[batch_size, seq_len, embedding_dim]`.
"""
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
shape = input_node.shape.as_list()
if len(shape) != 3:
raise ValueError('Expect the input tensor to have '
'3 dimensions for multi-head self-attention, '
'but got {shape}'.format(shape=input_node.shape))
# input.shape = [batch_size, seq_len, embedding_dim]
head_size = self.head_size or hp.Choice(
'head_size', [32, 64, 128, 256, 512], default=128)
num_heads = self.num_heads
if num_heads is None:
num_heads = 8
if head_size % num_heads != 0: # how to evaluate this condition
raise ValueError(f"embedding dimension = {head_size} should be "
f"divisible by number of heads = {num_heads}")
projection_dim = head_size // num_heads
query_dense = layers.Dense(head_size)
key_dense = layers.Dense(head_size)
value_dense = layers.Dense(head_size)
combine_heads = layers.Dense(head_size)
batch_size = tf.shape(input_node)[0]
query = query_dense(input_node) # (batch_size, seq_len, head_size)
key = key_dense(input_node) # (batch_size, seq_len, head_size)
value = value_dense(input_node) # (batch_size, seq_len, head_size)
query, key, value = [
self.separate_heads(var, batch_size, num_heads, projection_dim)
for var in [query, key, value]
]
attention, weights = self.attention(query, key, value)
attention = tf.transpose(attention, perm=[
0, 2, 1, 3
]) # (batch_size, seq_len, num_heads, projection_dim)
concat_attention = tf.reshape(
attention, (batch_size, tf.shape(attention)[1],
self.head_size)) # (batch_size, seq_len, head_size)
output = combine_heads(
concat_attention) # (batch_size, seq_len, head_size)
return output
def build(self, hp, inputs=None):
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
output_node = input_node
# No need to reduce.
if len(output_node.shape) <= 2:
return output_node
if self.reduction_type is None:
reduction_type = hp.Choice(REDUCTION_TYPE,
[FLATTEN, GLOBAL_MAX, GLOBAL_AVG])
with hp.conditional_scope(REDUCTION_TYPE, [reduction_type]):
return self._build_block(hp, output_node, reduction_type)
else:
return self._build_block(hp, output_node, self.reduction_type)
def build(self, hp, inputs=None):
if self.output_dim and self.output_shape[-1] != self.output_dim:
raise ValueError('The data doesn\'t match the output_dim. '
'Expecting {} but got {}'.format(
self.output_dim, self.output_shape[-1]))
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
output_node = input_node
dropout_rate = self.dropout_rate or hp.Choice(
'dropout_rate', [0.0, 0.25, 0.5], default=0)
if dropout_rate > 0:
output_node = layers.Dropout(dropout_rate)(output_node)
output_node = reduction.Flatten().build(hp, output_node)
output_node = layers.Dense(self.output_shape[-1],
name=self.name)(output_node)
return output_node
def build(self, hp, inputs=None):
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
output_node = input_node
kernel_size = utils.add_to_hp(self.kernel_size, hp)
separable = self.separable
if separable is None:
separable = hp.Boolean("separable", default=False)
if separable:
conv = layer_utils.get_sep_conv(input_node.shape)
else:
conv = layer_utils.get_conv(input_node.shape)
max_pooling = self.max_pooling
if max_pooling is None:
max_pooling = hp.Boolean("max_pooling", default=True)
pool = layer_utils.get_max_pooling(input_node.shape)
for i in range(utils.add_to_hp(self.num_blocks, hp)):
for j in range(utils.add_to_hp(self.num_layers, hp)):
output_node = conv(
utils.add_to_hp(
self.filters, hp, "filters_{i}_{j}".format(i=i, j=j)
),
kernel_size,
padding=self._get_padding(kernel_size, output_node),
activation="relu",
)(output_node)
if max_pooling:
output_node = pool(
kernel_size - 1,
padding=self._get_padding(kernel_size - 1, output_node),
)(output_node)
if utils.add_to_hp(self.dropout, hp) > 0:
output_node = layers.Dropout(utils.add_to_hp(self.dropout, hp))(
output_node
)
return output_node
def build(self, hp, inputs=None):
"""
# Arguments
hp: HyperParameters. The hyperparameters for building the model.
inputs: Tensor of Shape [batch_size, seq_len, embedding_dim]
# Returns
Self-Attention outputs of shape `[batch_size, seq_len, embedding_dim]`.
"""
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
num_heads = self.num_heads
head_size = (
self.head_size
or hp.Choice("head_size_factor", [4, 8, 16, 32, 64], default=16) *
num_heads)
projection_dim = head_size // num_heads
query_dense = layers.Dense(head_size)
key_dense = layers.Dense(head_size)
value_dense = layers.Dense(head_size)
combine_heads = layers.Dense(head_size)
batch_size = tf.shape(input_node)[0]
query = query_dense(input_node) # (batch_size, seq_len, head_size)
key = key_dense(input_node) # (batch_size, seq_len, head_size)
value = value_dense(input_node) # (batch_size, seq_len, head_size)
query, key, value = [
self.separate_heads(var, batch_size, num_heads, projection_dim)
for var in [query, key, value]
]
attention, weights = self.attention(query, key, value)
attention = tf.transpose(attention, perm=[
0, 2, 1, 3
]) # (batch_size, seq_len, num_heads, projection_dim)
concat_attention = tf.reshape(
attention, (batch_size, tf.shape(attention)[1],
self.head_size)) # (batch_size, seq_len, head_size)
output = combine_heads(
concat_attention) # (batch_size, seq_len, head_size)
return output
def build(self, hp, inputs=None):
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
output_node = input_node
# No need to reduce.
if len(output_node.shape) <= 2:
return output_node
reduction_type = self.reduction_type or hp.Choice(
'reduction_type', ['flatten', 'global_max', 'global_avg'],
default='global_avg')
if reduction_type == 'flatten':
output_node = Flatten().build(hp, output_node)
elif reduction_type == 'global_max':
output_node = layer_utils.get_global_max_pooling(
output_node.shape)()(output_node)
elif reduction_type == 'global_avg':
output_node = layer_utils.get_global_average_pooling(
output_node.shape)()(output_node)
return output_node
def build(self, hp, inputs=None):
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
output_node = input_node
# No need to reduce.
if len(output_node.shape) <= 2:
return output_node
reduction_type = self.reduction_type or hp.Choice(
'reduction_type', ['flatten', 'global_max', 'global_avg'],
default='global_avg')
if reduction_type == 'flatten':
output_node = Flatten().build(hp, output_node)
elif reduction_type == 'global_max':
output_node = tf.math.reduce_max(output_node, axis=-2)
elif reduction_type == 'global_avg':
output_node = tf.math.reduce_mean(output_node, axis=-2)
elif reduction_type == 'global_min':
output_node = tf.math.reduce_min(output_node, axis=-2)
return output_node
def test_validate_num_inputs_error():
with pytest.raises(ValueError) as info:
utils.validate_num_inputs([1, 2, 3], 2)
assert "Expected 2 elements in the inputs list" in str(info.value)
