def adapt(model, dataset):
"""Adapt the preprocessing layers in the model."""
# Currently, only support using the original dataset to adapt all the
# preprocessing layers before the first non-preprocessing layer.
# TODO: Use PreprocessingStage for preprocessing layers adapt.
# TODO: Use Keras Tuner for preprocessing layers adapt.
x = dataset.map(lambda x, y: x)
def get_output_layers(tensor):
output_layers = []
tensor = nest.flatten(tensor)[0]
for layer in model.layers:
if isinstance(layer, keras.layers.InputLayer):
continue
input_node = nest.flatten(layer.input)[0]
if input_node is tensor:
if isinstance(layer, preprocessing.PreprocessingLayer):
output_layers.append(layer)
return output_layers
dq = collections.deque()
for index, input_node in enumerate(nest.flatten(model.input)):
in_x = x.map(lambda *args: nest.flatten(args)[index])
for layer in get_output_layers(input_node):
dq.append((layer, in_x))
while len(dq):
layer, in_x = dq.popleft()
layer.adapt(in_x)
out_x = in_x.map(layer)
for next_layer in get_output_layers(layer.output):
dq.append((next_layer, out_x))
return model
def __init__(self,
update_ops=None,
collections=_tools.GraphKeys.CUSTOM_UPDATE_OPS,
save_secs=None,
save_steps=None):
"""Builds the object.
Args:
update_ops: an op or a list of ops.
collections: tensorflow collection key or a list of collections keys.
save_secs: number of seconds to wait until saving.
save_steps: number of steps to wait until saving.
"""
logging.info('Create UpdateOpsHook.')
update_ops = nest.flatten(update_ops) if update_ops else []
if collections:
for collection in nest.flatten(collections):
update_ops.extend(tf.get_collection(collection))
self._update_op = tf.group(update_ops)
self._timer = tf.train.SecondOrStepTimer(every_secs=save_secs,
every_steps=save_steps)
self._steps_per_run = 1
self._last_run = -1
def build(self, hp, inputs=None):
"""
# Arguments
hp: HyperParameters. The hyperparameters for building the model.
inputs: Tensor of Shape [batch_size, seq_len]
# Returns
Output Tensor of shape `[batch_size, seq_len, embedding_dim]`.
"""
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
pretraining = utils.add_to_hp(self.pretraining, hp)
embedding_dim = utils.add_to_hp(self.embedding_dim, hp)
num_heads = utils.add_to_hp(self.num_heads, hp)
dense_dim = utils.add_to_hp(self.dense_dim, hp)
dropout = utils.add_to_hp(self.dropout, hp)
ffn = keras.Sequential(
[
layers.Dense(dense_dim, activation="relu"),
layers.Dense(embedding_dim),
]
)
layernorm1 = layers.LayerNormalization(epsilon=1e-6)
layernorm2 = layers.LayerNormalization(epsilon=1e-6)
dropout1 = layers.Dropout(dropout)
dropout2 = layers.Dropout(dropout)
# Token and Position Embeddings
input_node = nest.flatten(inputs)[0]
token_embedding = Embedding(
max_features=self.max_features,
pretraining=pretraining,
embedding_dim=embedding_dim,
dropout=dropout,
).build(hp, input_node)
maxlen = input_node.shape[-1]
batch_size = tf.shape(input_node)[0]
positions = self.pos_array_funct(maxlen, batch_size)
position_embedding = Embedding(
max_features=maxlen,
pretraining=pretraining,
embedding_dim=embedding_dim,
dropout=dropout,
).build(hp, positions)
output_node = keras.layers.Add()([token_embedding, position_embedding])
attn_output = MultiHeadSelfAttention(embedding_dim, num_heads).build(
hp, output_node
)
attn_output = dropout1(attn_output)
add_inputs_1 = keras.layers.Add()([output_node, attn_output])
out1 = layernorm1(add_inputs_1)
ffn_output = ffn(out1)
ffn_output = dropout2(ffn_output)
add_inputs_2 = keras.layers.Add()([out1, ffn_output])
return layernorm2(add_inputs_2)
def get_output_layers(tensor):
output_layers = []
tensor = nest.flatten(tensor)[0]
for layer in model.layers:
if isinstance(layer, keras.layers.InputLayer):
continue
input_node = nest.flatten(layer.input)[0]
if input_node is tensor:
if isinstance(layer, preprocessing.PreprocessingLayer):
output_layers.append(layer)
return output_layers
def test_reduction_2d_tensor_return_input_node():
block = blocks.TemporalReduction()
input_node = keras.Input(shape=(32,), dtype=tf.float32)
outputs = block.build(
keras_tuner.HyperParameters(),
input_node,
)
assert len(nest.flatten(outputs)) == 1
assert nest.flatten(outputs)[0] is input_node
def call(self, inputs, *args, **kargs):
'''Call the layers in sequential order.'''
# assert self.input_shape == inputs.shape TODO
tdl.core.assert_initialized(self, 'call', ['layers'])
flatten_inputs = nest.flatten(inputs)
flatten_layers = nest.flatten(self.layers)
flatten_output = list()
for layer, input in zip(flatten_layers, flatten_inputs):
flatten_output.append(layer(input))
return nest.pack_sequence_as(
structure=self.layers,
flat_sequence=flatten_output)
def __init__(self, inputs=None, outputs=None, **kwargs):
super().__init__(**kwargs)
self.inputs = nest.flatten(inputs)
self.outputs = nest.flatten(outputs)
self._node_to_id = {}
self._nodes = []
self.blocks = []
self._block_to_id = {}
if inputs and outputs:
self._build_network()
# Temporary attributes
self.epochs = None
self.num_samples = None
def _analyze_data(self, dataset):
input_analysers = [node.get_analyser() for node in self.inputs]
output_analysers = [head.get_analyser() for head in self._heads]
analysers = input_analysers + output_analysers
for x, y in dataset:
x = nest.flatten(x)
y = nest.flatten(y)
for item, analyser in zip(x + y, analysers):
analyser.update(item)
for analyser in analysers:
analyser.finalize()
for hm, analyser in zip(self.inputs + self._heads, analysers):
hm.config_from_analyser(analyser)
def test_embed_build_return_tensor():
block = blocks.Embedding()
outputs = block.build(keras_tuner.HyperParameters(),
tf.keras.Input(shape=(32, ), dtype=tf.float32))
assert len(nest.flatten(outputs)) == 1
def build(self, hp, inputs=None):
input_node = nest.flatten(inputs)[0]
output_node = input_node
if self.normalize is None and hp.Boolean(NORMALIZE):
with hp.conditional_scope(NORMALIZE, [True]):
output_node = preprocessing.Normalization().build(hp, output_node)
elif self.normalize:
output_node = preprocessing.Normalization().build(hp, output_node)
if self.augment is None and hp.Boolean(AUGMENT):
with hp.conditional_scope(AUGMENT, [True]):
output_node = preprocessing.ImageAugmentation().build(
hp, output_node
)
elif self.augment:
output_node = preprocessing.ImageAugmentation().build(hp, output_node)
if self.block_type is None:
block_type = hp.Choice(
BLOCK_TYPE, [RESNET, XCEPTION, VANILLA, EFFICIENT]
)
with hp.conditional_scope(BLOCK_TYPE, [block_type]):
output_node = self._build_block(hp, output_node, block_type)
else:
output_node = self._build_block(hp, output_node, self.block_type)
return output_node
def test_dense_build_with_bn_return_tensor():
block = blocks.DenseBlock(use_batchnorm=True)
outputs = block.build(keras_tuner.HyperParameters(),
tf.keras.Input(shape=(32, ), dtype=tf.float32))
assert len(nest.flatten(outputs)) == 1
def test_ngram_build_with_ngrams_return_tensor():
block = blocks.TextToNgramVector(ngrams=2)
outputs = block.build(keras_tuner.HyperParameters(),
tf.keras.Input(shape=(1, ), dtype=tf.string))
assert len(nest.flatten(outputs)) == 1
def build(self, hp, inputs=None):
input_node = nest.flatten(inputs)[0]
# TODO: support more pretrained embedding layers.
# glove, fasttext, and word2vec
pretraining = utils.add_to_hp(self.pretraining, hp)
embedding_dim = utils.add_to_hp(self.embedding_dim, hp)
if pretraining != "none":
# TODO: load from pretrained weights
layer = layers.Embedding(
input_dim=self.max_features,
output_dim=embedding_dim,
input_length=input_node.shape[1],
)
# trainable=False,
# weights=[embedding_matrix])
else:
layer = layers.Embedding(
input_dim=self.max_features, output_dim=embedding_dim
)
# input_length=input_node.shape[1],
# trainable=True)
output_node = layer(input_node)
dropout = utils.add_to_hp(self.dropout, hp)
if dropout > 0:
output_node = layers.Dropout(dropout)(output_node)
return output_node
def validate_num_inputs(inputs, num):
inputs = nest.flatten(inputs)
if not len(inputs) == num:
raise ValueError(
"Expected {num} elements in the inputs list "
"but received {len} inputs.".format(num=num, len=len(inputs))
)
def build(self, hp, inputs=None):
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
shape = input_node.shape.as_list()
if len(shape) != 3:
raise ValueError(
"Expect the input tensor of RNNBlock to have dimensions of "
"[batch_size, time_steps, vec_len], "
"but got {shape}".format(shape=input_node.shape)
)
feature_size = shape[-1]
output_node = input_node
bidirectional = utils.add_to_hp(self.bidirectional, hp)
layer_type = utils.add_to_hp(self.layer_type, hp)
num_layers = utils.add_to_hp(self.num_layers, hp)
rnn_layers = {"gru": layers.GRU, "lstm": layers.LSTM}
in_layer = rnn_layers[layer_type]
for i in range(num_layers):
return_sequences = True
if i == num_layers - 1:
return_sequences = self.return_sequences
if bidirectional:
output_node = layers.Bidirectional(
in_layer(feature_size, return_sequences=return_sequences)
)(output_node)
else:
output_node = in_layer(
feature_size, return_sequences=return_sequences
)(output_node)
return output_node
def build(self, hp):
"""Build the HyperModel into a Keras Model."""
self.compile()
keras_nodes = {}
keras_input_nodes = []
for node in self.inputs:
node_id = self._node_to_id[node]
input_node = node.build_node(hp)
output_node = node.build(hp, input_node)
keras_input_nodes.append(input_node)
keras_nodes[node_id] = output_node
for block in self.blocks:
temp_inputs = [
keras_nodes[self._node_to_id[input_node]]
for input_node in block.inputs
]
outputs = block.build(hp, inputs=temp_inputs)
outputs = nest.flatten(outputs)
for output_node, real_output_node in zip(block.outputs, outputs):
keras_nodes[self._node_to_id[output_node]] = real_output_node
model = tf.keras.Model(
keras_input_nodes,
[
keras_nodes[self._node_to_id[output_node]]
for output_node in self.outputs
],
)
return self._compile_keras_model(hp, model)
def build(self, hp, inputs=None):
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
if len(input_node.shape) > 2:
return layers.Flatten()(input_node)
return input_node
def build(self, hp, inputs=None):
inputs = nest.flatten(inputs)
utils.validate_num_inputs(inputs, 1)
input_node = inputs[0]
output_node = input_node
# Reduce the tensor to a vector.
if len(output_node.shape) > 2:
output_node = reduction.SpatialReduction().build(hp, output_node)
if self.dropout is not None:
dropout = self.dropout
else:
dropout = hp.Choice("dropout", [0.0, 0.25, 0.5], default=0)
if dropout > 0:
output_node = layers.Dropout(dropout)(output_node)
output_node = layers.Dense(self.shape[-1])(output_node)
if isinstance(self.loss, keras.losses.BinaryCrossentropy):
output_node = layers.Activation(activations.sigmoid, name=self.name)(
output_node
)
else:
output_node = layers.Softmax(name=self.name)(output_node)
return output_node
def test_transformer_build_return_tensor():
block = blocks.Transformer()
outputs = block.build(keras_tuner.HyperParameters(),
tf.keras.Input(shape=(64, ), dtype=tf.float32))
assert len(nest.flatten(outputs)) == 1
def test_int_seq_build_with_seq_len_return_tensor():
block = blocks.TextToIntSequence(output_sequence_length=50)
outputs = block.build(keras_tuner.HyperParameters(),
tf.keras.Input(shape=(1, ), dtype=tf.string))
assert len(nest.flatten(outputs)) == 1
def test_bert_build_return_tensor():
block = blocks.BertBlock()
outputs = block.build(keras_tuner.HyperParameters(),
tf.keras.Input(shape=(1, ), dtype=tf.string))
assert len(nest.flatten(outputs)) == 1
def test_text_block_ngram_return_tensor():
block = blocks.TextBlock(block_type="ngram")
outputs = block.build(keras_tuner.HyperParameters(),
tf.keras.Input(shape=(1, ), dtype=tf.string))
assert len(nest.flatten(outputs)) == 1
def _build(self, data):
x = data[self._input_key]
presence = data[self._presence_key] if self._presence_key else None
inputs = nest.flatten(x)
if presence is not None:
inputs.append(presence)
h = self._encoder(*inputs)
res = self._decoder(h, *inputs)
n_points = int(res.posterior_mixing_probs.shape[1])
mass_explained_by_capsule = tf.reduce_sum(res.posterior_mixing_probs, 1)
(res.posterior_within_sparsity_loss,
res.posterior_between_sparsity_loss) = _capsule.sparsity_loss(
self._posterior_sparsity_loss_type,
mass_explained_by_capsule / n_points,
num_classes=self._n_classes)
(res.prior_within_sparsity_loss,
res.prior_between_sparsity_loss) = _capsule.sparsity_loss(
self._prior_sparsity_loss_type,
res.caps_presence_prob,
num_classes=self._n_classes,
within_example_constant=self._prior_within_example_constant)
return res
def build(self, hp, inputs=None):
input_node = nest.flatten(inputs)[0]
output_node = input_node
# Translate
translation_factor = self.translation_factor
if translation_factor is None:
translation_factor = hp.Choice("translation_factor", [0.0, 0.1])
if translation_factor not in [0, (0, 0)]:
height_factor, width_factor = self._get_fraction_value(
translation_factor)
output_node = preprocessing.RandomTranslation(
height_factor, width_factor)(output_node)
# Flip
horizontal_flip = self.horizontal_flip
if horizontal_flip is None:
horizontal_flip = hp.Boolean("horizontal_flip", default=True)
vertical_flip = self.vertical_flip
if self.vertical_flip is None:
vertical_flip = hp.Boolean("vertical_flip", default=True)
if not horizontal_flip and not vertical_flip:
flip_mode = ""
elif horizontal_flip and vertical_flip:
flip_mode = "horizontal_and_vertical"
elif horizontal_flip and not vertical_flip:
flip_mode = "horizontal"
elif not horizontal_flip and vertical_flip:
flip_mode = "vertical"
if flip_mode != "":
output_node = preprocessing.RandomFlip(mode=flip_mode)(output_node)
# Rotate
rotation_factor = self.rotation_factor
if rotation_factor is None:
rotation_factor = hp.Choice("rotation_factor", [0.0, 0.1])
if rotation_factor != 0:
output_node = preprocessing.RandomRotation(rotation_factor)(
output_node)
# Zoom
zoom_factor = self.zoom_factor
if zoom_factor is None:
zoom_factor = hp.Choice("zoom_factor", [0.0, 0.1])
if zoom_factor not in [0, (0, 0)]:
height_factor, width_factor = self._get_fraction_value(zoom_factor)
# TODO: Add back RandomZoom when it is ready.
# output_node = preprocessing.RandomZoom(
# height_factor, width_factor)(output_node)
# Contrast
contrast_factor = self.contrast_factor
if contrast_factor is None:
contrast_factor = hp.Choice("contrast_factor", [0.0, 0.1])
if contrast_factor not in [0, (0, 0)]:
output_node = preprocessing.RandomContrast(contrast_factor)(
output_node)
return output_node
def test_structured_data_get_col_names_from_df(fit, tmp_path):
clf = ak.StructuredDataClassifier(
directory=tmp_path,
seed=test_utils.SEED,
)
clf.fit(x=test_utils.TRAIN_CSV_PATH, y="survived")
assert nest.flatten(clf.inputs)[0].column_names[0] == "sex"
def preprocess(x):
"""Cast to float, normalize, and concatenate images along last axis."""
x = nest.map_structure(
lambda image: tf.image.convert_image_dtype(image, tf.float32), x)
x = nest.flatten(x)
x = tf.concat(x, axis=-1)
x = (tf.image.convert_image_dtype(x, tf.float32) - 0.5) * 2.0
return x
def test_dense_build_with_dropout_return_tensor():
block = blocks.DenseBlock(dropout=0.5)
outputs = block.build(
keras_tuner.HyperParameters(), keras.Input(shape=(32,), dtype=tf.float32)
)
assert len(nest.flatten(outputs)) == 1
def test_image_block_augment_return_tensor():
block = blocks.ImageBlock(augment=True)
outputs = block.build(
keras_tuner.HyperParameters(),
tf.keras.Input(shape=(32, 32, 3), dtype=tf.float32),
)
assert len(nest.flatten(outputs)) == 1
def test_structured_block_normalize_return_tensor():
block = blocks.StructuredDataBlock(normalize=True)
block.column_names = ["0", "1"]
block.column_types = {"0": analysers.NUMERICAL, "1": analysers.NUMERICAL}
outputs = block.build(keras_tuner.HyperParameters(),
tf.keras.Input(shape=(2, ), dtype=tf.string))
assert len(nest.flatten(outputs)) == 1
def test_clf_head_build_with_zero_dropout_return_tensor():
block = head_module.ClassificationHead(dropout=0, shape=(8,))
outputs = block.build(
keras_tuner.HyperParameters(),
tf.keras.Input(shape=(5,), dtype=tf.float32),
)
assert len(nest.flatten(outputs)) == 1
