def test_CrossNet(layer_num, ):
with CustomObjectScope({'CrossNet': layers.CrossNet}):
layer_test(layers.CrossNet,
kwargs={
'layer_num': layer_num,
},
input_shape=(2, 3))
def test_random_contrast_amplitude(self, amplitude):
with CustomObjectScope(
{'RandomContrast': image_preprocessing.RandomContrast}):
input_images = np.random.random((2, 5, 8, 3))
with tf_test_util.use_gpu():
layer = image_preprocessing.RandomContrast(amplitude)
layer(input_images)
def test_fit_embed(self):
model = keras.models.Sequential()
model.add(
keras.layers.Embedding(
input_shape=(None, ),
input_dim=5,
output_dim=16,
mask_zero=True,
))
model.add(keras.layers.Bidirectional(keras.layers.LSTM(units=8)))
model.add(keras.layers.Dense(units=2, activation='softmax'))
model.compile(AdamWarmup(
decay_steps=10000,
warmup_steps=5000,
learning_rate=1e-3,
min_lr=1e-4,
amsgrad=True,
weight_decay=1e-3,
),
loss='sparse_categorical_crossentropy')
x = np.random.randint(0, 5, (1024, 15))
y = (x[:, 1] > 2).astype('int32')
model.fit(x, y, epochs=10, verbose=1)
model_path = os.path.join(tempfile.gettempdir(),
'test_warmup_%f.h5' % np.random.random())
model.save(model_path)
from tensorflow.python.keras.utils.generic_utils import CustomObjectScope
with CustomObjectScope({
'AdamWarmup': AdamWarmup
}): # Workaround for incorrect global variable used in keras
keras.models.load_model(model_path,
custom_objects={'AdamWarmup': AdamWarmup})
def test_random_contrast_int_dtype(self):
with CustomObjectScope(
{'RandomContrast': image_preprocessing.RandomContrast}):
input_images = np.random.randint(low=0, high=255, size=(2, 5, 8, 3))
with tf_test_util.use_gpu():
layer = image_preprocessing.RandomContrast((0.1, 0.2))
layer(input_images)
def test_CIN(layer_size, split_half):
with CustomObjectScope({'CIN': layers.CIN}):
layer_test(layers.CIN,
kwargs={
"layer_size": layer_size,
"split_half": split_half
},
input_shape=(BATCH_SIZE, FIELD_SIZE, EMBEDDING_SIZE))
def test_FwFM(reg_strength):
with CustomObjectScope({'FwFMLayer': layers.FwFMLayer}):
layer_test(layers.FwFMLayer,
kwargs={
'num_fields': FIELD_SIZE,
'regularizer': reg_strength
},
input_shape=(BATCH_SIZE, FIELD_SIZE, EMBEDDING_SIZE))
def test_random_flip_inference(self):
with CustomObjectScope({'RandomFlip': image_preprocessing.RandomFlip}):
input_images = np.random.random((2, 5, 8, 3)).astype(np.float32)
expected_output = input_images
with tf_test_util.use_gpu():
layer = image_preprocessing.RandomFlip()
actual_output = layer(input_images, training=0)
self.assertAllClose(expected_output, actual_output)
def test_random_crop_output_shape(self, expected_height, expected_width):
if test.is_built_with_rocm():
# TODO(rocm):
# re-enable this test once ROCm adds support for
# the StatefulUniformFullInt Op (on the GPU)
self.skipTest('Feature not supported on ROCm')
with CustomObjectScope({'RandomCrop': image_preprocessing.RandomCrop}):
self._run_test(expected_height, expected_width)
def test_LocalActivationUnit(hidden_units, activation):
if tf.__version__ >= '1.13.0' and activation != 'sigmoid':
return
with CustomObjectScope({'LocalActivationUnit': layers.LocalActivationUnit}):
layer_test(layers.LocalActivationUnit,
kwargs={'hidden_units': hidden_units, 'activation': activation, 'dropout_rate': 0.5},
input_shape=[(BATCH_SIZE, 1, EMBEDDING_SIZE), (BATCH_SIZE, SEQ_LENGTH, EMBEDDING_SIZE)])
def test_KMaxPooling():
with CustomObjectScope({'KMaxPooling': sequence.KMaxPooling}):
layer_test(sequence.KMaxPooling,
kwargs={
'k': 3,
'axis': 1
},
input_shape=(BATCH_SIZE, SEQ_LENGTH, EMBEDDING_SIZE, 2))
def test_Linear():
with CustomObjectScope({'Linear': Linear}):
layer_test(Linear,
kwargs={
'mode': 1,
'use_bias': True
},
input_shape=(BATCH_SIZE, EMBEDDING_SIZE))
def test_random_flip_vertical_half(self):
with CustomObjectScope({'RandomFlip': image_preprocessing.RandomFlip}):
np.random.seed(1337)
mock_random = [1, 0]
mock_random = np.reshape(mock_random, [2, 1, 1, 1])
input_images = np.random.random((2, 5, 8, 3)).astype(np.float32)
expected_output = input_images.copy()
expected_output[0, :, :, :] = np.flip(input_images[0, :, :, :], axis=0)
self._run_test('vertical', expected_output, mock_random)
def test_AttentionSequencePoolingLayer(weight_normalization):
with CustomObjectScope({
'AttentionSequencePoolingLayer':
sequence.AttentionSequencePoolingLayer
}):
layer_test(sequence.AttentionSequencePoolingLayer,
kwargs={'weight_normalization': weight_normalization},
input_shape=[(BATCH_SIZE, 1, EMBEDDING_SIZE),
(BATCH_SIZE, SEQ_LENGTH, EMBEDDING_SIZE),
(BATCH_SIZE, 1)])
def _test_fit(self, optimizer, atol=1e-2):
x, y, w = self.gen_linear_data()
model = self.gen_linear_model(optimizer)
callbacks = [
keras.callbacks.EarlyStopping(monitor='loss',
patience=3,
min_delta=1e-8)
]
if isinstance(optimizer, RAdam):
model_path = os.path.join(
tempfile.gettempdir(),
'test_accumulation_%f.h5' % np.random.random())
model.save(model_path)
from tensorflow.python.keras.utils.generic_utils import CustomObjectScope
with CustomObjectScope({
'RAdam': RAdam
}): # Workaround for incorrect global variable used in keras
model = keras.models.load_model(
model_path, custom_objects={'RAdam': RAdam})
callbacks.append(
keras.callbacks.ReduceLROnPlateau(monitor='loss',
min_lr=1e-8,
patience=2,
verbose=True))
model.fit(x, y, epochs=100, batch_size=32, callbacks=callbacks)
model_path = os.path.join(
tempfile.gettempdir(),
'test_accumulation_%f.h5' % np.random.random())
model.save(model_path)
from tensorflow.python.keras.utils.generic_utils import CustomObjectScope
with CustomObjectScope(
{'RAdam':
RAdam}): # Workaround for incorrect global variable used in keras
model = keras.models.load_model(model_path,
custom_objects={'RAdam': RAdam})
x, y, w = self.gen_linear_data(w)
predicted = model.predict(x)
self.assertLess(np.max(np.abs(predicted - y)), atol)
def test_PositionEncoding(pos_embedding_trainable, zero_pad):
with CustomObjectScope({
'PositionEncoding': sequence.PositionEncoding,
"tf": tf
}):
layer_test(sequence.PositionEncoding,
kwargs={
'pos_embedding_trainable': pos_embedding_trainable,
'zero_pad': zero_pad
},
input_shape=(BATCH_SIZE, SEQ_LENGTH, EMBEDDING_SIZE))
def test_InteractingLayer(
head_num,
use_res,
):
with CustomObjectScope({'InteractingLayer': layers.InteractingLayer}):
layer_test(layers.InteractingLayer,
kwargs={
"head_num": head_num,
"use_res": use_res,
},
input_shape=(BATCH_SIZE, FIELD_SIZE, EMBEDDING_SIZE))
def test_BiLSTM(merge_mode):
with CustomObjectScope({'BiLSTM': sequence.BiLSTM}):
layer_test(
sequence.BiLSTM,
kwargs={
'merge_mode': merge_mode,
'units': EMBEDDING_SIZE,
'dropout_rate': 0.0
},
# todo 0.5
input_shape=(BATCH_SIZE, SEQ_LENGTH, EMBEDDING_SIZE))
def test_layer_api_compatibility(self):
cls = get_layer_class()
with CustomObjectScope({"Normalization": cls}):
output_data = testing_utils.layer_test(
cls,
kwargs={"axis": -1},
input_shape=(None, 3),
input_data=np.array([[3, 1, 2], [6, 5, 4]], dtype=np.float32),
validate_training=False,
adapt_data=np.array([[1, 2, 1], [2, 3, 4], [1, 2, 1], [2, 3, 4]]))
expected = np.array([[3., -3., -0.33333333], [9., 5., 1.]])
self.assertAllClose(expected, output_data)
def test_random_flip_default(self):
with CustomObjectScope({'RandomFlip': image_preprocessing.RandomFlip}):
input_images = np.random.random((2, 5, 8, 3)).astype(np.float32)
expected_output = np.flip(np.flip(input_images, axis=1), axis=2)
mock_random = [1, 1]
mock_random = np.reshape(mock_random, [2, 1, 1, 1])
with test.mock.patch.object(
random_ops, 'random_uniform', return_value=mock_random):
with self.cached_session(use_gpu=True):
layer = image_preprocessing.RandomFlip()
actual_output = layer(input_images, training=1)
self.assertAllClose(expected_output, actual_output)
def test_SequencePoolingLayer(mode, supports_masking, input_shape):
if version.parse(tf.__version__) >= version.parse(
'1.14.0') and mode != 'sum': # todo check further version
return
with CustomObjectScope(
{'SequencePoolingLayer': sequence.SequencePoolingLayer}):
layer_test(sequence.SequencePoolingLayer,
kwargs={
'mode': mode,
'supports_masking': supports_masking
},
input_shape=input_shape,
supports_masking=supports_masking)
def test_FGCNNLayer():
with CustomObjectScope({'FGCNNLayer': layers.FGCNNLayer}):
layer_test(layers.FGCNNLayer,
kwargs={
'filters': (
4,
6,
),
'kernel_width': (
7,
7,
)
},
input_shape=(BATCH_SIZE, FIELD_SIZE, EMBEDDING_SIZE))
def test_Transformer():
with CustomObjectScope({'Transformer': sequence.Transformer}):
layer_test(sequence.Transformer,
kwargs={
'att_embedding_size': 1,
'head_num': 8,
'use_layer_norm': True,
'supports_masking': False,
'attention_type': 'additive',
'dropout_rate': 0.5,
'output_type': 'sum'
},
input_shape=[(BATCH_SIZE, SEQ_LENGTH, EMBEDDING_SIZE),
(BATCH_SIZE, SEQ_LENGTH, EMBEDDING_SIZE),
(BATCH_SIZE, 1), (BATCH_SIZE, 1)])
def test_int_output_explicit_vocab_from_config(self):
vocab_data = ["earth", "wind", "and", "fire"]
input_array = np.array([["earth", "wind", "and", "fire"],
["fire", "and", "earth", "michigan"]])
expected_output = [[2, 3, 4, 5], [5, 4, 2, 1]]
input_data = keras.Input(shape=(None, ), dtype=dtypes.string)
layer = get_layer_class()(vocabulary=vocab_data)
int_data = layer(input_data)
model = keras.Model(inputs=input_data, outputs=int_data)
with CustomObjectScope({"IndexLookup": get_layer_class()}):
new_model = keras.Model.from_config(model.get_config())
output_dataset = new_model.predict(input_array)
self.assertAllEqual(expected_output, output_dataset)
def test_Hash(num_buckets, mask_zero, vocabulary_path, input_data,
expected_output):
if not hasattr(tf, 'version') or tf.version.VERSION < '2.0.0':
return
with CustomObjectScope({'Hash': Hash}):
layer_test(Hash,
kwargs={
'num_buckets': num_buckets,
'mask_zero': mask_zero,
'vocabulary_path': vocabulary_path
},
input_dtype=tf.string,
input_data=np.array(input_data, dtype='str'),
expected_output_dtype=tf.int64,
expected_output=expected_output)
def test_sample(self):
model = get_model(
token_num=200,
head_num=3,
transformer_num=2,
)
model_path = os.path.join(tempfile.gettempdir(),
'keras_bert_%f.h5' % np.random.random())
model.save(model_path)
from tensorflow.python.keras.utils.generic_utils import CustomObjectScope
with CustomObjectScope(get_custom_objects(
)): # Workaround for incorrect global variable used in keras
model = keras.models.load_model(
model_path,
custom_objects=get_custom_objects(),
)
model.summary(line_length=200)
def test_fit_embed(self):
optimizers = [RAdam]
for optimizer in optimizers:
for amsgrad in [False, True]:
model = keras.models.Sequential()
model.add(
keras.layers.Embedding(
input_shape=(None, ),
input_dim=5,
output_dim=16,
mask_zero=True,
))
model.add(
keras.layers.Bidirectional(keras.layers.LSTM(units=8)))
model.add(keras.layers.Dense(units=2, activation='softmax'))
model.compile(optimizer(
total_steps=38400,
warmup_proportion=0.1,
min_lr=1e-6,
weight_decay=1e-6,
amsgrad=amsgrad,
),
loss='sparse_categorical_crossentropy')
x = np.random.randint(0, 5, (64, 3))
y = []
for i in range(x.shape[0]):
if 2 in x[i]:
y.append(1)
else:
y.append(0)
y = np.array(y)
model.fit(x, y, epochs=10)
model_path = os.path.join(
tempfile.gettempdir(),
'test_accumulation_%f.h5' % np.random.random())
model.save(model_path)
from tensorflow.python.keras.utils.generic_utils import CustomObjectScope
with CustomObjectScope({
'RAdam': RAdam
}): # Workaround for incorrect global variable used in keras
keras.models.load_model(model_path,
custom_objects={'RAdam': RAdam})
def test_layer_end_to_end_with_adapt(self, vocab_data, input_data, kwargs,
use_dataset, expected_output,
input_dtype):
cls = get_layer_class()
if "invert" in kwargs and kwargs["invert"]:
expected_output_dtype = kwargs["dtype"]
else:
expected_output_dtype = dtypes.int64
input_shape = input_data.shape
if use_dataset:
# Keras APIs expect batched datasets.
# TODO(rachelim): `model.predict` predicts the result on each
# dataset batch separately, then tries to concatenate the results
# together. When the results have different shapes on the non-concat
# axis (which can happen in the output_mode = INT case for
# IndexLookup), the concatenation fails. In real use cases, this may
# not be an issue because users are likely to pipe the preprocessing layer
# into other keras layers instead of predicting it directly. A workaround
# for these unit tests is to have the dataset only contain one batch, so
# no concatenation needs to happen with the result. For consistency with
# numpy input, we should make `predict` join differently shaped results
# together sensibly, with 0 padding.
input_data = dataset_ops.Dataset.from_tensor_slices(input_data).batch(
input_shape[0])
vocab_data = dataset_ops.Dataset.from_tensor_slices(vocab_data).batch(
input_shape[0])
with CustomObjectScope({"IndexLookup": cls}):
output_data = testing_utils.layer_test(
cls,
kwargs=kwargs,
input_shape=input_shape,
input_data=input_data,
input_dtype=input_dtype,
expected_output_dtype=expected_output_dtype,
validate_training=False,
adapt_data=vocab_data)
if "invert" in kwargs and kwargs["invert"]:
self.assertAllEqual(expected_output, output_data)
else:
self.assertAllClose(expected_output, output_data)
def test_layer_end_to_end_with_adapt(self, vocab_data, input_data, kwargs,
expected_output):
cls = get_layer_class()
if kwargs.get("output_mode") == text_vectorization.TFIDF:
expected_output_dtype = dtypes.float32
else:
expected_output_dtype = dtypes.int64
with CustomObjectScope({"TextVectorization": cls}):
output_data = testing_utils.layer_test(
cls,
kwargs=kwargs,
input_shape=(None),
input_data=input_data,
input_dtype=dtypes.string,
expected_output_dtype=expected_output_dtype,
validate_training=False,
adapt_data=vocab_data)
self.assertAllClose(expected_output, output_data)
def test_layer_api_compatibility(self):
input_array = tf.constant([[1.0, 2.0, 3.0], [1.0, 2.0, 3.0],
[2.0, 3.0, 5.0]])
cls = affine_transform.AffineTransform
with CustomObjectScope({cls.__name__: cls}):
output = testing_utils.layer_test(
cls,
kwargs={
'output_size':
1,
'initializer':
tf.keras.initializers.TruncatedNormal(stddev=0.02)
},
input_shape=(None),
input_data=input_array)
expected_values = tf.constant([[0.01368301], [0.01368301],
[0.0314441]])
self.assertAllClose(expected_values, output)
def _test_fit(self, optmizer):
x = np.random.standard_normal((1000, 5))
y = np.dot(x, np.random.standard_normal((5, 2))).argmax(axis=-1)
model = keras.models.Sequential()
model.add(
keras.layers.Dense(
units=2,
input_shape=(5, ),
kernel_constraint=keras.constraints.MaxNorm(1000.0),
activation='softmax',
))
model.compile(
optimizer=optmizer,
loss='sparse_categorical_crossentropy',
)
model.fit(
x,
y,
batch_size=10,
epochs=110,
callbacks=[
keras.callbacks.EarlyStopping(monitor='loss',
min_delta=1e-4,
patience=3)
],
)
model_path = os.path.join(tempfile.gettempdir(),
'keras_warmup_%f.h5' % np.random.random())
model.save(model_path)
from tensorflow.python.keras.utils.generic_utils import CustomObjectScope
with CustomObjectScope({
'AdamWarmup': AdamWarmup
}): # Workaround for incorrect global variable used in keras
model = keras.models.load_model(
model_path, custom_objects={'AdamWarmup': AdamWarmup})
results = model.predict(x).argmax(axis=-1)
diff = np.sum(np.abs(y - results))
self.assertLess(diff, 100)