def test_linear_model_with_feature_column(self):
vocab_list = ['alpha', 'beta', 'gamma']
vocab_val = [0.4, 0.6, 0.9]
data = np.random.choice(vocab_list, size=256)
y = np.zeros_like(data, dtype=np.float32)
for vocab, val in zip(vocab_list, vocab_val):
indices = np.where(data == vocab)
y[indices] = val + np.random.uniform(
low=-0.01, high=0.01, size=indices[0].shape)
cat_column = tf.feature_column.categorical_column_with_vocabulary_list(
key='symbol', vocabulary_list=vocab_list)
ind_column = tf.feature_column.indicator_column(cat_column)
dense_feature_layer = dense_features_v2.DenseFeatures([ind_column])
linear_model = linear.LinearModel(use_bias=False,
kernel_initializer='zeros')
combined = sequential.Sequential([dense_feature_layer, linear_model])
opt = gradient_descent.SGD(learning_rate=0.1)
combined.compile(opt, 'mse', [])
combined.fit(x={'symbol': data}, y=y, batch_size=32, epochs=10)
self.assertAllClose([[0.4], [0.6], [0.9]],
combined.layers[1].dense_layers[0].kernel.numpy(),
atol=0.01)
def testIgnoreSaveCounter(self):
checkpoint_directory = self.get_temp_dir()
checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
with self.cached_session() as session:
# Create and save a model using Saver() before using a Checkpoint. This
# generates a snapshot without the Checkpoint's `save_counter`.
model = sequential.Sequential()
model.add(core.Flatten(input_shape=(1,)))
model.add(core.Dense(1))
name_saver = tf.compat.v1.train.Saver(model.trainable_variables)
save_path = name_saver.save(
sess=session, save_path=checkpoint_prefix, global_step=1)
# Checkpoint.restore must successfully load that checkpoint.
ckpt = tf.train.Checkpoint(model=model)
status = ckpt.restore(save_path)
status.assert_existing_objects_matched()
# It should, however, refuse to load a checkpoint where an unrelated
# `save_counter` variable is missing.
model.layers[1].var = tf.Variable(0., name="save_counter")
status = ckpt.restore(save_path)
with self.assertRaises(AssertionError):
status.assert_existing_objects_matched()
def test_wide_deep_model_as_layer(self):
linear_model = linear.LinearModel(units=1)
dnn_model = sequential.Sequential([core.Dense(units=1)])
linear_input = input_layer.Input(shape=(3, ), name='linear')
dnn_input = input_layer.Input(shape=(5, ), name='dnn')
wide_deep_model = wide_deep.WideDeepModel(linear_model, dnn_model)
wide_deep_output = wide_deep_model((linear_input, dnn_input))
input_b = input_layer.Input(shape=(1, ), name='b')
output_b = core.Dense(units=1)(input_b)
model = training.Model(inputs=[linear_input, dnn_input, input_b],
outputs=[wide_deep_output + output_b])
linear_input_np = np.random.uniform(low=-5, high=5, size=(64, 3))
dnn_input_np = np.random.uniform(low=-5, high=5, size=(64, 5))
input_b_np = np.random.uniform(low=-5, high=5, size=(64, ))
output_np = linear_input_np[:,
0] + .2 * dnn_input_np[:, 1] + input_b_np
model.compile(optimizer='sgd',
loss='mse',
metrics=[],
run_eagerly=testing_utils.should_run_eagerly())
model.fit([linear_input_np, dnn_input_np, input_b_np],
output_np,
epochs=5)
def mnist_model(input_shape):
"""Creates a MNIST model."""
model = sequential_model_lib.Sequential()
# Adding custom pass-through layer to visualize input images.
model.add(LayerForImageSummary())
model.add(
conv_layer_lib.Conv2D(32,
kernel_size=(3, 3),
activation='relu',
input_shape=input_shape))
model.add(conv_layer_lib.Conv2D(64, (3, 3), activation='relu'))
model.add(pool_layer_lib.MaxPooling2D(pool_size=(2, 2)))
model.add(layer_lib.Dropout(0.25))
model.add(layer_lib.Flatten())
model.add(layer_lib.Dense(128, activation='relu'))
model.add(layer_lib.Dropout(0.5))
model.add(layer_lib.Dense(NUM_CLASSES, activation='softmax'))
# Adding custom pass-through layer for summary recording.
model.add(LayerForHistogramSummary())
return model
def _model_compile(self,
steps_per_execution=1,
run_eagerly=False,
with_normalization_layer=False):
class ResultAssertingCallback(callbacks_lib.Callback):
def __init__(self):
self._prev_epoch = -1
def on_epoch_end(self, epoch, logs=None):
logging.info("testModelFit: epoch=%r, logs=%r", epoch, logs)
if epoch <= self._prev_epoch:
raise RuntimeError(
"Epoch is supposed to be larger than previous.")
self._prev_epoch = epoch
if (logs.get("loss", None) is None
or not isinstance(logs["loss"], np.floating)):
raise RuntimeError(
"loss is supposed to be in the logs and float.")
strategy = tf.distribute.experimental.ParameterServerStrategy(
make_cluster(3, 2),
variable_partitioner=tf.distribute.experimental.partitioners.
FixedShardsPartitioner(2))
with strategy.scope():
model = sequential.Sequential([core_layers.Dense(10)])
if with_normalization_layer:
norm = keras.layers.BatchNormalization(axis=-1,
input_shape=(4, 4, 3),
momentum=0.8)
model.add(norm)
model.compile(gradient_descent.SGD(),
loss="mse",
steps_per_execution=steps_per_execution,
run_eagerly=run_eagerly)
return model, [ResultAssertingCallback()]
def test_dataset_creator_input_options(self):
dataset_fn = lambda _: tf.data.Dataset.from_tensor_slices([1, 1])
input_options = tf.distribute.InputOptions(
experimental_fetch_to_device=True,
experimental_per_replica_buffer_size=2,
)
x = dataset_creator.DatasetCreator(dataset_fn,
input_options=input_options)
with tf.distribute.MultiWorkerMirroredStrategy().scope():
data_handler = data_adapter.get_data_handler(
x,
steps_per_epoch=2,
model=sequential.Sequential([core_layers.Dense(10)]),
)
# Ensuring the resulting `DistributedDatasetsFromFunction` has the right
# options.
self.assertTrue(
data_handler._dataset._options.experimental_fetch_to_device)
self.assertEqual(
data_handler._dataset._options.
experimental_per_replica_buffer_size,
2,
)
def test_TensorBoard(self):
np.random.seed(1337)
temp_dir = self.get_temp_dir()
self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
(x_train, y_train), (x_test, y_test) = test_utils.get_test_data(
train_samples=TRAIN_SAMPLES,
test_samples=TEST_SAMPLES,
input_shape=(INPUT_DIM, ),
num_classes=NUM_CLASSES,
)
y_test = np_utils.to_categorical(y_test)
y_train = np_utils.to_categorical(y_train)
def data_generator(train):
if train:
max_batch_index = len(x_train) // BATCH_SIZE
else:
max_batch_index = len(x_test) // BATCH_SIZE
i = 0
while 1:
if train:
yield (
x_train[i * BATCH_SIZE:(i + 1) * BATCH_SIZE],
y_train[i * BATCH_SIZE:(i + 1) * BATCH_SIZE],
)
else:
yield (
x_test[i * BATCH_SIZE:(i + 1) * BATCH_SIZE],
y_test[i * BATCH_SIZE:(i + 1) * BATCH_SIZE],
)
i += 1
i %= max_batch_index
# case: Sequential
with tf.Graph().as_default(), self.cached_session():
model = sequential.Sequential()
model.add(
layers.Dense(NUM_HIDDEN,
input_dim=INPUT_DIM,
activation="relu"))
# non_trainable_weights: moving_variance, moving_mean
model.add(layers.BatchNormalization())
model.add(layers.Dense(NUM_CLASSES, activation="softmax"))
model.compile(
loss="categorical_crossentropy",
optimizer="sgd",
metrics=["accuracy"],
)
tsb = callbacks_v1.TensorBoard(
log_dir=temp_dir,
histogram_freq=1,
write_images=True,
write_grads=True,
batch_size=5,
)
cbks = [tsb]
# fit with validation data
model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=3,
verbose=0,
)
# fit with validation data and accuracy
model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=2,
verbose=0,
)
# fit generator with validation data
model.fit_generator(
data_generator(True),
len(x_train),
epochs=2,
validation_data=(x_test, y_test),
callbacks=cbks,
verbose=0,
)
# fit generator without validation data
# histogram_freq must be zero
tsb.histogram_freq = 0
model.fit_generator(
data_generator(True),
len(x_train),
epochs=2,
callbacks=cbks,
verbose=0,
)
# fit generator with validation data and accuracy
tsb.histogram_freq = 1
model.fit_generator(
data_generator(True),
len(x_train),
epochs=2,
validation_data=(x_test, y_test),
callbacks=cbks,
verbose=0,
)
# fit generator without validation data and accuracy
tsb.histogram_freq = 0
model.fit_generator(data_generator(True),
len(x_train),
epochs=2,
callbacks=cbks)
assert os.path.exists(temp_dir)
def test_Tensorboard_histogram_summaries_in_test_function(self):
class FileWriterStub:
def __init__(self, logdir, graph=None):
self.logdir = logdir
self.graph = graph
self.steps_seen = []
def add_summary(self, summary, global_step):
summary_obj = tf.compat.v1.Summary()
# ensure a valid Summary proto is being sent
if isinstance(summary, bytes):
summary_obj.ParseFromString(summary)
else:
assert isinstance(summary, tf.compat.v1.Summary)
summary_obj = summary
# keep track of steps seen for the merged_summary op,
# which contains the histogram summaries
if len(summary_obj.value) > 1:
self.steps_seen.append(global_step)
def flush(self):
pass
def close(self):
pass
def _init_writer(obj, _):
obj.writer = FileWriterStub(obj.log_dir)
np.random.seed(1337)
tmpdir = self.get_temp_dir()
self.addCleanup(shutil.rmtree, tmpdir, ignore_errors=True)
(x_train, y_train), (x_test, y_test) = test_utils.get_test_data(
train_samples=TRAIN_SAMPLES,
test_samples=TEST_SAMPLES,
input_shape=(INPUT_DIM, ),
num_classes=NUM_CLASSES,
)
y_test = np_utils.to_categorical(y_test)
y_train = np_utils.to_categorical(y_train)
with tf.Graph().as_default(), self.cached_session():
model = sequential.Sequential()
model.add(
layers.Dense(NUM_HIDDEN,
input_dim=INPUT_DIM,
activation="relu"))
# non_trainable_weights: moving_variance, moving_mean
model.add(layers.BatchNormalization())
model.add(layers.Dense(NUM_CLASSES, activation="softmax"))
model.compile(
loss="categorical_crossentropy",
optimizer="sgd",
metrics=["accuracy"],
)
callbacks_v1.TensorBoard._init_writer = _init_writer
tsb = callbacks_v1.TensorBoard(
log_dir=tmpdir,
histogram_freq=1,
write_images=True,
write_grads=True,
batch_size=5,
)
cbks = [tsb]
# fit with validation data
model.fit(
x_train,
y_train,
batch_size=BATCH_SIZE,
validation_data=(x_test, y_test),
callbacks=cbks,
epochs=3,
verbose=0,
)
self.assertAllEqual(tsb.writer.steps_seen, [0, 1, 2, 3, 4, 5])
def _model_compile(self,
strategy,
steps_per_execution=1,
run_eagerly=False,
with_normalization_layer=False):
class ResultAssertingCallback(callbacks_lib.Callback):
def __init__(self):
self._prev_epoch = -1
self._loss_to_compare_against = 2 # Empirical initial value
def on_epoch_end(self, epoch, logs=None):
logging.info("testModelFit: epoch=%r, logs=%r", epoch, logs)
if epoch <= self._prev_epoch:
raise RuntimeError(
"Epoch is supposed to be larger than previous.")
self._prev_epoch = epoch
is_loss_float = (logs.get("loss", None) is not None
and isinstance(logs["loss"],
(float, np.floating)))
if not is_loss_float:
raise RuntimeError(
"loss is supposed to be in the logs and float.")
if epoch == 0 or epoch == 9:
# Making sure the loss of first epoch is below 1, and that of last
# epoch is smaller than the first epoch.
if logs["loss"] > self._loss_to_compare_against:
raise RuntimeError(
"loss at epoch {} is larger than previous.".format(
epoch))
self._loss_to_compare_against = logs["loss"]
def on_train_end(self, logs=None):
if self._prev_epoch != 9:
raise RuntimeError("Unexpected last epoch: {}".format(
self._prev_epoch))
# TODO(b/182193218): Use ParameterServerStrategy as a proper strategy
# combination.
if strategy == "ParameterServerStrategy":
gpu_devices = tf.config.list_physical_devices("GPU")
if len(gpu_devices) > 1:
self.skipTest("b/178452835: Multi-GPUs not supported in "
"ParameterServerStrategy.")
strategy = tf.distribute.experimental.ParameterServerStrategy(
multi_worker_testing_utils.make_parameter_server_cluster(3, 2),
variable_partitioner=tf.distribute.experimental.partitioners.
FixedShardsPartitioner(2))
with strategy.scope():
model = sequential.Sequential([core_layers.Dense(10)])
if with_normalization_layer:
norm = keras.layers.BatchNormalization(axis=-1,
input_shape=(4, 4, 3),
momentum=0.8)
model.add(norm)
model.compile(gradient_descent.SGD(),
loss="mse",
steps_per_execution=steps_per_execution,
run_eagerly=run_eagerly)
return model, [ResultAssertingCallback()]
def directory_iterator_with_validation_split_test_helper(
self, validation_split):
if PIL is None:
return # Skip test if PIL is not available.
num_classes = 2
tmp_folder = tempfile.mkdtemp(prefix='test_images')
# create folders and subfolders
paths = []
for cl in range(num_classes):
class_directory = 'class-{}'.format(cl)
classpaths = [
class_directory,
os.path.join(class_directory, 'subfolder-1'),
os.path.join(class_directory, 'subfolder-2'),
os.path.join(class_directory, 'subfolder-1', 'sub-subfolder')
]
for path in classpaths:
os.mkdir(os.path.join(tmp_folder, path))
paths.append(classpaths)
# save the images in the paths
count = 0
filenames = []
for test_images in _generate_test_images():
for im in test_images:
# rotate image class
im_class = count % num_classes
# rotate subfolders
classpaths = paths[im_class]
filename = os.path.join(classpaths[count % len(classpaths)],
'image-{}.jpg'.format(count))
filenames.append(filename)
im.save(os.path.join(tmp_folder, filename))
count += 1
# create iterator
generator = preprocessing_image.ImageDataGenerator(
validation_split=validation_split)
with self.assertRaises(ValueError):
generator.flow_from_directory(tmp_folder, subset='foo')
num_validation = int(count * validation_split)
num_training = count - num_validation
train_iterator = generator.flow_from_directory(tmp_folder,
subset='training')
self.assertEqual(train_iterator.samples, num_training)
valid_iterator = generator.flow_from_directory(tmp_folder,
subset='validation')
self.assertEqual(valid_iterator.samples, num_validation)
# check number of classes and images
self.assertEqual(len(train_iterator.class_indices), num_classes)
self.assertEqual(len(train_iterator.classes), num_training)
self.assertEqual(len(set(train_iterator.filenames) & set(filenames)),
num_training)
model = sequential.Sequential([layers.Flatten(), layers.Dense(2)])
model.compile(optimizer='sgd', loss='mse')
model.fit(train_iterator, epochs=1)
shutil.rmtree(tmp_folder)
def ConvNeXt(
depths,
projection_dims,
drop_path_rate=0.0,
layer_scale_init_value=1e-6,
default_size=224,
model_name="convnext",
include_preprocessing=True,
include_top=True,
weights=None,
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
classifier_activation="softmax",
):
"""Instantiates ConvNeXt architecture given specific configuration.
Args:
depths: An iterable containing depths for each individual stages.
projection_dims: An iterable containing output number of channels of
each individual stages.
drop_path_rate: Stochastic depth probability. If 0.0, then stochastic
depth won't be used.
layer_scale_init_value: Layer scale coefficient. If 0.0, layer scaling
won't be used.
default_size: Default input image size.
model_name: An optional name for the model.
include_preprocessing: boolean denoting whther to include preprocessing in
the model. When `weights="imagenet"` this should be always set to True.
But for other models (e.g., randomly initialized) users should set it
to False and apply preprocessing to data accordingly.
include_top: Boolean denoting whether to include classification head to
the model.
weights: one of `None` (random initialization), `"imagenet"` (pre-training
on ImageNet-1k), or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to
use as image input for the model.
input_shape: optional shape tuple, only to be specified if `include_top`
is False. It should have exactly 3 inputs channels.
pooling: optional pooling mode for feature extraction when `include_top`
is `False`.
- `None` means that the output of the model will be the 4D tensor output
of the last convolutional layer.
- `avg` means that global average pooling will be applied to the output
of the last convolutional layer, and thus the output of the model will
be a 2D tensor.
- `max` means that global max pooling will be applied.
classes: optional number of classes to classify images into, only to be
specified if `include_top` is True, and if no `weights` argument is
specified.
classifier_activation: A `str` or callable. The activation function to use
on the "top" layer. Ignored unless `include_top=True`. Set
`classifier_activation=None` to return the logits of the "top" layer.
Returns:
A `keras.Model` instance.
Raises:
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
ValueError: if `classifier_activation` is not `softmax`, or `None`
when using a pretrained top layer.
ValueError: if `include_top` is True but `num_classes` is not 1000
when using ImageNet.
"""
if not (weights in {"imagenet", None} or tf.io.gfile.exists(weights)):
raise ValueError(
"The `weights` argument should be either "
"`None` (random initialization), `imagenet` "
"(pre-training on ImageNet), "
"or the path to the weights file to be loaded."
)
if weights == "imagenet" and include_top and classes != 1000:
raise ValueError(
"If using `weights` as `'imagenet'` with `include_top`"
" as true, `classes` should be 1000"
)
# Determine proper input shape.
input_shape = imagenet_utils.obtain_input_shape(
input_shape,
default_size=default_size,
min_size=32,
data_format=backend.image_data_format(),
require_flatten=include_top,
weights=weights,
)
if input_tensor is None:
img_input = layers.Input(shape=input_shape)
else:
if not backend.is_keras_tensor(input_tensor):
img_input = layers.Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if input_tensor is not None:
inputs = utils.layer_utils.get_source_inputs(input_tensor)
else:
inputs = img_input
x = inputs
if include_preprocessing:
channel_axis = (
3 if backend.image_data_format() == "channels_last" else 1
)
num_channels = input_shape[channel_axis - 1]
if num_channels == 3:
x = PreStem(name=model_name)(x)
# Stem block.
stem = sequential.Sequential(
[
layers.Conv2D(
projection_dims[0],
kernel_size=4,
strides=4,
name=model_name + "_stem_conv",
),
layers.LayerNormalization(
epsilon=1e-6, name=model_name + "_stem_layernorm"
),
],
name=model_name + "_stem",
)
# Downsampling blocks.
downsample_layers = []
downsample_layers.append(stem)
num_downsample_layers = 3
for i in range(num_downsample_layers):
downsample_layer = sequential.Sequential(
[
layers.LayerNormalization(
epsilon=1e-6,
name=model_name + "_downsampling_layernorm_" + str(i),
),
layers.Conv2D(
projection_dims[i + 1],
kernel_size=2,
strides=2,
name=model_name + "_downsampling_conv_" + str(i),
),
],
name=model_name + "_downsampling_block_" + str(i),
)
downsample_layers.append(downsample_layer)
# Stochastic depth schedule.
# This is referred from the original ConvNeXt codebase:
# https://github.com/facebookresearch/ConvNeXt/blob/main/models/convnext.py#L86
depth_drop_rates = [
float(x) for x in np.linspace(0.0, drop_path_rate, sum(depths))
]
# First apply downsampling blocks and then apply ConvNeXt stages.
cur = 0
num_convnext_blocks = 4
for i in range(num_convnext_blocks):
x = downsample_layers[i](x)
for j in range(depths[i]):
x = ConvNeXtBlock(
projection_dim=projection_dims[i],
drop_path_rate=depth_drop_rates[cur + j],
layer_scale_init_value=layer_scale_init_value,
name=model_name + f"_stage_{i}_block_{j}",
)(x)
cur += depths[i]
if include_top:
x = Head(num_classes=classes, name=model_name)(x)
imagenet_utils.validate_activation(classifier_activation, weights)
else:
if pooling == "avg":
x = layers.GlobalAveragePooling2D()(x)
elif pooling == "max":
x = layers.GlobalMaxPooling2D()(x)
x = layers.LayerNormalization(epsilon=1e-6)(x)
model = training_lib.Model(inputs=inputs, outputs=x, name=model_name)
# Load weights.
if weights == "imagenet":
if include_top:
file_suffix = ".h5"
file_hash = WEIGHTS_HASHES[model_name][0]
else:
file_suffix = "_notop.h5"
file_hash = WEIGHTS_HASHES[model_name][1]
file_name = model_name + file_suffix
weights_path = utils.data_utils.get_file(
file_name,
BASE_WEIGHTS_PATH + file_name,
cache_subdir="models",
file_hash=file_hash,
)
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
return model
