def test_linear_model_with_feature_column(self): vocab_list = ['alpha', 'beta', 'gamma'] vocab_val = [0.4, 0.6, 0.9] data = np.random.choice(vocab_list, size=256) y = np.zeros_like(data, dtype=np.float32) for vocab, val in zip(vocab_list, vocab_val): indices = np.where(data == vocab) y[indices] = val + np.random.uniform( low=-0.01, high=0.01, size=indices[0].shape) cat_column = tf.feature_column.categorical_column_with_vocabulary_list( key='symbol', vocabulary_list=vocab_list) ind_column = tf.feature_column.indicator_column(cat_column) dense_feature_layer = dense_features_v2.DenseFeatures([ind_column]) linear_model = linear.LinearModel(use_bias=False, kernel_initializer='zeros') combined = sequential.Sequential([dense_feature_layer, linear_model]) opt = gradient_descent.SGD(learning_rate=0.1) combined.compile(opt, 'mse', []) combined.fit(x={'symbol': data}, y=y, batch_size=32, epochs=10) self.assertAllClose([[0.4], [0.6], [0.9]], combined.layers[1].dense_layers[0].kernel.numpy(), atol=0.01)
def testIgnoreSaveCounter(self): checkpoint_directory = self.get_temp_dir() checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") with self.cached_session() as session: # Create and save a model using Saver() before using a Checkpoint. This # generates a snapshot without the Checkpoint's `save_counter`. model = sequential.Sequential() model.add(core.Flatten(input_shape=(1,))) model.add(core.Dense(1)) name_saver = tf.compat.v1.train.Saver(model.trainable_variables) save_path = name_saver.save( sess=session, save_path=checkpoint_prefix, global_step=1) # Checkpoint.restore must successfully load that checkpoint. ckpt = tf.train.Checkpoint(model=model) status = ckpt.restore(save_path) status.assert_existing_objects_matched() # It should, however, refuse to load a checkpoint where an unrelated # `save_counter` variable is missing. model.layers[1].var = tf.Variable(0., name="save_counter") status = ckpt.restore(save_path) with self.assertRaises(AssertionError): status.assert_existing_objects_matched()
def test_wide_deep_model_as_layer(self): linear_model = linear.LinearModel(units=1) dnn_model = sequential.Sequential([core.Dense(units=1)]) linear_input = input_layer.Input(shape=(3, ), name='linear') dnn_input = input_layer.Input(shape=(5, ), name='dnn') wide_deep_model = wide_deep.WideDeepModel(linear_model, dnn_model) wide_deep_output = wide_deep_model((linear_input, dnn_input)) input_b = input_layer.Input(shape=(1, ), name='b') output_b = core.Dense(units=1)(input_b) model = training.Model(inputs=[linear_input, dnn_input, input_b], outputs=[wide_deep_output + output_b]) linear_input_np = np.random.uniform(low=-5, high=5, size=(64, 3)) dnn_input_np = np.random.uniform(low=-5, high=5, size=(64, 5)) input_b_np = np.random.uniform(low=-5, high=5, size=(64, )) output_np = linear_input_np[:, 0] + .2 * dnn_input_np[:, 1] + input_b_np model.compile(optimizer='sgd', loss='mse', metrics=[], run_eagerly=testing_utils.should_run_eagerly()) model.fit([linear_input_np, dnn_input_np, input_b_np], output_np, epochs=5)
def mnist_model(input_shape): """Creates a MNIST model.""" model = sequential_model_lib.Sequential() # Adding custom pass-through layer to visualize input images. model.add(LayerForImageSummary()) model.add( conv_layer_lib.Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) model.add(conv_layer_lib.Conv2D(64, (3, 3), activation='relu')) model.add(pool_layer_lib.MaxPooling2D(pool_size=(2, 2))) model.add(layer_lib.Dropout(0.25)) model.add(layer_lib.Flatten()) model.add(layer_lib.Dense(128, activation='relu')) model.add(layer_lib.Dropout(0.5)) model.add(layer_lib.Dense(NUM_CLASSES, activation='softmax')) # Adding custom pass-through layer for summary recording. model.add(LayerForHistogramSummary()) return model
def _model_compile(self, steps_per_execution=1, run_eagerly=False, with_normalization_layer=False): class ResultAssertingCallback(callbacks_lib.Callback): def __init__(self): self._prev_epoch = -1 def on_epoch_end(self, epoch, logs=None): logging.info("testModelFit: epoch=%r, logs=%r", epoch, logs) if epoch <= self._prev_epoch: raise RuntimeError( "Epoch is supposed to be larger than previous.") self._prev_epoch = epoch if (logs.get("loss", None) is None or not isinstance(logs["loss"], np.floating)): raise RuntimeError( "loss is supposed to be in the logs and float.") strategy = tf.distribute.experimental.ParameterServerStrategy( make_cluster(3, 2), variable_partitioner=tf.distribute.experimental.partitioners. FixedShardsPartitioner(2)) with strategy.scope(): model = sequential.Sequential([core_layers.Dense(10)]) if with_normalization_layer: norm = keras.layers.BatchNormalization(axis=-1, input_shape=(4, 4, 3), momentum=0.8) model.add(norm) model.compile(gradient_descent.SGD(), loss="mse", steps_per_execution=steps_per_execution, run_eagerly=run_eagerly) return model, [ResultAssertingCallback()]
def test_dataset_creator_input_options(self): dataset_fn = lambda _: tf.data.Dataset.from_tensor_slices([1, 1]) input_options = tf.distribute.InputOptions( experimental_fetch_to_device=True, experimental_per_replica_buffer_size=2, ) x = dataset_creator.DatasetCreator(dataset_fn, input_options=input_options) with tf.distribute.MultiWorkerMirroredStrategy().scope(): data_handler = data_adapter.get_data_handler( x, steps_per_epoch=2, model=sequential.Sequential([core_layers.Dense(10)]), ) # Ensuring the resulting `DistributedDatasetsFromFunction` has the right # options. self.assertTrue( data_handler._dataset._options.experimental_fetch_to_device) self.assertEqual( data_handler._dataset._options. experimental_per_replica_buffer_size, 2, )
def test_TensorBoard(self): np.random.seed(1337) temp_dir = self.get_temp_dir() self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True) (x_train, y_train), (x_test, y_test) = test_utils.get_test_data( train_samples=TRAIN_SAMPLES, test_samples=TEST_SAMPLES, input_shape=(INPUT_DIM, ), num_classes=NUM_CLASSES, ) y_test = np_utils.to_categorical(y_test) y_train = np_utils.to_categorical(y_train) def data_generator(train): if train: max_batch_index = len(x_train) // BATCH_SIZE else: max_batch_index = len(x_test) // BATCH_SIZE i = 0 while 1: if train: yield ( x_train[i * BATCH_SIZE:(i + 1) * BATCH_SIZE], y_train[i * BATCH_SIZE:(i + 1) * BATCH_SIZE], ) else: yield ( x_test[i * BATCH_SIZE:(i + 1) * BATCH_SIZE], y_test[i * BATCH_SIZE:(i + 1) * BATCH_SIZE], ) i += 1 i %= max_batch_index # case: Sequential with tf.Graph().as_default(), self.cached_session(): model = sequential.Sequential() model.add( layers.Dense(NUM_HIDDEN, input_dim=INPUT_DIM, activation="relu")) # non_trainable_weights: moving_variance, moving_mean model.add(layers.BatchNormalization()) model.add(layers.Dense(NUM_CLASSES, activation="softmax")) model.compile( loss="categorical_crossentropy", optimizer="sgd", metrics=["accuracy"], ) tsb = callbacks_v1.TensorBoard( log_dir=temp_dir, histogram_freq=1, write_images=True, write_grads=True, batch_size=5, ) cbks = [tsb] # fit with validation data model.fit( x_train, y_train, batch_size=BATCH_SIZE, validation_data=(x_test, y_test), callbacks=cbks, epochs=3, verbose=0, ) # fit with validation data and accuracy model.fit( x_train, y_train, batch_size=BATCH_SIZE, validation_data=(x_test, y_test), callbacks=cbks, epochs=2, verbose=0, ) # fit generator with validation data model.fit_generator( data_generator(True), len(x_train), epochs=2, validation_data=(x_test, y_test), callbacks=cbks, verbose=0, ) # fit generator without validation data # histogram_freq must be zero tsb.histogram_freq = 0 model.fit_generator( data_generator(True), len(x_train), epochs=2, callbacks=cbks, verbose=0, ) # fit generator with validation data and accuracy tsb.histogram_freq = 1 model.fit_generator( data_generator(True), len(x_train), epochs=2, validation_data=(x_test, y_test), callbacks=cbks, verbose=0, ) # fit generator without validation data and accuracy tsb.histogram_freq = 0 model.fit_generator(data_generator(True), len(x_train), epochs=2, callbacks=cbks) assert os.path.exists(temp_dir)
def test_Tensorboard_histogram_summaries_in_test_function(self): class FileWriterStub: def __init__(self, logdir, graph=None): self.logdir = logdir self.graph = graph self.steps_seen = [] def add_summary(self, summary, global_step): summary_obj = tf.compat.v1.Summary() # ensure a valid Summary proto is being sent if isinstance(summary, bytes): summary_obj.ParseFromString(summary) else: assert isinstance(summary, tf.compat.v1.Summary) summary_obj = summary # keep track of steps seen for the merged_summary op, # which contains the histogram summaries if len(summary_obj.value) > 1: self.steps_seen.append(global_step) def flush(self): pass def close(self): pass def _init_writer(obj, _): obj.writer = FileWriterStub(obj.log_dir) np.random.seed(1337) tmpdir = self.get_temp_dir() self.addCleanup(shutil.rmtree, tmpdir, ignore_errors=True) (x_train, y_train), (x_test, y_test) = test_utils.get_test_data( train_samples=TRAIN_SAMPLES, test_samples=TEST_SAMPLES, input_shape=(INPUT_DIM, ), num_classes=NUM_CLASSES, ) y_test = np_utils.to_categorical(y_test) y_train = np_utils.to_categorical(y_train) with tf.Graph().as_default(), self.cached_session(): model = sequential.Sequential() model.add( layers.Dense(NUM_HIDDEN, input_dim=INPUT_DIM, activation="relu")) # non_trainable_weights: moving_variance, moving_mean model.add(layers.BatchNormalization()) model.add(layers.Dense(NUM_CLASSES, activation="softmax")) model.compile( loss="categorical_crossentropy", optimizer="sgd", metrics=["accuracy"], ) callbacks_v1.TensorBoard._init_writer = _init_writer tsb = callbacks_v1.TensorBoard( log_dir=tmpdir, histogram_freq=1, write_images=True, write_grads=True, batch_size=5, ) cbks = [tsb] # fit with validation data model.fit( x_train, y_train, batch_size=BATCH_SIZE, validation_data=(x_test, y_test), callbacks=cbks, epochs=3, verbose=0, ) self.assertAllEqual(tsb.writer.steps_seen, [0, 1, 2, 3, 4, 5])
def _model_compile(self, strategy, steps_per_execution=1, run_eagerly=False, with_normalization_layer=False): class ResultAssertingCallback(callbacks_lib.Callback): def __init__(self): self._prev_epoch = -1 self._loss_to_compare_against = 2 # Empirical initial value def on_epoch_end(self, epoch, logs=None): logging.info("testModelFit: epoch=%r, logs=%r", epoch, logs) if epoch <= self._prev_epoch: raise RuntimeError( "Epoch is supposed to be larger than previous.") self._prev_epoch = epoch is_loss_float = (logs.get("loss", None) is not None and isinstance(logs["loss"], (float, np.floating))) if not is_loss_float: raise RuntimeError( "loss is supposed to be in the logs and float.") if epoch == 0 or epoch == 9: # Making sure the loss of first epoch is below 1, and that of last # epoch is smaller than the first epoch. if logs["loss"] > self._loss_to_compare_against: raise RuntimeError( "loss at epoch {} is larger than previous.".format( epoch)) self._loss_to_compare_against = logs["loss"] def on_train_end(self, logs=None): if self._prev_epoch != 9: raise RuntimeError("Unexpected last epoch: {}".format( self._prev_epoch)) # TODO(b/182193218): Use ParameterServerStrategy as a proper strategy # combination. if strategy == "ParameterServerStrategy": gpu_devices = tf.config.list_physical_devices("GPU") if len(gpu_devices) > 1: self.skipTest("b/178452835: Multi-GPUs not supported in " "ParameterServerStrategy.") strategy = tf.distribute.experimental.ParameterServerStrategy( multi_worker_testing_utils.make_parameter_server_cluster(3, 2), variable_partitioner=tf.distribute.experimental.partitioners. FixedShardsPartitioner(2)) with strategy.scope(): model = sequential.Sequential([core_layers.Dense(10)]) if with_normalization_layer: norm = keras.layers.BatchNormalization(axis=-1, input_shape=(4, 4, 3), momentum=0.8) model.add(norm) model.compile(gradient_descent.SGD(), loss="mse", steps_per_execution=steps_per_execution, run_eagerly=run_eagerly) return model, [ResultAssertingCallback()]
def directory_iterator_with_validation_split_test_helper( self, validation_split): if PIL is None: return # Skip test if PIL is not available. num_classes = 2 tmp_folder = tempfile.mkdtemp(prefix='test_images') # create folders and subfolders paths = [] for cl in range(num_classes): class_directory = 'class-{}'.format(cl) classpaths = [ class_directory, os.path.join(class_directory, 'subfolder-1'), os.path.join(class_directory, 'subfolder-2'), os.path.join(class_directory, 'subfolder-1', 'sub-subfolder') ] for path in classpaths: os.mkdir(os.path.join(tmp_folder, path)) paths.append(classpaths) # save the images in the paths count = 0 filenames = [] for test_images in _generate_test_images(): for im in test_images: # rotate image class im_class = count % num_classes # rotate subfolders classpaths = paths[im_class] filename = os.path.join(classpaths[count % len(classpaths)], 'image-{}.jpg'.format(count)) filenames.append(filename) im.save(os.path.join(tmp_folder, filename)) count += 1 # create iterator generator = preprocessing_image.ImageDataGenerator( validation_split=validation_split) with self.assertRaises(ValueError): generator.flow_from_directory(tmp_folder, subset='foo') num_validation = int(count * validation_split) num_training = count - num_validation train_iterator = generator.flow_from_directory(tmp_folder, subset='training') self.assertEqual(train_iterator.samples, num_training) valid_iterator = generator.flow_from_directory(tmp_folder, subset='validation') self.assertEqual(valid_iterator.samples, num_validation) # check number of classes and images self.assertEqual(len(train_iterator.class_indices), num_classes) self.assertEqual(len(train_iterator.classes), num_training) self.assertEqual(len(set(train_iterator.filenames) & set(filenames)), num_training) model = sequential.Sequential([layers.Flatten(), layers.Dense(2)]) model.compile(optimizer='sgd', loss='mse') model.fit(train_iterator, epochs=1) shutil.rmtree(tmp_folder)
def ConvNeXt( depths, projection_dims, drop_path_rate=0.0, layer_scale_init_value=1e-6, default_size=224, model_name="convnext", include_preprocessing=True, include_top=True, weights=None, input_tensor=None, input_shape=None, pooling=None, classes=1000, classifier_activation="softmax", ): """Instantiates ConvNeXt architecture given specific configuration. Args: depths: An iterable containing depths for each individual stages. projection_dims: An iterable containing output number of channels of each individual stages. drop_path_rate: Stochastic depth probability. If 0.0, then stochastic depth won't be used. layer_scale_init_value: Layer scale coefficient. If 0.0, layer scaling won't be used. default_size: Default input image size. model_name: An optional name for the model. include_preprocessing: boolean denoting whther to include preprocessing in the model. When `weights="imagenet"` this should be always set to True. But for other models (e.g., randomly initialized) users should set it to False and apply preprocessing to data accordingly. include_top: Boolean denoting whether to include classification head to the model. weights: one of `None` (random initialization), `"imagenet"` (pre-training on ImageNet-1k), or the path to the weights file to be loaded. input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: optional shape tuple, only to be specified if `include_top` is False. It should have exactly 3 inputs channels. pooling: optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling will be applied. classes: optional number of classes to classify images into, only to be specified if `include_top` is True, and if no `weights` argument is specified. classifier_activation: A `str` or callable. The activation function to use on the "top" layer. Ignored unless `include_top=True`. Set `classifier_activation=None` to return the logits of the "top" layer. Returns: A `keras.Model` instance. Raises: ValueError: in case of invalid argument for `weights`, or invalid input shape. ValueError: if `classifier_activation` is not `softmax`, or `None` when using a pretrained top layer. ValueError: if `include_top` is True but `num_classes` is not 1000 when using ImageNet. """ if not (weights in {"imagenet", None} or tf.io.gfile.exists(weights)): raise ValueError( "The `weights` argument should be either " "`None` (random initialization), `imagenet` " "(pre-training on ImageNet), " "or the path to the weights file to be loaded." ) if weights == "imagenet" and include_top and classes != 1000: raise ValueError( "If using `weights` as `'imagenet'` with `include_top`" " as true, `classes` should be 1000" ) # Determine proper input shape. input_shape = imagenet_utils.obtain_input_shape( input_shape, default_size=default_size, min_size=32, data_format=backend.image_data_format(), require_flatten=include_top, weights=weights, ) if input_tensor is None: img_input = layers.Input(shape=input_shape) else: if not backend.is_keras_tensor(input_tensor): img_input = layers.Input(tensor=input_tensor, shape=input_shape) else: img_input = input_tensor if input_tensor is not None: inputs = utils.layer_utils.get_source_inputs(input_tensor) else: inputs = img_input x = inputs if include_preprocessing: channel_axis = ( 3 if backend.image_data_format() == "channels_last" else 1 ) num_channels = input_shape[channel_axis - 1] if num_channels == 3: x = PreStem(name=model_name)(x) # Stem block. stem = sequential.Sequential( [ layers.Conv2D( projection_dims[0], kernel_size=4, strides=4, name=model_name + "_stem_conv", ), layers.LayerNormalization( epsilon=1e-6, name=model_name + "_stem_layernorm" ), ], name=model_name + "_stem", ) # Downsampling blocks. downsample_layers = [] downsample_layers.append(stem) num_downsample_layers = 3 for i in range(num_downsample_layers): downsample_layer = sequential.Sequential( [ layers.LayerNormalization( epsilon=1e-6, name=model_name + "_downsampling_layernorm_" + str(i), ), layers.Conv2D( projection_dims[i + 1], kernel_size=2, strides=2, name=model_name + "_downsampling_conv_" + str(i), ), ], name=model_name + "_downsampling_block_" + str(i), ) downsample_layers.append(downsample_layer) # Stochastic depth schedule. # This is referred from the original ConvNeXt codebase: # https://github.com/facebookresearch/ConvNeXt/blob/main/models/convnext.py#L86 depth_drop_rates = [ float(x) for x in np.linspace(0.0, drop_path_rate, sum(depths)) ] # First apply downsampling blocks and then apply ConvNeXt stages. cur = 0 num_convnext_blocks = 4 for i in range(num_convnext_blocks): x = downsample_layers[i](x) for j in range(depths[i]): x = ConvNeXtBlock( projection_dim=projection_dims[i], drop_path_rate=depth_drop_rates[cur + j], layer_scale_init_value=layer_scale_init_value, name=model_name + f"_stage_{i}_block_{j}", )(x) cur += depths[i] if include_top: x = Head(num_classes=classes, name=model_name)(x) imagenet_utils.validate_activation(classifier_activation, weights) else: if pooling == "avg": x = layers.GlobalAveragePooling2D()(x) elif pooling == "max": x = layers.GlobalMaxPooling2D()(x) x = layers.LayerNormalization(epsilon=1e-6)(x) model = training_lib.Model(inputs=inputs, outputs=x, name=model_name) # Load weights. if weights == "imagenet": if include_top: file_suffix = ".h5" file_hash = WEIGHTS_HASHES[model_name][0] else: file_suffix = "_notop.h5" file_hash = WEIGHTS_HASHES[model_name][1] file_name = model_name + file_suffix weights_path = utils.data_utils.get_file( file_name, BASE_WEIGHTS_PATH + file_name, cache_subdir="models", file_hash=file_hash, ) model.load_weights(weights_path) elif weights is not None: model.load_weights(weights) return model