def test_build_ensemble_spec( self, want_logits, want_loss=None, want_adanet_loss=None, want_ensemble_trainable_vars=None, adanet_lambda=0., adanet_beta=0., ensemble_spec_fn=lambda: None, use_bias=False, use_logits_last_layer=False, mixture_weight_type=MixtureWeightType.MATRIX, mixture_weight_initializer=tf_compat.v1.zeros_initializer(), warm_start_mixture_weights=True, subnetwork_builder_class=_Builder, mode=tf.estimator.ModeKeys.TRAIN, multi_head=False, want_subnetwork_trainable_vars=2, ensembler_class=ComplexityRegularizedEnsembler, my_ensemble_index=None, want_replay_indices=None, want_predictions=None, export_subnetworks=False, previous_ensemble_spec=None, previous_iteration_checkpoint=None): seed = 64 if multi_head: head = multi_head_lib.MultiHead(heads=[ binary_class_head.BinaryClassHead( name="head1", loss_reduction=tf_compat.SUM), binary_class_head.BinaryClassHead(name="head2", loss_reduction=tf_compat.SUM) ]) else: head = binary_class_head.BinaryClassHead( loss_reduction=tf_compat.SUM) builder = _EnsembleBuilder( head=head, export_subnetwork_logits=export_subnetworks, export_subnetwork_last_layer=export_subnetworks) def _subnetwork_train_op_fn(loss, var_list): self.assertLen(var_list, want_subnetwork_trainable_vars) self.assertEqual( var_list, tf_compat.v1.get_collection( tf_compat.v1.GraphKeys.TRAINABLE_VARIABLES)) # Subnetworks get iteration steps instead of global steps. self.assertEqual("subnetwork_test/iteration_step", tf_compat.v1.train.get_global_step().op.name) # Subnetworks get scoped summaries. self.assertEqual("fake_scalar", tf_compat.v1.summary.scalar("scalar", 1.)) self.assertEqual("fake_image", tf_compat.v1.summary.image("image", 1.)) self.assertEqual("fake_histogram", tf_compat.v1.summary.histogram("histogram", 1.)) self.assertEqual("fake_audio", tf_compat.v1.summary.audio("audio", 1., 1.)) optimizer = tf_compat.v1.train.GradientDescentOptimizer( learning_rate=.1) return optimizer.minimize(loss, var_list=var_list) def _mixture_weights_train_op_fn(loss, var_list): self.assertLen(var_list, want_ensemble_trainable_vars) self.assertEqual( var_list, tf_compat.v1.get_collection( tf_compat.v1.GraphKeys.TRAINABLE_VARIABLES)) # Subnetworks get iteration steps instead of global steps. self.assertEqual("ensemble_test/iteration_step", tf_compat.v1.train.get_global_step().op.name) # Subnetworks get scoped summaries. self.assertEqual("fake_scalar", tf_compat.v1.summary.scalar("scalar", 1.)) self.assertEqual("fake_image", tf_compat.v1.summary.image("image", 1.)) self.assertEqual("fake_histogram", tf_compat.v1.summary.histogram("histogram", 1.)) self.assertEqual("fake_audio", tf_compat.v1.summary.audio("audio", 1., 1.)) if not var_list: return tf.no_op() optimizer = tf_compat.v1.train.GradientDescentOptimizer( learning_rate=.1) return optimizer.minimize(loss, var_list=var_list) previous_ensemble = None previous_ensemble_spec = ensemble_spec_fn() if previous_ensemble_spec: previous_ensemble = previous_ensemble_spec.ensemble subnetwork_manager = _SubnetworkManager(head) subnetwork_builder = subnetwork_builder_class( _subnetwork_train_op_fn, _mixture_weights_train_op_fn, use_logits_last_layer, seed, multi_head=multi_head) with tf.Graph().as_default() as g: tf_compat.v1.train.get_or_create_global_step() # A trainable variable to later verify that creating models does not # affect the global variables collection. _ = tf_compat.v1.get_variable("some_var", shape=0, trainable=True) features = {"x": tf.constant([[1.], [2.]])} if multi_head: labels = { "head1": tf.constant([0, 1]), "head2": tf.constant([0, 1]) } else: labels = tf.constant([0, 1]) session_config = tf.compat.v1.ConfigProto( gpu_options=tf.compat.v1.GPUOptions(allow_growth=True)) subnetwork_spec = subnetwork_manager.build_subnetwork_spec( name="test", subnetwork_builder=subnetwork_builder, summary=_FakeSummary(), features=features, mode=mode, labels=labels, previous_ensemble=previous_ensemble) ensembler_kwargs = {} if ensembler_class is ComplexityRegularizedEnsembler: ensembler_kwargs.update({ "mixture_weight_type": mixture_weight_type, "mixture_weight_initializer": mixture_weight_initializer, "warm_start_mixture_weights": warm_start_mixture_weights, "model_dir": self.test_subdirectory, "adanet_lambda": adanet_lambda, "adanet_beta": adanet_beta, "use_bias": use_bias }) if ensembler_class is MeanEnsembler: ensembler_kwargs.update( {"add_mean_last_layer_predictions": True}) ensemble_spec = builder.build_ensemble_spec( # Note: when ensemble_spec is not None and warm_start_mixture_weights # is True, we need to make sure that the bias and mixture weights are # already saved to the checkpoint_dir. name="test", previous_ensemble_spec=previous_ensemble_spec, candidate=EnsembleCandidate("foo", [subnetwork_builder], None), ensembler=ensembler_class(**ensembler_kwargs), subnetwork_specs=[subnetwork_spec], summary=_FakeSummary(), features=features, iteration_number=1, labels=labels, my_ensemble_index=my_ensemble_index, mode=mode, previous_iteration_checkpoint=previous_iteration_checkpoint) if want_replay_indices: self.assertAllEqual(want_replay_indices, ensemble_spec.architecture.replay_indices) with tf_compat.v1.Session( graph=g, config=session_config).as_default() as sess: sess.run(tf_compat.v1.global_variables_initializer()) # Equals the number of subnetwork and ensemble trainable variables, # plus the one 'some_var' created earlier. self.assertLen( tf_compat.v1.trainable_variables(), want_subnetwork_trainable_vars + want_ensemble_trainable_vars + 1) # Get the real global step outside a subnetwork's context. self.assertEqual("global_step", tf_compat.v1.train.get_global_step().op.name) self.assertEqual("global_step", train.get_global_step().op.name) self.assertEqual("global_step", tf_v1.train.get_global_step().op.name) self.assertEqual("global_step", training_util.get_global_step().op.name) self.assertEqual( "global_step", tf_compat.v1.train.get_or_create_global_step().op.name) self.assertEqual("global_step", train.get_or_create_global_step().op.name) self.assertEqual( "global_step", tf_v1.train.get_or_create_global_step().op.name) self.assertEqual( "global_step", training_util.get_or_create_global_step().op.name) # Get global tf.summary outside a subnetwork's context. self.assertNotEqual("fake_scalar", tf_compat.v1.summary.scalar("scalar", 1.)) self.assertNotEqual("fake_image", tf_compat.v1.summary.image("image", 1.)) self.assertNotEqual( "fake_histogram", tf_compat.v1.summary.histogram("histogram", 1.)) self.assertNotEqual( "fake_audio", tf_compat.v1.summary.audio("audio", 1., 1.)) if mode == tf.estimator.ModeKeys.PREDICT: self.assertAllClose(want_logits, sess.run( ensemble_spec.ensemble.logits), atol=1e-3) self.assertIsNone(ensemble_spec.loss) self.assertIsNone(ensemble_spec.adanet_loss) self.assertIsNone(ensemble_spec.train_op) self.assertIsNotNone(ensemble_spec.export_outputs) if not export_subnetworks: return if not multi_head: subnetwork_logits = sess.run( ensemble_spec.export_outputs[ _EnsembleBuilder. _SUBNETWORK_LOGITS_EXPORT_SIGNATURE].outputs) self.assertAllClose( subnetwork_logits["test"], sess.run(subnetwork_spec.subnetwork.logits)) subnetwork_last_layer = sess.run( ensemble_spec.export_outputs[ _EnsembleBuilder. _SUBNETWORK_LAST_LAYER_EXPORT_SIGNATURE]. outputs) self.assertAllClose( subnetwork_last_layer["test"], sess.run(subnetwork_spec.subnetwork.last_layer)) else: self.assertIn("subnetwork_logits_head2", ensemble_spec.export_outputs) subnetwork_logits_head1 = sess.run( ensemble_spec. export_outputs["subnetwork_logits_head1"].outputs) self.assertAllClose( subnetwork_logits_head1["test"], sess.run( subnetwork_spec.subnetwork.logits["head1"])) self.assertIn("subnetwork_logits_head2", ensemble_spec.export_outputs) subnetwork_last_layer_head1 = sess.run( ensemble_spec.export_outputs[ "subnetwork_last_layer_head1"].outputs) self.assertAllClose( subnetwork_last_layer_head1["test"], sess.run(subnetwork_spec.subnetwork. last_layer["head1"])) return # Verify that train_op works, previous loss should be greater than loss # after a train op. loss = sess.run(ensemble_spec.loss) train_op = tf.group(subnetwork_spec.train_op.train_op, ensemble_spec.train_op.train_op) for _ in range(3): sess.run(train_op) self.assertGreater(loss, sess.run(ensemble_spec.loss)) self.assertAllClose(want_logits, sess.run(ensemble_spec.ensemble.logits), atol=1e-3) if ensembler_class is ComplexityRegularizedEnsembler: # Bias should learn a non-zero value when used. bias = sess.run(ensemble_spec.ensemble.bias) if isinstance(bias, dict): bias = sum(abs(b) for b in bias.values()) if use_bias: self.assertNotEqual(0., bias) else: self.assertAlmostEqual(0., bias) self.assertAlmostEqual(want_loss, sess.run(ensemble_spec.loss), places=3) self.assertAlmostEqual(want_adanet_loss, sess.run(ensemble_spec.adanet_loss), places=3) if want_predictions: self.assertAllClose( want_predictions, sess.run(ensemble_spec.ensemble.predictions), atol=1e-3)
def testConvertTfjsLayersModelIntoShardedWeights(self): with tf.Graph().as_default(), tf.compat.v1.Session(): x = np.random.randn(8, 10) # 1. Run the model.predict(), store the result. Then saved the model # as a SavedModel. model = self._createNestedSequentialModel() y = model.predict(x) weights = model.get_weights() total_weight_bytes = sum(np.size(w) for w in weights) * 4 tf.keras.models.save_model(model, self._tmp_dir) # 2. Convert the keras saved model to tfjs_layers_model format. tfjs_output_dir = os.path.join(self._tmp_dir, 'tfjs') # Implicit value of --output_format: tfjs_layers_model process = subprocess.Popen([ 'tensorflowjs_converter', '--input_format', 'keras_saved_model', self._tmp_dir, tfjs_output_dir ]) process.communicate() self.assertEqual(0, process.returncode) # 3. Convert the tfjs_layers_model to another tfjs_layers_model, # with sharded weights. weight_shard_size_bytes = int(total_weight_bytes * 0.3) # Due to the shard size, there ought to be 4 shards after conversion. sharded_model_dir = os.path.join(self._tmp_dir, 'tfjs_sharded') process = subprocess.Popen([ 'tensorflowjs_converter', '--input_format', 'tfjs_layers_model', '--output_format', 'tfjs_layers_model', '--weight_shard_size_bytes', str(weight_shard_size_bytes), os.path.join(tfjs_output_dir, 'model.json'), sharded_model_dir ]) process.communicate() self.assertEqual(0, process.returncode) # 4. Check the sharded weight files and their sizes. weight_files = sorted( glob.glob(os.path.join(sharded_model_dir, 'group*.bin'))) self.assertEqual(len(weight_files), 4) weight_file_sizes = [os.path.getsize(f) for f in weight_files] self.assertEqual(sum(weight_file_sizes), total_weight_bytes) self.assertEqual(weight_file_sizes[0], weight_file_sizes[1]) self.assertEqual(weight_file_sizes[0], weight_file_sizes[2]) self.assertLess(weight_file_sizes[3], weight_file_sizes[0]) # 5. Convert the sharded tfjs_layers_model back into a keras h5 file. new_h5_path = os.path.join(self._tmp_dir, 'new_h5.h5') process = subprocess.Popen([ 'tensorflowjs_converter', '--input_format', 'tfjs_layers_model', os.path.join(sharded_model_dir, 'model.json'), new_h5_path ]) process.communicate() self.assertEqual(0, process.returncode) with tf.Graph().as_default(), tf.compat.v1.Session(): # 6. Load the keras model and check the predict() output is close to # before. new_model = keras.models.load_model(new_h5_path) new_y = new_model.predict(x) self.assertAllClose(new_y, y)
def test_generate_candidates(self, want_names, want_subnetwork_losses, want_mixture_weight_losses, want_complexities, learn_mixture_weights=False, initial_num_layers=0, previous_ensemble=None): feature_columns = [tf.feature_column.numeric_column("x")] generator = simple_dnn.Generator( feature_columns=feature_columns, optimizer=tf.compat.v1.train.GradientDescentOptimizer(.1), layer_size=3, initial_num_layers=initial_num_layers, learn_mixture_weights=learn_mixture_weights, seed=42) with context.graph_mode(), tf.Graph().as_default() as g: iteration_step = tf.compat.v1.train.create_global_step() features = {"x": [[1.], [2.]]} labels = tf.constant([[0.], [1.]]) names = [] subnetwork_losses = [] mixture_weight_losses = [] complexities = [] for builder in generator.generate_candidates( previous_ensemble, # The following arguments are not used by # simple_dnn.BuilderGenerator's generate_candidates. iteration_number=0, previous_ensemble_reports=[], all_reports=[]): names.append(builder.name) # 1. Build subnetwork graph. subnetwork = builder.build_subnetwork( features, logits_dimension=1, training=True, iteration_step=iteration_step, summary=tf.summary, previous_ensemble=previous_ensemble) # 2. Build subnetwork train ops. subnetwork_loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits( logits=subnetwork.logits, labels=labels)) subnetwork_train_op = builder.build_subnetwork_train_op( subnetwork, subnetwork_loss, var_list=None, labels=labels, iteration_step=iteration_step, summary=tf.summary, previous_ensemble=None) # 3. Build mixture weight train ops. # Stop gradients since mixture weights should have not propagate # beyond top layer. subnetwork_logits = tf.stop_gradient(subnetwork.logits) # Mixture weight will initialize to a one-valued scalar. mixture_weight_logits = tf.compat.v1.layers.dense( subnetwork_logits, units=1, use_bias=False, kernel_initializer=tf.ones_initializer()) mixture_weight_loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits( logits=mixture_weight_logits, labels=labels)) mixture_weight_train_op = builder.build_mixture_weights_train_op( mixture_weight_loss, var_list=None, labels=labels, logits=mixture_weight_logits, iteration_step=iteration_step, summary=tf.summary) with self.test_session(graph=g) as sess: sess.run(tf.compat.v1.global_variables_initializer()) sess.run(subnetwork_train_op) sess.run(mixture_weight_train_op) subnetwork_losses.append(sess.run(subnetwork_loss)) mixture_weight_losses.append(sess.run(mixture_weight_loss)) complexities.append(sess.run(subnetwork.complexity)) self.assertEqual(want_names, names) self.assertAllClose(want_subnetwork_losses, subnetwork_losses, atol=1e-3) self.assertAllClose(want_mixture_weight_losses, mixture_weight_losses, atol=1e-3) self.assertAllClose(want_complexities, complexities, atol=1e-3)
def testDenseWithLearningRateInverseTimeDecay(self): # TODO(tanzheny, omalleyt): Fix test in eager mode. with tf.Graph().as_default(): var0_np = np.array([1.0, 2.0]) grads0_np = np.array([0.1, 0.2]) var1_np = np.array([3.0, 4.0]) grads1_np = np.array([0.01, 0.2]) var0 = tf.Variable(var0_np) var1 = tf.Variable(var1_np) grads0 = tf.constant(grads0_np) grads1 = tf.constant(grads1_np) learning_rate = 0.01 rho = 0.9 momentum = 0.0 epsilon = 1e-7 centered = False decay = 0.5 lr_schedule = learning_rate_schedule.InverseTimeDecay( learning_rate, decay_steps=1.0, decay_rate=decay) opt = rmsprop.RMSprop(learning_rate=lr_schedule, rho=rho, momentum=momentum, epsilon=epsilon, centered=centered) update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) self.evaluate(tf.compat.v1.global_variables_initializer()) rms0 = opt.get_slot(var0, "rms") self.assertIsNotNone(rms0) rms1 = opt.get_slot(var1, "rms") self.assertIsNotNone(rms1) if momentum > 0.: mom0 = opt.get_slot(var0, "momentum") mom1 = opt.get_slot(var1, "momentum") else: mom0 = None mom1 = None mg0_np = np.array([0.0, 0.0]) mg1_np = np.array([0.0, 0.0]) rms0_np = np.array([0.0, 0.0]) rms1_np = np.array([0.0, 0.0]) mom0_np = np.array([0.0, 0.0]) mom1_np = np.array([0.0, 0.0]) # Fetch params to validate initial values self.assertAllClose([1.0, 2.0], self.evaluate(var0)) self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run 4 steps of RMSprop for t in range(2): self.evaluate(update) lr = learning_rate / (1 + decay * t) var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy( var0_np, grads0_np, mg0_np, rms0_np, mom0_np, lr, rho, momentum, epsilon, centered) var1_np, mg1_np, rms1_np, mom1_np = self._rmsprop_update_numpy( var1_np, grads1_np, mg1_np, rms1_np, mom1_np, lr, rho, momentum, epsilon, centered) # Validate updated params self.assertAllCloseAccordingToType(rms0_np, self.evaluate(rms0)) self.assertAllCloseAccordingToType(rms1_np, self.evaluate(rms1)) if momentum > 0.: self.assertAllCloseAccordingToType(mom0_np, self.evaluate(mom0)) self.assertAllCloseAccordingToType(mom1_np, self.evaluate(mom1)) self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
def _import_and_infer(save_dir, inputs, signature_key="serving_default"): """Import a SavedModel into a TF 1.x-style graph and run `signature_key`.""" graph = tf.Graph() with graph.as_default(), tf.compat.v1.Session() as session: model = tf.compat.v1.saved_model.load(session, ["serve"], save_dir) return _run_signature(session, model, inputs, signature_key)
def _export_mode(mode, has_saved_vars, builder, model, custom_objects, checkpoint_path, input_signature): """Exports a model, and optionally saves new vars from the clone model. Args: mode: A `tf.estimator.ModeKeys` string. has_saved_vars: A `boolean` indicating whether the SavedModel has already exported variables. builder: A `SavedModelBuilder` object. model: A `tf.keras.Model` object. custom_objects: A dictionary mapping string names to custom classes or functions. checkpoint_path: String path to checkpoint. input_signature: Nested TensorSpec containing the expected inputs. Can be `None`, in which case the signature will be inferred from the model. Raises: ValueError: If the train/eval mode is being exported, but the model does not have an optimizer. """ compile_clone = (mode != mode_keys.ModeKeys.PREDICT) if compile_clone and not model.optimizer: raise ValueError( 'Model does not have an optimizer. Cannot export mode %s' % mode) model_graph = tf.compat.v1.get_default_graph() with tf.Graph().as_default() as g, backend.learning_phase_scope( mode == mode_keys.ModeKeys.TRAIN): if input_signature is None: input_tensors = None else: input_tensors = tf.nest.map_structure(create_placeholder, input_signature) # Clone the model into blank graph. This will create placeholders for inputs # and targets. clone = models_lib.clone_and_build_model(model, input_tensors=input_tensors, custom_objects=custom_objects, compile_clone=compile_clone) # Make sure that iterations variable is added to the global step collection, # to ensure that, when the SavedModel graph is loaded, the iterations # variable is returned by `tf.compat.v1.train.get_global_step()`. This is # required for compatibility with the SavedModelEstimator. if compile_clone: g.add_to_collection(tf.compat.v1.GraphKeys.GLOBAL_STEP, clone.optimizer.iterations) # Extract update and train ops from train/test/predict functions. train_op = None if mode == mode_keys.ModeKeys.TRAIN: clone._make_train_function() # pylint: disable=protected-access train_op = clone.train_function.updates_op elif mode == mode_keys.ModeKeys.TEST: clone._make_test_function() # pylint: disable=protected-access else: clone._make_predict_function() # pylint: disable=protected-access g.get_collection_ref(tf.compat.v1.GraphKeys.UPDATE_OPS).extend( clone.state_updates) with tf.compat.v1.Session().as_default(): clone_var_list = _get_var_list(clone) if has_saved_vars: # Confirm all variables in the clone have an entry in the checkpoint. status = clone.load_weights(checkpoint_path) status.assert_existing_objects_matched() else: # Confirm that variables between the clone and model match up exactly, # not counting optimizer objects. Optimizer objects are ignored because # if the model has not trained, the slot variables will not have been # created yet. # TODO(b/113179535): Replace with trackable equivalence. _assert_same_non_optimizer_objects(model, model_graph, clone, g) # TODO(b/113178242): Use value transfer for trackable objects. clone.load_weights(checkpoint_path) # Add graph and variables to SavedModel. # TODO(b/113134168): Switch to add_meta_graph_and_variables. clone.save_weights(checkpoint_path, save_format='tf', overwrite=True) builder._has_saved_variables = True # pylint: disable=protected-access # Add graph to the SavedModel builder. builder.add_meta_graph( model_utils.EXPORT_TAG_MAP[mode], signature_def_map=_create_signature_def_map(clone, mode), saver=tf.compat.v1.train.Saver( clone_var_list, # Allow saving Models with no variables. This is somewhat odd, but # it's not necessarily a bug. allow_empty=True), init_op=tf.compat.v1.local_variables_initializer(), train_op=train_op) return None
def test_with_1d_unknown_shape_sparse_tensor(self): embedding_values = ( (1.0, 2.0), # id 0 (6.0, 7.0), # id 1 (11.0, 12.0), # id 2 ) def _initializer(shape, dtype, partition_info=None): del shape, dtype, partition_info return embedding_values # price has 1 dimension in dense_features price = tf.feature_column.numeric_column("price") # one_hot_body_style has 3 dims in dense_features. body_style = tf.feature_column.categorical_column_with_vocabulary_list( "body-style", vocabulary_list=["hardtop", "wagon", "sedan"] ) one_hot_body_style = tf.feature_column.indicator_column(body_style) # embedded_body_style has 5 dims in dense_features. country = tf.feature_column.categorical_column_with_vocabulary_list( "country", vocabulary_list=["US", "JP", "CA"] ) embedded_country = tf.feature_column.embedding_column( country, dimension=2, initializer=_initializer ) # Provides 1-dim tensor and dense tensor. with tf.Graph().as_default(): features = { "price": tf.compat.v1.placeholder(tf.float32), "body-style": tf.compat.v1.sparse_placeholder(tf.string), # This is dense tensor for the categorical_column. "country": tf.compat.v1.placeholder(tf.string), } self.assertIsNone(features["price"].shape.ndims) self.assertIsNone(features["body-style"].get_shape().ndims) self.assertIsNone(features["country"].shape.ndims) price_data = np.array([11.0, 12.0]) body_style_data = tf.compat.v1.SparseTensorValue( indices=((0,), (1,)), values=("sedan", "hardtop"), dense_shape=(2,), ) country_data = np.array([["US"], ["CA"]]) net = df.DenseFeatures( [price, one_hot_body_style, embedded_country] )(features) self.assertEqual(1 + 3 + 2, net.shape[1]) with _initialized_session() as sess: # Each row is formed by concatenating `embedded_body_style`, # `one_hot_body_style`, and `price` in order. self.assertAllEqual( [ [0.0, 0.0, 1.0, 1.0, 2.0, 11.0], [1.0, 0.0, 0.0, 11.0, 12.0, 12.0], ], sess.run( net, feed_dict={ features["price"]: price_data, features["body-style"]: body_style_data, features["country"]: country_data, }, ), )
def main(): #download google pre-trained neural network local_zip_file = 'inception5h.zip' if not os.path.exists(local_zip_file): #download model_url = urllib.request.urlopen(url) with open(local_zip_file, 'wb') as output: output.write(model_url.read()) #extract with zipfile.ZipFile(local_zip_file, 'r') as zip_ref: zip_ref.extractall(data_dir) model_fn = 'tensorflow_inseption_graph.pb' #Creating tf session and loading the model graph = tf.Graph() sess = tfc.InteractiveSession(graph=graph) with tfc.gfile.FastGFile((local_zip_file), 'rb') as f: graph_def = tf.io.gfile.GFile() graph_def.ParseFromString(f.read()) t_input = tf.placeholder(np.float32, name='input') #define input tensor imagenet_mean = 117.0 t_preprocessed = tf.expand_dims(t_input - imagenet_mean, 0) tf.import_graph_def(graph_def, {'input': t_preprocessed}) layers = [ op.name for op in graph.get_operations() if op.type == 'Cony2D' and 'import/' in op.name ] feature_nums = [ int(graph.get_tensor_by_name(model_name + ':0').get_shape()[-1]) for name in layers ] print('Number of layers: ', len(layers)) print('Total numbers of feature channels:', sum(feature_nums)) def render_deepdream(t_obj, img0=img_noise, iter_n=10, step=1.5, octave_n=4, octave_scale=1.4): t_score = tf.reduce_mean(t_obj) #defining optimization objective t_grad = tf.gradients(t_score, t_input)[0] #split the image into a number of octaves img = img0 octaves = [] for _ in range(octave_n - 1): hw = img.shape[:2] lo = resize(img, np.int32(np.float32(hw) / octave_scale)) hi = img - resize(low, hw) img = lo octaves.append(hi) #generate details octave by octave for octave in range(octave_n): if octave > 0: hi = octaves[-octave] img = resize(img, hi.shape[:2]) + hi for _ in range(iter_n): g = calc_grad_tiled(img, t_grad) img += g * (step / (np.abs(g).mean() + 1e-7)) #output deep dreamed image showarray(img / 255.0) #Pick a layer to enchance my image layer = 'mixed4d_3x3_bottleneck_pre_relu' channel = 139 img0 = PIL.Image.open('image.jpg') img0 = np.float32(img0) #Apply gradient ascent to the layer render_deepdream(tf.square(T('mixed4c')), img0)
def testSparse(self): # TODO(tanzheny, omalleyt): Fix test in eager mode. with tf.Graph().as_default(): for dtype in [tf.half, tf.float32, tf.float64]: var0 = tf.Variable(tf.zeros([4, 2], dtype=dtype)) var1 = tf.Variable(tf.constant(1.0, dtype, [4, 2])) grads0 = tf.IndexedSlices(tf.constant([[.1, .1]], dtype=dtype), tf.constant([1]), tf.constant([4, 2])) grads1 = tf.IndexedSlices( tf.constant([[.01, .01], [.01, .01]], dtype=dtype), tf.constant([2, 3]), tf.constant([4, 2])) mom_opt = gradient_descent.SGD(learning_rate=2.0, momentum=0.9) mom_update = mom_opt.apply_gradients( zip([grads0, grads1], [var0, var1])) self.evaluate(tf.compat.v1.global_variables_initializer()) # Check we have slots slot0 = mom_opt.get_slot(var0, "momentum") self.assertEqual(slot0.shape, var0.shape) slot1 = mom_opt.get_slot(var1, "momentum") self.assertEqual(slot1.shape, var1.shape) # Fetch params to validate initial values self.assertAllClose([0, 0], self.evaluate(var0)[0]) self.assertAllClose([0, 0], self.evaluate(var0)[1]) self.assertAllClose([1, 1], self.evaluate(var1)[2]) # Step 1: the momentum accumulators are 0. So we should see a normal # update: v -= grad * learning_rate self.evaluate(mom_update) # Check that the momentum accumulators have been updated. self.assertAllCloseAccordingToType(np.array([0, 0]), self.evaluate(slot0)[0]) self.assertAllCloseAccordingToType( np.array([-2.0 * .1, -2.0 * .1]), self.evaluate(slot0)[1]) self.assertAllCloseAccordingToType( np.array([-2.0 * .01, -2.0 * .01]), self.evaluate(slot1)[2]) # Check that the parameters have been updated. self.assertAllCloseAccordingToType(np.array([0, 0]), self.evaluate(var0)[0]) self.assertAllCloseAccordingToType( np.array([-(0.1 * 2.0), -(0.1 * 2.0)]), self.evaluate(var0)[1]) self.assertAllCloseAccordingToType( np.array([1.0 - (0.01 * 2.0), 1.0 - (0.01 * 2.0)]), self.evaluate(var1)[2]) # Step 2: the momentum accumulators contain the previous update. self.evaluate(mom_update) # Check that the momentum accumulators have been updated. self.assertAllClose(np.array([0, 0]), self.evaluate(slot0)[0]) self.assertAllCloseAccordingToType( np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), self.evaluate(slot0)[1]) self.assertAllCloseAccordingToType( np.array([(0.9 * (-0.02) - 2.0 * 0.01), (0.9 * (-0.02) - 2.0 * 0.01)]), self.evaluate(slot1)[2]) # Check that the parameters have been updated. self.assertAllClose(np.array([0, 0]), self.evaluate(var0)[0]) self.assertAllCloseAccordingToType( np.array([ -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) ]), self.evaluate(var0)[1]) self.assertAllCloseAccordingToType( np.array([ 0.98 - ((0.9 * 0.01 + 0.01) * 2.0), 0.98 - ((0.9 * 0.01 + 0.01) * 2.0) ]), self.evaluate(var1)[2])
def test_import_tower(self, shared_input): np.random.seed(42) test_input = np.random.random([1, 32, 32, 3]) # Force graph mode with tf.compat.v1.Graph().as_default(): directory = self.get_temp_dir() architecture = np.array([1, 3, 34]) phoenix_spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN) phoenix_spec.is_input_shared = shared_input features = {} shared_input_tensor = None with self.test_session(graph=tf.Graph()) as sess: input_tensor_1 = tf.compat.v1.placeholder( dtype=tf.float32, shape=[None, 32, 32, 3], name="input_1") tf.random.set_seed(1234) tower_spec_1 = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor_1, tower_name="test_tower", architecture=architecture, is_training=True, lengths=None, logits_dimension=10, is_frozen=False, dropout_rate=None) saver = tf.compat.v1.train.Saver() sess.run(tf.compat.v1.global_variables_initializer()) sess.run(tf.compat.v1.local_variables_initializer()) logits_val_1 = sess.run(tower_spec_1.logits_spec.logits, feed_dict={input_tensor_1: test_input}) saver.save(sess, directory + "/ckpt") with self.test_session(graph=tf.Graph()) as sess: input_tensor_2 = tf.compat.v1.placeholder( dtype=tf.float32, shape=[None, 32, 32, 3], name="input_2") if shared_input: shared_input_tensor = input_tensor_2 def _input_layer_fn(features, is_training, scope_name="Phoenix/Input", lengths_feature_name=None): del features, is_training, scope_name, lengths_feature_name return None, None else: features = {"x": input_tensor_2} def _input_layer_fn(features, is_training, scope_name="Phoenix/Input", lengths_feature_name=None): del is_training, lengths_feature_name with tf.compat.v1.variable_scope(scope_name): return tf.cast(features["x"], dtype=tf.float32), None tf.random.set_seed(1234) tower_spec_2 = architecture_utils.import_tower( features=features, input_layer_fn=_input_layer_fn, phoenix_spec=phoenix_spec, shared_input_tensor=shared_input_tensor, original_tower_name="test_tower", new_tower_name="imported_tower", model_directory=directory, is_training=True, logits_dimension=10, shared_lengths=None, force_snapshot=False, force_freeze=False) sess.run(tf.compat.v1.global_variables_initializer()) sess.run(tf.compat.v1.local_variables_initializer()) logits_val_2 = sess.run(tower_spec_2.logits_spec.logits, feed_dict={input_tensor_2: test_input}) self.assertAllClose(logits_val_1, logits_val_2, rtol=1e-3)
def testSaveAndLoadSavedModelExport( self, model_builder, uses_learning_phase, optimizer_cls, train_before_export): optimizer = None if optimizer_cls is None else optimizer_cls() saved_model_dir = self._save_model_dir() np.random.seed(130) input_arr = np.random.random((1, 3)) target_arr = np.random.random((1, 3)) model = model_builder(uses_learning_phase) if optimizer is not None: model.compile( loss='mse', optimizer=optimizer, metrics=['mae']) if train_before_export: model.train_on_batch(input_arr, target_arr) ref_loss, ref_mae = model.evaluate(input_arr, target_arr) ref_predict = model.predict(input_arr) # Export SavedModel keras_saved_model.export_saved_model(model, saved_model_dir) input_name = model.input_names[0] output_name = model.output_names[0] target_name = output_name + '_target' # Load predict graph, and test predictions with tf.compat.v1.Session(graph=tf.Graph()) as sess: inputs, outputs, _ = load_model(sess, saved_model_dir, mode_keys.ModeKeys.PREDICT) predictions = sess.run(outputs[output_name], {inputs[input_name]: input_arr}) self.assertAllClose(ref_predict, predictions, atol=1e-05) if optimizer: # Load eval graph, and test predictions, loss and metric values with tf.compat.v1.Session(graph=tf.Graph()) as sess: inputs, outputs, _ = load_model(sess, saved_model_dir, mode_keys.ModeKeys.TEST) # First obtain the loss and predictions, and run the metric update op by # feeding in the inputs and targets. metrics_name = 'mae' if tf.__internal__.tf2.enabled() else 'mean_absolute_error' metrics_update_op_key = 'metrics/' + metrics_name + '/update_op' metrics_value_op_key = 'metrics/' + metrics_name + '/value' loss, predictions, _ = sess.run( (outputs['loss'], outputs['predictions/' + output_name], outputs[metrics_update_op_key]), { inputs[input_name]: input_arr, inputs[target_name]: target_arr }) # The metric value should be run after the update op, to ensure that it # reflects the correct value. metric_value = sess.run(outputs[metrics_value_op_key]) self.assertEqual(int(train_before_export), sess.run(tf.compat.v1.train.get_global_step())) self.assertAllClose(ref_loss, loss, atol=1e-05) self.assertAllClose(ref_mae, metric_value, atol=1e-05) self.assertAllClose(ref_predict, predictions, atol=1e-05) # Load train graph, and check for the train op, and prediction values with tf.compat.v1.Session(graph=tf.Graph()) as sess: inputs, outputs, meta_graph_def = load_model( sess, saved_model_dir, mode_keys.ModeKeys.TRAIN) self.assertEqual(int(train_before_export), sess.run(tf.compat.v1.train.get_global_step())) self.assertIn('loss', outputs) self.assertIn(metrics_update_op_key, outputs) self.assertIn(metrics_value_op_key, outputs) self.assertIn('predictions/' + output_name, outputs) # Train for a step train_op = get_train_op(meta_graph_def) train_outputs, _ = sess.run( [outputs, train_op], {inputs[input_name]: input_arr, inputs[target_name]: target_arr}) self.assertEqual(int(train_before_export) + 1, sess.run(tf.compat.v1.train.get_global_step())) if uses_learning_phase: self.assertAllClose( [[0, 0, 0]], train_outputs['predictions/' + output_name], atol=1e-05) else: self.assertNotAllClose( [[0, 0, 0]], train_outputs['predictions/' + output_name], atol=1e-05)
"""Script to generate fake 'Lost and Found' data.""" import tensorflow.compat.v2 as tf import tensorflow_datasets.testing.cityscapes as cityscapes if __name__ == '__main__': example_dir = ('tensorflow_datasets/testing/test_data/fake_examples/' 'lost_and_found') base_path = example_dir + '/{}.zip' # generate image ids matching between zipfiles train_ids = list(cityscapes.generate_ids('01_Turmstr_17')) + list( cityscapes.generate_ids('02_Goethe_Str_6')) test_ids = list(cityscapes.generate_ids('03_Schlossallee_1')) splits = {'train': train_ids, 'test': test_ids} with tf.Graph().as_default(): cityscapes.create_zipfile( base_path.format('leftImg8bit'), splits_with_ids=splits, suffixes=['leftImg8bit']) cityscapes.create_zipfile( base_path.format('gtCoarse'), splits_with_ids=splits, suffixes=[ 'gtCoarse_instanceIds', 'gtCoarse_labelIds', 'gtCoarse_color' ]) cityscapes.create_zipfile( base_path.format('rightImg8bit'), splits_with_ids=splits, suffixes=['rightImg8bit']) cityscapes.create_zipfile(
def testBasicWithLearningRateDecay(self): for i, dtype in enumerate([tf.half, tf.float32, tf.float64]): with self.session(graph=tf.Graph(), use_gpu=True): # Initialize variables for numpy implementation. m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype) var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype) var0 = tf.Variable(var0_np, name="var0_%d" % i) var1 = tf.Variable(var1_np, name="var1_%d" % i) grads0 = tf.constant(grads0_np) grads1 = tf.constant(grads1_np) learning_rate = 0.001 decay = 0.002 opt = adamax.Adamax(learning_rate=learning_rate, decay=decay) if not tf.executing_eagerly(): update = opt.apply_gradients( zip([grads0, grads1], [var0, var1])) if not tf.executing_eagerly(): self.evaluate(tf.compat.v1.global_variables_initializer()) # Fetch params to validate initial values self.assertAllClose([1.0, 2.0], self.evaluate(var0)) self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Run 3 steps of Adamax for t in range(3): beta_1_power = get_beta_accumulators(opt, dtype) self.assertAllCloseAccordingToType( 0.9**(t + 1), self.evaluate(beta_1_power)) if not tf.executing_eagerly(): self.evaluate(update) else: opt.apply_gradients(zip([grads0, grads1], [var0, var1])) lr = learning_rate / (1 + decay * t) var0_np, m0, v0 = adamax_update_numpy(var0_np, grads0_np, t, m0, v0, alpha=lr) var1_np, m1, v1 = adamax_update_numpy(var1_np, grads1_np, t, m1, v1, alpha=lr) # Validate updated params self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0), rtol=1e-2) self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1), rtol=1e-2)
def testSparseBasic(self): # TODO(tanzheny, omalleyt): Fix test in eager mode. with tf.Graph().as_default(): for dtype in _DATA_TYPES: var0_np = np.array([1.0, 1.0, 2.0], dtype=dtype.as_numpy_dtype) grads0_np = np.array([0.1, 0, 0.1], dtype=dtype.as_numpy_dtype) var1_np = np.array([3.0, 3.0, 4.0], dtype=dtype.as_numpy_dtype) grads1_np = np.array([0.01, 0, 0.01], dtype=dtype.as_numpy_dtype) var0 = tf.Variable(var0_np) var1 = tf.Variable(var1_np) grads0_np_indices = np.array([0, 2], dtype=np.int32) grads0 = tf.IndexedSlices( tf.constant(grads0_np[grads0_np_indices]), tf.constant(grads0_np_indices), tf.constant([3]), ) grads1_np_indices = np.array([0, 2], dtype=np.int32) grads1 = tf.IndexedSlices( tf.constant(grads1_np[grads1_np_indices]), tf.constant(grads1_np_indices), tf.constant([3]), ) learning_rate = 3.0 ada_opt = adagrad.Adagrad(learning_rate) ada_update = ada_opt.apply_gradients( zip([grads0, grads1], [var0, var1])) self.evaluate(tf.compat.v1.global_variables_initializer()) # Fetch params to validate initial values self.assertAllClose([1.0, 1.0, 2.0], self.evaluate(var0)) self.assertAllClose([3.0, 3.0, 4.0], self.evaluate(var1)) accum0_np = np.array([0.1, 0.1, 0.1], dtype=dtype.as_numpy_dtype) accum1_np = np.array([0.1, 0.1, 0.1], dtype=dtype.as_numpy_dtype) # Run 3 step of sgd for _ in range(3): self.evaluate(ada_update) var0_np, accum0_np = sparse_adagrad_update_numpy( var0_np, accum0_np, grads0_np_indices, grads0_np[grads0_np_indices], learning_rate, ) var1_np, accum1_np = sparse_adagrad_update_numpy( var1_np, accum1_np, grads1_np_indices, grads1_np[grads1_np_indices], learning_rate, ) self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0)) self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
def test_shared_embedding_column(self): with tf.Graph().as_default(): vocabulary_size = 3 sparse_input_a = tf.compat.v1.SparseTensorValue( # example 0, ids [2] # example 1, ids [0, 1] indices=((0, 0), (1, 0), (1, 1)), values=(2, 0, 1), dense_shape=(2, 2)) sparse_input_b = tf.compat.v1.SparseTensorValue( # example 0, ids [1] # example 1, ids [2, 0] indices=((0, 0), (1, 0), (1, 1)), values=(1, 2, 0), dense_shape=(2, 2)) embedding_dimension = 2 embedding_values = ( (1., 2.), # id 0 (3., 4.), # id 1 (5., 6.) # id 2 ) def _get_initializer(embedding_dimension, embedding_values): def _initializer(shape, dtype, partition_info=None): self.assertAllEqual((vocabulary_size, embedding_dimension), shape) self.assertEqual(tf.float32, dtype) self.assertIsNone(partition_info) return embedding_values return _initializer expected_input_layer = [ # example 0, ids_a [2], ids_b [1] [[5., 6., 3., 4.], [0., 0., 0., 0.]], # example 1, ids_a [0, 1], ids_b [2, 0] [[1., 2., 5., 6.], [3., 4., 1., 2.]], ] expected_sequence_length = [1, 2] categorical_column_a = tf.feature_column.sequence_categorical_column_with_identity( key='aaa', num_buckets=vocabulary_size) categorical_column_b = tf.feature_column.sequence_categorical_column_with_identity( key='bbb', num_buckets=vocabulary_size) # Test that columns are reordered alphabetically. shared_embedding_columns = tf.feature_column.shared_embeddings( [categorical_column_b, categorical_column_a], dimension=embedding_dimension, initializer=_get_initializer(embedding_dimension, embedding_values)) sequence_input_layer = ksfc.SequenceFeatures( shared_embedding_columns) input_layer, sequence_length = sequence_input_layer({ 'aaa': sparse_input_a, 'bbb': sparse_input_b }) global_vars = tf.compat.v1.get_collection( tf.compat.v1.GraphKeys.GLOBAL_VARIABLES) self.assertCountEqual(('aaa_bbb_shared_embedding:0', ), tuple([v.name for v in global_vars])) with _initialized_session() as sess: self.assertAllEqual(embedding_values, global_vars[0].eval(session=sess)) self.assertAllEqual(expected_input_layer, input_layer.eval(session=sess)) self.assertAllEqual(expected_sequence_length, sequence_length.eval(session=sess))
def test_TensorBoard_multi_input_output(self): np.random.seed(1337) tmpdir = self.get_temp_dir() self.addCleanup(shutil.rmtree, tmpdir, ignore_errors=True) with tf.Graph().as_default(), self.cached_session(): filepath = os.path.join(tmpdir, 'logs') (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( train_samples=TRAIN_SAMPLES, test_samples=TEST_SAMPLES, input_shape=(INPUT_DIM, ), num_classes=NUM_CLASSES) y_test = np_utils.to_categorical(y_test) y_train = np_utils.to_categorical(y_train) def data_generator(train): if train: max_batch_index = len(x_train) // BATCH_SIZE else: max_batch_index = len(x_test) // BATCH_SIZE i = 0 while 1: if train: # simulate multi-input/output models yield ([x_train[i * BATCH_SIZE:(i + 1) * BATCH_SIZE]] * 2, [y_train[i * BATCH_SIZE:(i + 1) * BATCH_SIZE]] * 2) else: yield ([x_test[i * BATCH_SIZE:(i + 1) * BATCH_SIZE]] * 2, [y_test[i * BATCH_SIZE:(i + 1) * BATCH_SIZE]] * 2) i += 1 i %= max_batch_index inp1 = input_layer.Input((INPUT_DIM, )) inp2 = input_layer.Input((INPUT_DIM, )) inp = layers.add([inp1, inp2]) hidden = layers.Dense(2, activation='relu')(inp) hidden = layers.Dropout(0.1)(hidden) output1 = layers.Dense(NUM_CLASSES, activation='softmax')(hidden) output2 = layers.Dense(NUM_CLASSES, activation='softmax')(hidden) model = training.Model([inp1, inp2], [output1, output2]) model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy']) # we must generate new callbacks for each test, as they aren't stateless def callbacks_factory(histogram_freq): return [ callbacks_v1.TensorBoard(log_dir=filepath, histogram_freq=histogram_freq, write_images=True, write_grads=True, batch_size=5) ] # fit without validation data model.fit([x_train] * 2, [y_train] * 2, batch_size=BATCH_SIZE, callbacks=callbacks_factory(histogram_freq=0), epochs=3) # fit with validation data and accuracy model.fit([x_train] * 2, [y_train] * 2, batch_size=BATCH_SIZE, validation_data=([x_test] * 2, [y_test] * 2), callbacks=callbacks_factory(histogram_freq=1), epochs=2) # fit generator without validation data model.fit_generator(data_generator(True), len(x_train), epochs=2, callbacks=callbacks_factory(histogram_freq=0)) # fit generator with validation data and accuracy model.fit_generator(data_generator(True), len(x_train), epochs=2, validation_data=([x_test] * 2, [y_test] * 2), callbacks=callbacks_factory(histogram_freq=1)) assert os.path.isdir(filepath)
def test_with_graph(self): with tf.Graph().as_default(): with tf.Graph().as_default() as g: ds = _create_dataset(range(10)) np_ds = dataset_utils.as_numpy(ds, graph=g) self.assertEqual(list(range(10)), [int(el) for el in list(np_ds)])
def test_Tensorboard_histogram_summaries_in_test_function(self): class FileWriterStub(object): def __init__(self, logdir, graph=None): self.logdir = logdir self.graph = graph self.steps_seen = [] def add_summary(self, summary, global_step): summary_obj = tf.compat.v1.Summary() # ensure a valid Summary proto is being sent if isinstance(summary, bytes): summary_obj.ParseFromString(summary) else: assert isinstance(summary, tf.compat.v1.Summary) summary_obj = summary # keep track of steps seen for the merged_summary op, # which contains the histogram summaries if len(summary_obj.value) > 1: self.steps_seen.append(global_step) def flush(self): pass def close(self): pass def _init_writer(obj, _): obj.writer = FileWriterStub(obj.log_dir) np.random.seed(1337) tmpdir = self.get_temp_dir() self.addCleanup(shutil.rmtree, tmpdir, ignore_errors=True) (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( train_samples=TRAIN_SAMPLES, test_samples=TEST_SAMPLES, input_shape=(INPUT_DIM, ), num_classes=NUM_CLASSES) y_test = np_utils.to_categorical(y_test) y_train = np_utils.to_categorical(y_train) with tf.Graph().as_default(), self.cached_session(): model = sequential.Sequential() model.add( layers.Dense(NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu')) # non_trainable_weights: moving_variance, moving_mean model.add(layers.BatchNormalization()) model.add(layers.Dense(NUM_CLASSES, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy']) callbacks_v1.TensorBoard._init_writer = _init_writer tsb = callbacks_v1.TensorBoard(log_dir=tmpdir, histogram_freq=1, write_images=True, write_grads=True, batch_size=5) cbks = [tsb] # fit with validation data model.fit(x_train, y_train, batch_size=BATCH_SIZE, validation_data=(x_test, y_test), callbacks=cbks, epochs=3, verbose=0) self.assertAllEqual(tsb.writer.steps_seen, [0, 1, 2, 3, 4, 5])
def test_with_1d_sparse_tensor(self): embedding_values = ( (1.0, 2.0, 3.0, 4.0, 5.0), # id 0 (6.0, 7.0, 8.0, 9.0, 10.0), # id 1 (11.0, 12.0, 13.0, 14.0, 15.0), # id 2 ) def _initializer(shape, dtype, partition_info=None): del shape, dtype, partition_info return embedding_values # price has 1 dimension in dense_features price = tf.feature_column.numeric_column("price") # one_hot_body_style has 3 dims in dense_features. body_style = tf.feature_column.categorical_column_with_vocabulary_list( "body-style", vocabulary_list=["hardtop", "wagon", "sedan"] ) one_hot_body_style = tf.feature_column.indicator_column(body_style) # embedded_body_style has 5 dims in dense_features. country = tf.feature_column.categorical_column_with_vocabulary_list( "country", vocabulary_list=["US", "JP", "CA"] ) embedded_country = tf.feature_column.embedding_column( country, dimension=5, initializer=_initializer ) with tf.Graph().as_default(): # Provides 1-dim tensor and dense tensor. features = { "price": tf.constant( [ 11.0, 12.0, ] ), "body-style": tf.SparseTensor( indices=((0,), (1,)), values=("sedan", "hardtop"), dense_shape=(2,), ), # This is dense tensor for the categorical_column. "country": tf.constant(["CA", "US"]), } self.assertEqual(1, features["price"].shape.ndims) self.assertEqual( 1, features["body-style"].dense_shape.get_shape()[0] ) self.assertEqual(1, features["country"].shape.ndims) net = df.DenseFeatures( [price, one_hot_body_style, embedded_country] )(features) self.assertEqual(1 + 3 + 5, net.shape[1]) with _initialized_session() as sess: # Each row is formed by concatenating `embedded_body_style`, # `one_hot_body_style`, and `price` in order. self.assertAllEqual( [ [0.0, 0.0, 1.0, 11.0, 12.0, 13.0, 14.0, 15.0, 11.0], [1.0, 0.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 12.0], ], sess.run(net), )
def test_TensorBoard(self): np.random.seed(1337) temp_dir = self.get_temp_dir() self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True) (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data( train_samples=TRAIN_SAMPLES, test_samples=TEST_SAMPLES, input_shape=(INPUT_DIM, ), num_classes=NUM_CLASSES) y_test = np_utils.to_categorical(y_test) y_train = np_utils.to_categorical(y_train) def data_generator(train): if train: max_batch_index = len(x_train) // BATCH_SIZE else: max_batch_index = len(x_test) // BATCH_SIZE i = 0 while 1: if train: yield (x_train[i * BATCH_SIZE:(i + 1) * BATCH_SIZE], y_train[i * BATCH_SIZE:(i + 1) * BATCH_SIZE]) else: yield (x_test[i * BATCH_SIZE:(i + 1) * BATCH_SIZE], y_test[i * BATCH_SIZE:(i + 1) * BATCH_SIZE]) i += 1 i %= max_batch_index # case: Sequential with tf.Graph().as_default(), self.cached_session(): model = sequential.Sequential() model.add( layers.Dense(NUM_HIDDEN, input_dim=INPUT_DIM, activation='relu')) # non_trainable_weights: moving_variance, moving_mean model.add(layers.BatchNormalization()) model.add(layers.Dense(NUM_CLASSES, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy']) tsb = callbacks_v1.TensorBoard(log_dir=temp_dir, histogram_freq=1, write_images=True, write_grads=True, batch_size=5) cbks = [tsb] # fit with validation data model.fit(x_train, y_train, batch_size=BATCH_SIZE, validation_data=(x_test, y_test), callbacks=cbks, epochs=3, verbose=0) # fit with validation data and accuracy model.fit(x_train, y_train, batch_size=BATCH_SIZE, validation_data=(x_test, y_test), callbacks=cbks, epochs=2, verbose=0) # fit generator with validation data model.fit_generator(data_generator(True), len(x_train), epochs=2, validation_data=(x_test, y_test), callbacks=cbks, verbose=0) # fit generator without validation data # histogram_freq must be zero tsb.histogram_freq = 0 model.fit_generator(data_generator(True), len(x_train), epochs=2, callbacks=cbks, verbose=0) # fit generator with validation data and accuracy tsb.histogram_freq = 1 model.fit_generator(data_generator(True), len(x_train), epochs=2, validation_data=(x_test, y_test), callbacks=cbks, verbose=0) # fit generator without validation data and accuracy tsb.histogram_freq = 0 model.fit_generator(data_generator(True), len(x_train), epochs=2, callbacks=cbks) assert os.path.exists(temp_dir)
def testSparse(self): # TODO(tanzheny, omalleyt): Fix test in eager mode. sparse_epsilon = 1e-7 for dtype in [tf.half, tf.float32, tf.float64]: with tf.Graph().as_default(), self.cached_session(): # Initialize variables for numpy implementation. m0, v0, m1, v1, mcache = 0.0, 0.0, 0.0, 0.0, 1.0 var0_np = np.array([1.0, 1.0, 2.0], dtype=dtype.as_numpy_dtype) grads0_np = np.array([0.1, 0, 0.1], dtype=dtype.as_numpy_dtype) var1_np = np.array([3.0, 3.0, 4.0], dtype=dtype.as_numpy_dtype) grads1_np = np.array([0.01, 0, 0.01], dtype=dtype.as_numpy_dtype) var0 = tf.Variable(var0_np) var1 = tf.Variable(var1_np) grads0_np_indices = np.array([0, 2], dtype=np.int32) grads0 = tf.IndexedSlices( tf.constant(grads0_np[grads0_np_indices]), tf.constant(grads0_np_indices), tf.constant([3])) grads1_np_indices = np.array([0, 2], dtype=np.int32) grads1 = tf.IndexedSlices( tf.constant(grads1_np[grads1_np_indices]), tf.constant(grads1_np_indices), tf.constant([3])) opt = nadam.Nadam(epsilon=sparse_epsilon) update = opt.apply_gradients( zip([grads0, grads1], [var0, var1])) self.evaluate(tf.compat.v1.global_variables_initializer()) # Fetch params to validate initial values self.assertAllClose([1.0, 1.0, 2.0], var0) self.assertAllClose([3.0, 3.0, 4.0], var1) beta1_power, beta2_power = get_beta_accumulators(opt, dtype) # Run 3 steps of Nadam for t in range(3): self.assertAllCloseAccordingToType(0.9**(t + 1), beta1_power) self.assertAllCloseAccordingToType(0.999**(t + 1), beta2_power) update.run() mcache = update_m_cache(mcache, t) var0_np, m0, v0 = nadam_update_numpy( var0_np, grads0_np, t, m0, v0, mcache, epsilon=sparse_epsilon) var1_np, m1, v1 = nadam_update_numpy( var1_np, grads1_np, t, m1, v1, mcache, epsilon=sparse_epsilon) # Validate updated params self.assertAllCloseAccordingToType(var0_np, var0) self.assertAllCloseAccordingToType(var1_np, var1)
def test_TensorBoard_update_freq(self): class FileWriterStub(object): def __init__(self, logdir, graph=None): self.logdir = logdir self.graph = graph self.batch_summaries = [] self.epoch_summaries = [] def add_summary(self, summary, step): if 'batch_' in summary.value[0].tag: self.batch_summaries.append((step, summary)) elif 'epoch_' in summary.value[0].tag: self.epoch_summaries.append((step, summary)) def flush(self): pass def close(self): pass with tf.Graph().as_default(): temp_dir = self.get_temp_dir() self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True) # Epoch mode tb_cbk = callbacks_v1.TensorBoard(temp_dir, update_freq='epoch') tb_cbk.writer = FileWriterStub(temp_dir) tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 1}) self.assertEqual(tb_cbk.writer.batch_summaries, []) tb_cbk.on_epoch_end(0, {'acc': 10.0, 'size': 1}) self.assertLen(tb_cbk.writer.epoch_summaries, 1) tb_cbk.on_train_end() # Batch mode tb_cbk = callbacks_v1.TensorBoard(temp_dir, update_freq='batch') tb_cbk.writer = FileWriterStub(temp_dir) tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 1}) self.assertLen(tb_cbk.writer.batch_summaries, 1) tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 1}) self.assertLen(tb_cbk.writer.batch_summaries, 2) self.assertFalse(tb_cbk.writer.epoch_summaries) tb_cbk.on_train_end() # Integer mode tb_cbk = callbacks_v1.TensorBoard(temp_dir, update_freq=20) tb_cbk.writer = FileWriterStub(temp_dir) tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 10}) self.assertFalse(tb_cbk.writer.batch_summaries) tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 10}) self.assertLen(tb_cbk.writer.batch_summaries, 1) tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 10}) self.assertLen(tb_cbk.writer.batch_summaries, 1) tb_cbk.on_batch_end(0, {'acc': 5.0, 'size': 10}) self.assertLen(tb_cbk.writer.batch_summaries, 2) tb_cbk.on_batch_end(0, {'acc': 10.0, 'size': 10}) self.assertLen(tb_cbk.writer.batch_summaries, 2) self.assertFalse(tb_cbk.writer.epoch_summaries) tb_cbk.on_train_end()
def test_multiple_layers_with_same_shared_embedding_column_diff_graphs( self, ): categorical_column_a = ( tf.feature_column.categorical_column_with_identity(key="aaa", num_buckets=3)) categorical_column_b = ( tf.feature_column.categorical_column_with_identity(key="bbb", num_buckets=3)) embedding_dimension = 2 ( embedding_column_b, embedding_column_a, ) = tf.feature_column.shared_embeddings( [categorical_column_b, categorical_column_a], dimension=embedding_dimension, ) all_cols = [embedding_column_a, embedding_column_b] with tf.Graph().as_default(): features = { "aaa": tf.SparseTensor( indices=((0, 0), (1, 0), (1, 1)), values=(0, 1, 0), dense_shape=(2, 2), ), "bbb": tf.SparseTensor( indices=((0, 0), (1, 0), (1, 1)), values=(1, 2, 1), dense_shape=(2, 2), ), } df.DenseFeatures(all_cols)(features) # Make sure that only 1 variable gets created in this case. self.assertEqual( 1, len( tf.compat.v1.get_collection( tf.compat.v1.GraphKeys.GLOBAL_VARIABLES)), ) with tf.Graph().as_default(): features1 = { "aaa": tf.SparseTensor( indices=((0, 0), (1, 0), (1, 1)), values=(0, 1, 0), dense_shape=(2, 2), ), "bbb": tf.SparseTensor( indices=((0, 0), (1, 0), (1, 1)), values=(1, 2, 1), dense_shape=(2, 2), ), } df.DenseFeatures(all_cols)(features1) # Make sure that only 1 variable gets created in this case. self.assertEqual( 1, len( tf.compat.v1.get_collection( tf.compat.v1.GraphKeys.GLOBAL_VARIABLES)), ) self.assertCountEqual( ["aaa_bbb_shared_embedding:0"], [ v.name for v in tf.compat.v1.get_collection( tf.compat.v1.GraphKeys.GLOBAL_VARIABLES) ], )
def testSharing(self): # TODO(tanzheny, omalleyt): Fix test in eager mode. with tf.Graph().as_default(): for dtype in [tf.half, tf.float32, tf.float64]: var0 = tf.Variable([1.0, 2.0], dtype=dtype) var1 = tf.Variable([3.0, 4.0], dtype=dtype) grads0 = tf.constant([0.1, 0.1], dtype=dtype) grads1 = tf.constant([0.01, 0.01], dtype=dtype) mom_opt = gradient_descent.SGD(learning_rate=2.0, momentum=0.9) mom_update1 = mom_opt.apply_gradients( zip([grads0, grads1], [var0, var1])) mom_update2 = mom_opt.apply_gradients( zip([grads0, grads1], [var0, var1])) self.evaluate(tf.compat.v1.global_variables_initializer()) slot0 = mom_opt.get_slot(var0, "momentum") self.assertEqual(slot0.shape, var0.shape) slot1 = mom_opt.get_slot(var1, "momentum") self.assertEqual(slot1.shape, var1.shape) # Fetch params to validate initial values self.assertAllClose([1.0, 2.0], self.evaluate(var0)) self.assertAllClose([3.0, 4.0], self.evaluate(var1)) # Step 1: the momentum accumulators where 0. So we should see a normal # update: v -= grad * learning_rate self.evaluate(mom_update1) # Check that the momentum accumulators have been updated. self.assertAllCloseAccordingToType(np.array([-0.2, -0.2]), self.evaluate(slot0)) self.assertAllCloseAccordingToType(np.array([-0.02, -0.02]), self.evaluate(slot1)) # Check that the parameters have been updated. self.assertAllCloseAccordingToType( np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), self.evaluate(var0), ) self.assertAllCloseAccordingToType( np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), self.evaluate(var1), ) # Step 2: the second momentum accumulators contain the previous update. self.evaluate(mom_update2) # Check that the momentum accumulators have been updated. self.assertAllCloseAccordingToType( np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), self.evaluate(slot0), ) self.assertAllCloseAccordingToType( np.array([ (0.9 * (-0.02) - 2.0 * 0.01), (0.9 * (-0.02) - 2.0 * 0.01), ]), self.evaluate(slot1), ) # Check that the parameters have been updated. self.assertAllCloseAccordingToType( np.array([ 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), ]), self.evaluate(var0), ) self.assertAllCloseAccordingToType( np.array([ 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), 3.98 - ((0.9 * 0.01 + 0.01) * 2.0), ]), self.evaluate(var1), )