def _do_sampling(self, logits, num_samples, sampler): """Samples using the supplied sampler and inputs. Args: logits: Numpy ndarray of shape [batch_size, num_classes]. num_samples: Int; number of samples to draw. sampler: A sampler function that takes (1) a [batch_size, num_classes] Tensor, (2) num_samples and returns a [batch_size, num_samples] Tensor. Returns: Frequencies from sampled classes; shape [batch_size, num_classes]. """ with self.test_session(use_gpu=True) as sess: random_seed.set_random_seed(1618) op = sampler(constant_op.constant(logits), num_samples) d = sess.run(op) batch_size, num_classes = logits.shape freqs_mat = [] for i in range(batch_size): cnts = dict(collections.Counter(d[i, :])) # Requires drawn class labels be in range. self.assertLess(max(cnts.keys()), num_classes) self.assertGreaterEqual(min(cnts.keys()), 0) freqs = [(cnts[k] * 1. / num_samples if k in cnts else 0) for k in range(num_classes)] freqs_mat.append(freqs) return freqs_mat
def testTwoConvLayers(self): if test.is_gpu_available(cuda_only=True): random_seed.set_random_seed(0) x = random_ops.truncated_normal([1, 784], seed=0) output = two_layer_model(x) with session.Session() as sess: output_val_ref = sess.run(output) with session.Session(config=get_config()) as sess: metadata = config_pb2.RunMetadata() output_val = sess.run(output, run_metadata=metadata) nodes = [] num_transposes = 0 for node in metadata.cost_graph.node: if node.name.startswith('LayoutOptimizerTranspose'): num_transposes += 1 nodes.append(node.name) # Four transposes were initially added in the Expand phase of # LayoutOptimizer; two of them are cancelled out in the Collapse phase. expected_num_transposes = 2 self.assertEqual(expected_num_transposes, num_transposes) self.assertIn('LayoutOptimizerTransposeNHWCToNCHW-Conv2D-Reshape-0', nodes) self.assertIn('LayoutOptimizerTransposeNCHWToNHWC-Relu_1-MaxPool_1', nodes) self.assertAllClose(output_val_ref, output_val, atol=1e-3)
def testResumeTrainAchievesRoughlyTheSameLoss(self): number_of_steps = [300, 1, 5] logdir = os.path.join(self.get_temp_dir(), 'resume_train_same_loss') for i in range(len(number_of_steps)): with ops.Graph().as_default(): random_seed.set_random_seed(i) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = logistic_classifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = training.create_train_op(total_loss, optimizer) saver = saver_lib.Saver() loss = training.train( train_op, logdir, hooks=[ basic_session_run_hooks.StopAtStepHook( num_steps=number_of_steps[i]), basic_session_run_hooks.CheckpointSaverHook( logdir, save_steps=50, saver=saver), ]) self.assertIsNotNone(loss) self.assertLess(loss, .015)
def testNegativeMinLogits(self): random_seed.set_random_seed(78844) with self.test_session(use_gpu=True): logits = constant_op.constant([[np.finfo(np.float32).min] * 1023 + [0]]) num_samples = 1000 samples = random_ops.multinomial(logits, num_samples).eval() self.assertAllEqual([[1023] * num_samples], samples)
def _infer_model( self, input_fn, feed_fn=None, outputs=None, as_iterable=False): # Check that model has been trained. checkpoint_path = saver.latest_checkpoint(self._model_dir) if not checkpoint_path: raise NotFittedError("Couldn't find trained model at %s." % self._model_dir) with ops.Graph().as_default() as g: random_seed.set_random_seed(self._config.tf_random_seed) contrib_framework.create_global_step(g) features = self._get_features_from_input_fn(input_fn) predictions = self._get_predict_ops(features) # If predictions is single output - wrap it into dict, and remember to # return not a dict. return_dict = isinstance(predictions, dict) if not return_dict: predictions = {'predictions': predictions} # Filter what to run predictions on, if outputs provided. if outputs: existing_keys = predictions.keys() predictions = { key: value for key, value in predictions.items() if key in outputs } if not predictions: raise ValueError('Expected to run at least one output from %s, ' 'provided %s.' % (existing_keys, outputs)) if as_iterable: return self._infer_model_as_iterable( checkpoint_path, predictions, feed_fn, return_dict) else: return self._infer_model_single( checkpoint_path, predictions, feed_fn, return_dict)
def testEmptyUpdateOps(self): with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = batchnorm_classifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = training.create_train_op(total_loss, optimizer, update_ops=[]) moving_mean = variables_lib.get_variables_by_name('moving_mean')[0] moving_variance = variables_lib.get_variables_by_name('moving_variance')[ 0] with session_lib.Session() as sess: # Initialize all variables sess.run(variables_lib2.global_variables_initializer()) mean, variance = sess.run([moving_mean, moving_variance]) # After initialization moving_mean == 0 and moving_variance == 1. self.assertAllClose(mean, [0] * 4) self.assertAllClose(variance, [1] * 4) for _ in range(10): sess.run([train_op]) mean = moving_mean.eval() variance = moving_variance.eval() # Since we skip update_ops the moving_vars are not updated. self.assertAllClose(mean, [0] * 4) self.assertAllClose(variance, [1] * 4)
def testNoneGlobalStep(self): with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = batchnorm_classifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = training.create_train_op( total_loss, optimizer, global_step=None) global_step = variables_lib.get_or_create_global_step() with session_lib.Session() as sess: # Initialize all variables sess.run(variables_lib2.global_variables_initializer()) for _ in range(10): sess.run([train_op]) global_step = global_step.eval() # Since train_op don't use global_step it shouldn't change. self.assertAllClose(global_step, 0)
def testRegisterCategoricalPredictiveDistributionBatchSize1(self): with ops.Graph().as_default(): random_seed.set_random_seed(200) logits = random_ops.random_normal((1, 2)) lc = layer_collection.LayerCollection() lc.register_categorical_predictive_distribution(logits, seed=200)
def testRegisterBlocks(self): with ops.Graph().as_default(): random_seed.set_random_seed(200) lc = layer_collection.LayerCollection() lc.register_fully_connected( array_ops.constant(1), array_ops.constant(2), array_ops.constant(3)) lc.register_fully_connected( array_ops.constant(1), array_ops.constant(2), array_ops.constant(3), approx=layer_collection.APPROX_DIAGONAL_NAME) lc.register_conv2d( array_ops.constant(4), [1, 1, 1, 1], 'SAME', array_ops.ones((1, 1, 1, 1)), array_ops.constant(3)) lc.register_conv2d( array_ops.constant(4), [1, 1, 1, 1], 'SAME', array_ops.ones((1, 1, 1, 1)), array_ops.constant(3), approx=layer_collection.APPROX_DIAGONAL_NAME) lc.register_generic( array_ops.constant(5), 16, approx=layer_collection.APPROX_FULL_NAME) lc.register_generic( array_ops.constant(6), 16, approx=layer_collection.APPROX_DIAGONAL_NAME) self.assertEqual(6, len(lc.get_blocks()))
def testSampleMultipleTimes(self): # 5 component mixture. logits = [-10.0, -5.0, 0.0, 5.0, 10.0] mus = [-5.0, 0.0, 5.0, 4.0, 20.0] sigmas = [0.1, 5.0, 3.0, 0.2, 4.0] with self.test_session(): n = 100 random_seed.set_random_seed(654321) components = [ ds.Normal( loc=mu, scale=sigma) for mu, sigma in zip(mus, sigmas) ] cat = ds.Categorical( logits, dtype=dtypes.int32, name="cat1") dist1 = ds.Mixture(cat, components, name="mixture1", use_static_graph=self.use_static_graph) samples1 = dist1.sample(n, seed=123456).eval() random_seed.set_random_seed(654321) components2 = [ ds.Normal( loc=mu, scale=sigma) for mu, sigma in zip(mus, sigmas) ] cat2 = ds.Categorical( logits, dtype=dtypes.int32, name="cat2") dist2 = ds.Mixture(cat2, components2, name="mixture2", use_static_graph=self.use_static_graph) samples2 = dist2.sample(n, seed=123456).eval() self.assertAllClose(samples1, samples2)
def _runSamplingBenchmark(self, name, create_distribution, use_gpu, num_components, batch_size, num_features, sample_size): config = config_pb2.ConfigProto() config.allow_soft_placement = True np.random.seed(127) with session.Session(config=config, graph=ops.Graph()) as sess: random_seed.set_random_seed(0) with ops.device("/device:GPU:0" if use_gpu else "/cpu:0"): mixture = create_distribution( num_components=num_components, batch_size=batch_size, num_features=num_features) sample_op = mixture.sample(sample_size).op sess.run(variables.global_variables_initializer()) reported = self.run_op_benchmark( sess, sample_op, min_iters=10, name=("%s_%s_components_%d_batch_%d_features_%d_sample_%d" % (name, use_gpu, num_components, batch_size, num_features, sample_size))) logging.vlog(2, "\t".join(["%s", "%d", "%d", "%d", "%d", "%g"]) % ( use_gpu, num_components, batch_size, num_features, sample_size, reported["wall_time"]))
def testSampleFromDatasets(self): random_seed.set_random_seed(1618) num_samples = 10000 rand_probs = self._normalize(np.random.random_sample((10,))) rand_probs2 = self._normalize(np.random.random_sample((15,))) for probs in [[.5, .5], [.85, .05, .1], rand_probs, rand_probs2]: probs = np.asarray(probs) # Create a dataset that samples each integer in `[0, probs.shape[0])` # with probability given by `probs[i]`. dataset = interleave_ops.sample_from_datasets([ dataset_ops.Dataset.from_tensors(i).repeat(None) for i in range(probs.shape[0]) ], probs) dataset = dataset.take(num_samples) iterator = dataset.make_one_shot_iterator() next_element = iterator.get_next() with self.test_session() as sess: freqs = np.zeros_like(probs) for _ in range(num_samples): freqs[sess.run(next_element)] += 1 with self.assertRaises(errors.OutOfRangeError): sess.run(next_element) # Use chi-squared test to assert that the observed distribution # matches the expected distribution. Based on the implementation # in "tensorflow/python/kernel_tests/multinomial_op_test.py". self.assertLess(self._chi2(probs, freqs / num_samples), 1e-3)
def testGradient(self): if not test.is_gpu_available(cuda_only=True): self.skipTest('GPU required') random_seed.set_random_seed(0) x = random_ops.truncated_normal([1, 200, 200, 3], seed=0) y = conv_layers.conv2d(x, 32, [3, 3]) z = conv_layers.conv2d(y, 32, [3, 3]) optimizer = gradient_descent.GradientDescentOptimizer(1e-4) loss = math_ops.reduce_mean(z) train_op = optimizer.minimize(loss) graph = ops.get_default_graph() graph.add_to_collection('train_op', train_op) meta_graph = saver_lib.export_meta_graph(graph_def=graph.as_graph_def()) rewrite_options = rewriter_config_pb2.RewriterConfig( optimize_tensor_layout=True) optimized_graph = tf_optimizer.OptimizeGraph(rewrite_options, meta_graph) found = 0 for node in optimized_graph.node: if node.op in ['Conv2D', 'Conv2DBackpropFilter', 'Conv2DBackpropInput']: found += 1 self.assertEqual(node.attr['data_format'].s, 'NCHW') self.assertEqual(found, 5)
def testTrainWithTrace(self): logdir = os.path.join( tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = LogisticClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() summary.scalar('total_loss', total_loss) optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) loss = learning.train( train_op, logdir, number_of_steps=300, log_every_n_steps=10, trace_every_n_steps=100) self.assertIsNotNone(loss) for trace_step in [1, 101, 201]: trace_filename = 'tf_trace-%d.json' % trace_step self.assertTrue(os.path.isfile(os.path.join(logdir, trace_filename)))
def testResumeTrainAchievesRoughlyTheSameLoss(self): logdir = os.path.join( tempfile.mkdtemp(prefix=self.get_temp_dir()), 'tmp_logs') number_of_steps = [300, 301, 305] for i in range(len(number_of_steps)): with ops.Graph().as_default(): random_seed.set_random_seed(i) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = LogisticClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) loss = learning.train( train_op, logdir, number_of_steps=number_of_steps[i], log_every_n_steps=10) self.assertIsNotNone(loss) self.assertLess(loss, .015)
def testMultiplyInverseAgainstExplicit(self): with ops.Graph().as_default(), self.test_session() as sess: random_seed.set_random_seed(200) params = array_ops.zeros((2, 2, 2, 2)) inputs = array_ops.zeros((2, 2, 2, 2)) outputs = array_ops.zeros((2, 2, 2, 2)) block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, (1, 1, 1, 1), 'SAME') block.register_additional_minibatch(inputs, outputs) grads = outputs**2 damping = 0. # This test is only valid without damping. block.instantiate_factors(([grads],), damping) sess.run(state_ops.assign(block._input_factor._cov, _make_psd(8))) sess.run(state_ops.assign(block._output_factor._cov, _make_psd(2))) sess.run(block._input_factor.make_inverse_update_ops()) sess.run(block._output_factor.make_inverse_update_ops()) v_flat = np.arange(16, dtype=np.float32) vector = utils.column_to_tensors(params, array_ops.constant(v_flat)) output = block.multiply_inverse(vector) output_flat = sess.run(utils.tensors_to_column(output)).ravel() full = sess.run(block.full_fisher_block()) explicit = np.dot(np.linalg.inv(full + damping * np.eye(16)), v_flat) self.assertAllClose(output_flat, explicit)
def testTrainWithSessionWrapper(self): """Test that slim.learning.train can take `session_wrapper` args. One of the applications of `session_wrapper` is the wrappers of TensorFlow Debugger (tfdbg), which intercept methods calls to `tf.Session` (e.g., run) to achieve debugging. `DumpingDebugWrapperSession` is used here for testing purpose. """ dump_root = tempfile.mkdtemp() def dumping_wrapper(sess): # pylint: disable=invalid-name return dumping_wrapper_lib.DumpingDebugWrapperSession(sess, dump_root) with ops.Graph().as_default(): random_seed.set_random_seed(0) tf_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32) tf_labels = constant_op.constant(self._labels, dtype=dtypes.float32) tf_predictions = LogisticClassifier(tf_inputs) loss_ops.log_loss(tf_predictions, tf_labels) total_loss = loss_ops.get_total_loss() optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0) train_op = learning.create_train_op(total_loss, optimizer) loss = learning.train( train_op, None, number_of_steps=1, session_wrapper=dumping_wrapper) self.assertIsNotNone(loss) run_root = glob.glob(os.path.join(dump_root, 'run_*'))[-1] dump = debug_data.DebugDumpDir(run_root) self.assertAllEqual(0, dump.get_tensors('global_step', 0, 'DebugIdentity')[0])
def testMultiplyInverseTuple(self): with ops.Graph().as_default(), self.test_session() as sess: random_seed.set_random_seed(200) inputs = array_ops.constant([[1., 2., 3.], [3., 4., 5.], [5., 6., 7.]]) outputs = array_ops.constant([[3., 4.], [5., 6.]]) block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=False) block.register_additional_minibatch(inputs, outputs) grads = outputs**2 block.instantiate_factors(([grads],), 0.5) # Make sure our inverse is something other than the identity. sess.run(tf_variables.global_variables_initializer()) sess.run(block._input_factor.make_inverse_update_ops()) sess.run(block._output_factor.make_inverse_update_ops()) vector = ( np.arange(2, 6).reshape(2, 2).astype(np.float32), # np.arange(1, 3).reshape(2, 1).astype(np.float32)) output = block.multiply_inverse((array_ops.constant(vector[0]), array_ops.constant(vector[1]))) output = sess.run(output) self.assertAllClose([[0.686291, 1.029437], [1.372583, 1.715729]], output[0]) self.assertAllClose([0.343146, 0.686291], output[1])
def testMultiplyInverseTuple(self): with ops.Graph().as_default(), self.test_session() as sess: random_seed.set_random_seed(200) params = random_ops.random_normal((2, 2, 2, 2)) inputs = random_ops.random_normal((2, 2, 2, 2)) outputs = random_ops.random_normal((2, 2, 2, 2)) block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, (1, 1, 1, 1), 'SAME') block.register_additional_minibatch(inputs, outputs) grads = outputs**2 block.instantiate_factors(([grads],), 0.5) # Make sure our inverse is something other than the identity. sess.run(tf_variables.global_variables_initializer()) sess.run(block._input_factor.make_inverse_update_ops()) sess.run(block._output_factor.make_inverse_update_ops()) vector = (np.arange(1, 15).reshape(7, 2).astype(np.float32), np.arange(2, 4).reshape(2, 1).astype(np.float32)) output = block.multiply_inverse((array_ops.constant(vector[0]), array_ops.constant(vector[1]))) output = sess.run(output) self.assertAllClose([0.136455, 0.27291], output[0][0]) self.assertAllClose([0.27291, 0.409365], output[1])
def testSliceWithNonConstAxis(self): if test.is_gpu_available(cuda_only=True): random_seed.set_random_seed(0) x = random_ops.truncated_normal([1, 784], seed=0) conv = _two_layer_model(x) size = array_ops.placeholder(dtype='int32') s = array_ops.slice(conv, [0, 0, 0, 0], size) output = array_ops.identity(s) size_val = [1, 2, 3, 4] with session.Session() as sess: output_val_ref = sess.run(output, feed_dict={size: size_val}) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() output_val = sess.run( output, run_metadata=metadata, feed_dict={ size: size_val }) nodes = [] num_transposes = 0 for node in metadata.cost_graph.node: if _is_transpose(node.name): num_transposes += 1 nodes.append(node.name) # Four transposes were initially added in the Expand phase of # LayoutOptimizer; two of them are cancelled out in the Collapse phase. expected_num_transposes = 2 self.assertEqual(expected_num_transposes, num_transposes) self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes) self._assert_trans_nchw_to_nhwc('Slice-0-0', nodes) self._assert_vec_nhwc_to_nchw('Slice-2', nodes) self.assertAllClose(output_val_ref, output_val, atol=1e-3)
def testStridedSliceWithMask1011(self): if test.is_gpu_available(cuda_only=True): random_seed.set_random_seed(0) x = random_ops.truncated_normal([1, 784], seed=0) conv = _two_layer_model(x) # This will generate a StridedSlice op with begin mask and # end mask 11(1011). s = conv[:, :, 1:-1, :] output = array_ops.identity(s) with session.Session() as sess: output_val_ref = sess.run(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() output_val = sess.run(output, run_metadata=metadata) nodes = [] num_transposes = 0 for node in metadata.cost_graph.node: if _is_transpose(node.name): num_transposes += 1 nodes.append(node.name) # Four transposes were initially added in the Expand phase of # LayoutOptimizer; two of them are cancelled out in the Collapse phase. expected_num_transposes = 2 self.assertEqual(expected_num_transposes, num_transposes) self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes) self._assert_trans_nchw_to_nhwc('strided_slice-0-0', nodes) self.assertIn('strided_slice-1-LayoutOptimizer', nodes) self.assertIn('strided_slice-2-LayoutOptimizer', nodes) self.assertIn('strided_slice-3-LayoutOptimizer', nodes) self.assertAllClose(output_val_ref, output_val, atol=1e-3)
def testSelectOpScalarCondition(self): if test.is_gpu_available(cuda_only=True): random_seed.set_random_seed(0) x = random_ops.truncated_normal([1, 784], seed=0) conv = _two_layer_model(x) add = math_ops.add(conv, conv) condition = constant_op.constant(True) select = gen_math_ops._select(condition, conv, add) output = array_ops.identity(select) with session.Session() as sess: output_val_ref = sess.run(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() output_val = sess.run(output, run_metadata=metadata) nodes = [] num_transposes = 0 for node in metadata.cost_graph.node: if _is_transpose(node.name): num_transposes += 1 nodes.append(node.name) expected_num_transposes = 2 self.assertEqual(expected_num_transposes, num_transposes) self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes) self._assert_trans_nchw_to_nhwc('Select-0-0', nodes) self.assertAllClose(output_val_ref, output_val, atol=1e-3)
def testConcatWithControlDependency(self): if test.is_gpu_available(cuda_only=True): random_seed.set_random_seed(0) x = random_ops.truncated_normal([1, 784], seed=0) conv = _two_layer_model(x) axis = constant_op.constant(3) var = variables.Variable(3) assign = state_ops.assign(var, 6) with ops.control_dependencies([assign]): concat = array_ops.concat([conv, conv], axis) output = array_ops.identity(concat) with session.Session() as sess: output_val_ref = sess.run(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() output_val = sess.run(output, run_metadata=metadata) nodes = [] num_transposes = 0 for node in metadata.cost_graph.node: if _is_transpose(node.name): num_transposes += 1 nodes.append(node.name) # Four transposes were initially added in the Expand phase of # LayoutOptimizer; two of them are cancelled out in the Collapse phase. expected_num_transposes = 2 self.assertEqual(expected_num_transposes, num_transposes) self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes) self._assert_trans_nchw_to_nhwc('concat-0-0', nodes) self.assertIn('concat-2-LayoutOptimizer', nodes) self.assertAllClose(output_val_ref, output_val, atol=1e-3)
def testSplitVWithNonConstAxis(self): if test.is_gpu_available(cuda_only=True): random_seed.set_random_seed(0) x = random_ops.truncated_normal([1, 784], seed=0) conv = _two_layer_model(x) dim = array_ops.placeholder(dtype='int32') sizes = constant_op.constant([50, 10, 4], shape=[3]) split = gen_array_ops._split_v( value=conv, size_splits=sizes, axis=dim, num_split=3) output = math_ops.reduce_sum(split[0]) with session.Session() as sess: output_val_ref = sess.run(output, feed_dict={dim: 3}) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() output_val = sess.run(output, run_metadata=metadata, feed_dict={dim: 3}) nodes = [] num_transposes = 0 for node in metadata.cost_graph.node: if _is_transpose(node.name): num_transposes += 1 nodes.append(node.name) # Four transposes were initially added in the Expand phase of # LayoutOptimizer; two of them are cancelled out in the Collapse phase. expected_num_transposes = 2 self.assertEqual(expected_num_transposes, num_transposes) self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes) self._assert_trans_nchw_to_nhwc('SplitV-0-0', nodes) self._assert_map_nhwc_to_nchw('SplitV-2', nodes) self.assertAllClose(output_val_ref, output_val, atol=1e-3)
def testReduceSumAlongC(self): if test.is_gpu_available(cuda_only=True): random_seed.set_random_seed(0) x = random_ops.truncated_normal([1, 784], seed=0) conv = _two_layer_model(x) reduce_sum = math_ops.reduce_sum(conv, axis=[3]) output = array_ops.identity(reduce_sum) with session.Session() as sess: output_val_ref = sess.run(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() output_val = sess.run(output, run_metadata=metadata) nodes = [] num_transposes = 0 for node in metadata.cost_graph.node: if _is_transpose(node.name): num_transposes += 1 nodes.append(node.name) # Three transposes were initially added in the Expand phase of # LayoutOptimizer; two of them are cancelled out in the Collapse phase. expected_num_transposes = 1 self.assertEqual(expected_num_transposes, num_transposes) self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes) self.assertAllClose(output_val_ref, output_val, atol=1e-3)
def _save_first_checkpoint(keras_model, estimator, custom_objects, keras_weights): """Save first checkpoint for the keras Estimator. Args: keras_model: an instance of compiled keras model. estimator: keras estimator. custom_objects: Dictionary for custom objects. keras_weights: A flat list of Numpy arrays for weights of given keras_model. Returns: The model_fn for a keras Estimator. """ # Load weights and save to checkpoint if there is no checkpoint latest_path = saver_lib.latest_checkpoint(estimator.model_dir) if not latest_path: with ops.Graph().as_default(): random_seed.set_random_seed(estimator.config.tf_random_seed) training_util.create_global_step() model = _clone_and_build_model(model_fn_lib.ModeKeys.TRAIN, keras_model, custom_objects) # save to checkpoint with session.Session(config=estimator._session_config) as sess: if keras_weights: model.set_weights(keras_weights) # Make update ops and initialize all variables. if not model.train_function: # pylint: disable=protected-access model._make_train_function() K._initialize_variables(sess) # pylint: enable=protected-access saver = saver_lib.Saver() saver.save(sess, os.path.join(estimator.model_dir, 'keras_model.ckpt'))
def testSplitWithNonConstAxis(self): if test.is_gpu_available(cuda_only=True): random_seed.set_random_seed(0) x = random_ops.truncated_normal([1, 784], seed=0) conv = _two_layer_model(x) dim = array_ops.placeholder(dtype='int32') split = array_ops.split(conv, 2, axis=dim) scale = constant_op.constant(0.1, shape=[32]) offset = constant_op.constant(0.3, shape=[32]) bn0 = nn.fused_batch_norm(split[0], scale, offset) bn1 = nn.fused_batch_norm(split[1], scale, offset) add = bn0[0] + bn1[0] output = array_ops.identity(add) with session.Session() as sess: output_val_ref = sess.run(output, feed_dict={dim: 3}) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() output_val = sess.run(output, run_metadata=metadata, feed_dict={dim: 3}) nodes = [] num_transposes = 0 for node in metadata.cost_graph.node: if _is_transpose(node.name): num_transposes += 1 nodes.append(node.name) expected_num_transposes = 2 self.assertEqual(expected_num_transposes, num_transposes) self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes) self._assert_trans_nchw_to_nhwc('add_2-0-0', nodes) self._assert_map_nhwc_to_nchw('split-0', nodes) self.assertAllClose(output_val_ref, output_val, atol=1e-3)
def testReverseWithConstDims(self): if test.is_gpu_available(cuda_only=True): random_seed.set_random_seed(0) x = random_ops.truncated_normal([1, 784], seed=0) conv = _two_layer_model(x) dims = constant_op.constant([3, 1], name='DimsConst') reverse = array_ops.reverse(conv, dims) output = array_ops.identity(reverse) with session.Session() as sess: output_val_ref = sess.run(output) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() output_val = sess.run(output, run_metadata=metadata) nodes = [] num_transposes = 0 for node in metadata.cost_graph.node: if _is_transpose(node.name): num_transposes += 1 nodes.append(node.name) # Four transposes were initially added in the Expand phase of # LayoutOptimizer; two of them are cancelled out in the Collapse phase. expected_num_transposes = 2 self.assertEqual(expected_num_transposes, num_transposes) self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes) self._assert_trans_nchw_to_nhwc('ReverseV2-0-0', nodes) self.assertIn('ReverseV2-1-LayoutOptimizer', nodes) self.assertAllClose(output_val_ref, output_val, atol=1e-3)
def testAtrousFullyConvolutionalValues(self): """Verify dense feature extraction with atrous convolution.""" nominal_stride = 32 for output_stride in [4, 8, 16, 32, None]: with arg_scope(resnet_utils.resnet_arg_scope()): with ops.Graph().as_default(): with self.test_session() as sess: random_seed.set_random_seed(0) inputs = create_test_input(2, 81, 81, 3) # Dense feature extraction followed by subsampling. output, _ = self._resnet_small( inputs, None, is_training=False, global_pool=False, output_stride=output_stride) if output_stride is None: factor = 1 else: factor = nominal_stride // output_stride output = resnet_utils.subsample(output, factor) # Make the two networks use the same weights. variable_scope.get_variable_scope().reuse_variables() # Feature extraction at the nominal network rate. expected, _ = self._resnet_small( inputs, None, is_training=False, global_pool=False) sess.run(variables.global_variables_initializer()) self.assertAllClose( output.eval(), expected.eval(), atol=1e-4, rtol=1e-4)
def testMaxPoolV2(self): if test.is_gpu_available(cuda_only=True): random_seed.set_random_seed(0) x = random_ops.truncated_normal([1, 784], seed=0) conv = _two_layer_model(x) ksize = constant_op.constant([1, 2, 3, 1], shape=[4]) strides = array_ops.placeholder(dtype='int32', shape=[4]) max_pool = gen_nn_ops._max_pool_v2(conv, ksize, strides, 'VALID') output = array_ops.identity(max_pool) strides_val = [1, 3, 2, 1] with session.Session() as sess: output_val_ref = sess.run(output, feed_dict={strides: strides_val}) with session.Session(config=_get_config()) as sess: metadata = config_pb2.RunMetadata() output_val = sess.run( output, run_metadata=metadata, feed_dict={ strides: strides_val }) nodes = [] num_transposes = 0 for node in metadata.cost_graph.node: if _is_transpose(node.name): num_transposes += 1 nodes.append(node.name) expected_num_transposes = 2 self.assertEqual(expected_num_transposes, num_transposes) self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes) self._assert_trans_nchw_to_nhwc('MaxPoolV2-0-0', nodes) self._assert_vec_nhwc_to_nchw('MaxPoolV2-2', nodes) self.assertIn('MaxPoolV2-1-LayoutOptimizer', nodes) self.assertAllClose(output_val_ref, output_val, atol=1e-3)
def test_model_equivalent_to_chained_model_chunk_size_five(self): numpy.random.seed(4) random_seed.set_random_seed(5) self._model_equivalent_to_chained_model_test_template(5)
from __future__ import absolute_import from __future__ import division from __future__ import print_function import numpy as np from tensorflow.contrib import linalg as linalg_lib from tensorflow.contrib.linalg.python.ops import linear_operator_test_util from tensorflow.python.framework import dtypes from tensorflow.python.framework import random_seed from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.platform import test linalg = linalg_lib random_seed.set_random_seed(23) rng = np.random.RandomState(0) class BaseLinearOperatorUDVHUpdatetest(object): """Base test for this type of operator.""" # Subclasses should set these attributes to either True or False. # If True, A = L + UDV^H # If False, A = L + UV^H or A = L + UU^H, depending on _use_v. _use_diag_perturbation = None # If True, diag is > 0, which means D is symmetric positive definite. _is_diag_positive = None
def custom_predict(self, perturb, ranker, input_fn, predict_keys=None, hooks=None, checkpoint_path=None, yield_single_examples=True): if not checkpoint_path: checkpoint_path = checkpoint_management.latest_checkpoint( ranker._model_dir) if not checkpoint_path: logging.info( 'Could not find trained model in model_dir: {}, running ' 'initialization to predict.'.format(ranker._model_dir)) with tf.Graph().as_default() as g: self.perturb_on = tf.compat.v1.placeholder(tf.bool) random_seed.set_random_seed(ranker._config.tf_random_seed) ranker._create_and_assert_global_step(g) features, input_hooks = ranker._get_features_from_input_fn( input_fn, ModeKeys.PREDICT) estimator_spec = ranker._call_model_fn(features, None, ModeKeys.PREDICT, ranker.config) # Call to warm_start has to be after model_fn is called. ranker._maybe_warm_start(checkpoint_path) predictions = estimator_spec.predictions all_hooks = list(input_hooks) all_hooks.extend(list([])) self.grad_variable_pair_tensor = calculate_grad_var_pair(self) with training.MonitoredSession( session_creator=training.ChiefSessionCreator( checkpoint_filename_with_path=checkpoint_path, master=ranker._config.master, scaffold=estimator_spec.scaffold, config=ranker._session_config), hooks=all_hooks) as mon_sess: while not mon_sess.should_stop(): [ preds_evaluated, temp_query_features_evaluated, temp_answer_features_evaluated, temp_embedded_features_evaluated, temp_labels_evaluated, temp_normalized_features_evaluated, self.grad_variable_pair_evaluated, ] = mon_sess.run([ predictions, self.query_features, self.answer_features, self.embedded_features_tensor, self.labels_tensor, self.normalized_features, self.grad_variable_pair_tensor, ], {self.perturb_on: perturb}) # Save values for tensors during first nonperturbed evaluation to be # used in next execution. if self.first_eval: self.query_features_evaluated = temp_query_features_evaluated self.answer_features_evaluated = temp_answer_features_evaluated self.embedded_features_evaluated = temp_embedded_features_evaluated self.labels_evaluated = temp_labels_evaluated self.normalized_features_evaluated = temp_normalized_features_evaluated self.first_eval = False if not yield_single_examples: yield preds_evaluated elif not isinstance(predictions, dict): for pred in preds_evaluated: yield pred else: for i in range( self._extract_batch_length(preds_evaluated)): yield { key: value[i] for key, value in six.iteritems( preds_evaluated) }
def _setup_training(self): """Sets up graph, model and trainer.""" # Create config if not given. if self._config is None: self._config = RunConfig(verbose=self.verbose) # Create new graph. self._graph = ops.Graph() self._graph.add_to_collection("IS_TRAINING", True) with self._graph.as_default(): random_seed.set_random_seed(self._config.tf_random_seed) self._global_step = variables.Variable(0, name="global_step", trainable=False) # Setting up inputs and outputs. self._inp, self._out = self._data_feeder.input_builder() # If class weights are provided, add them to the graph. # Different loss functions can use this tensor by name. if self.class_weight: self._class_weight_node = constant_op.constant( self.class_weight, name='class_weight') # Add histograms for X and y if they are floats. if self._data_feeder.input_dtype in (np.float32, np.float64): logging_ops.histogram_summary("X", self._inp) if self._data_feeder.output_dtype in (np.float32, np.float64): logging_ops.histogram_summary("y", self._out) # Create model's graph. self._model_predictions, self._model_loss = self.model_fn( self._inp, self._out) # Create trainer and augment graph with gradients and optimizer. # Additionally creates initialization ops. learning_rate = self.learning_rate optimizer = self.optimizer if callable(learning_rate): learning_rate = learning_rate(self._global_step) if callable(optimizer): optimizer = optimizer(learning_rate) self._train = optimizers.optimize_loss( self._model_loss, self._global_step, learning_rate=learning_rate, optimizer=optimizer, clip_gradients=self.clip_gradients) # Update ops during training, e.g. batch_norm_ops self._train = control_flow_ops.group( self._train, *ops.get_collection('update_ops')) # Merge all summaries into single tensor. self._summaries = logging_ops.merge_all_summaries() # Get all initializers for all trainable variables. self._initializers = variables.initialize_all_variables() # Create model's saver capturing all the nodes created up until now. self._saver = train.Saver( max_to_keep=self._config.keep_checkpoint_max, keep_checkpoint_every_n_hours=self._config. keep_checkpoint_every_n_hours) # Enable monitor to create validation data dict with appropriate tf placeholders self._monitor.create_val_feed_dict(self._inp, self._out) # Create session to run model with. self._session = session.Session(self._config.tf_master, config=self._config.tf_config) # Run parameter initializers. self._session.run(self._initializers)
def setUp(self): super(TestDistributionStrategyDnnCorrectness, self).setUp() v2_compat.enable_v2_behavior() np.random.seed(_RANDOM_SEED) random_seed.set_random_seed(_RANDOM_SEED)
def setUp(self): self._shape_param = 5. self._rate_param = 10. random_seed.set_random_seed(10003) np.random.seed(10003)
def setUp(self): np.random.seed(_RANDOM_SEED) random_seed.set_random_seed(_RANDOM_SEED) super(TestSavedModel, self).setUp()
def _infer_model(self, mode, input_fn=None, predict_keys=None, hooks=None, checkpoint_path=None): """Returns predictions for given features given an inference mode. Args: mode: The inference to use, possible values: PREDICT, GENERATE, ENCODE. input_fn: Input function returning features which is a dictionary of string feature name to `Tensor` or `SparseTensor`. If it returns a tuple, first item is extracted as features. Prediction continues until `input_fn` raises an end-of-input exception (`OutOfRangeError` or `StopIteration`). predict_keys: list of `str`, name of the keys to predict. It is used if the `EstimatorSpec.predictions` is a `dict`. If `predict_keys` is used then rest of the predictions will be filtered from the dictionary. If `None`, returns all. hooks: List of `SessionRunHook` subclass instances. Used for callbacks inside the prediction call. checkpoint_path: Path of a specific checkpoint to predict. If `None`, the latest checkpoint in `model_dir` is used. Yields: Evaluated values of `predictions` tensors. Raises: ValueError: Could not find a trained model in model_dir. ValueError: if batch length of predictions are not same. ValueError: If there is a conflict between `predict_keys` and `predictions`. For example if `predict_keys` is not `None` but `EstimatorSpec.predictions` is not a `dict`. """ hooks = self._check_hooks(hooks) # Check that model has been trained. if not checkpoint_path: checkpoint_path = saver.latest_checkpoint(self._model_dir) if not checkpoint_path: raise ValueError("Could not find trained model at %s." % self._model_dir) with ops.Graph().as_default() as g: random_seed.set_random_seed(self._config.tf_random_seed) training.get_or_create_global_step(g) features = self._get_features_from_input_fn(input_fn) estimator_spec = self._call_model_fn(features, None, mode) predictions = self._extract_keys(estimator_spec.predictions, predict_keys) with monitored_session.MonitoredSession( session_creator=monitored_session.ChiefSessionCreator( checkpoint_filename_with_path=checkpoint_path, scaffold=estimator_spec.scaffold, config=self._session_config), hooks=hooks) as mon_sess: while not mon_sess.should_stop(): preds_evaluated = mon_sess.run(predictions) if not isinstance(predictions, dict): for pred in preds_evaluated: yield pred else: for i in xrange(extract_batch_length(preds_evaluated)): yield { key: value[i] for key, value in six.iteritems( preds_evaluated) }
def export_savedmodel(self, export_dir_base, serving_input_receiver_fn, assets_extra=None, as_text=False, checkpoint_path=None): """Exports inference graph as a SavedModel into given dir. This method builds a new graph by first calling the serving_input_receiver_fn to obtain feature `Tensor`s, and then calling this `Estimator`'s model_fn to generate the model graph based on those features. It restores the given checkpoint (or, lacking that, the most recent checkpoint) into this graph in a fresh session. Finally it creates a timestamped export directory below the given export_dir_base, and writes a `SavedModel` into it containing a single `MetaGraphDef` saved from this session. The exported `MetaGraphDef` will provide one `SignatureDef` for each element of the export_outputs dict returned from the model_fn, named using the same keys. One of these keys is always signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY, indicating which signature will be served when a serving request does not specify one. For each signature, the outputs are provided by the corresponding `ExportOutput`s, and the inputs are always the input receivers provided by the serving_input_receiver_fn. Extra assets may be written into the SavedModel via the extra_assets argument. This should be a dict, where each key gives a destination path (including the filename) relative to the assets.extra directory. The corresponding value gives the full path of the source file to be copied. For example, the simple case of copying a single file without renaming it is specified as `{'my_asset_file.txt': '/path/to/my_asset_file.txt'}`. Args: export_dir_base: A string containing a directory in which to create timestamped subdirectories containing exported SavedModels. serving_input_receiver_fn: A function that takes no argument and returns a `ServingInputReceiver`. assets_extra: A dict specifying how to populate the assets.extra directory within the exported SavedModel, or `None` if no extra assets are needed. as_text: whether to write the SavedModel proto in text format. checkpoint_path: The checkpoint path to export. If `None` (the default), the most recent checkpoint found within the model directory is chosen. Returns: The string path to the exported directory. Raises: ValueError: if no serving_input_receiver_fn is provided, no export_outputs are provided, or no checkpoint can be found. """ if serving_input_receiver_fn is None: raise ValueError('serving_input_receiver_fn must be defined.') with ops.Graph().as_default() as g: training.get_or_create_global_step(g) random_seed.set_random_seed(self._config.tf_random_seed) serving_input_receiver = serving_input_receiver_fn() # Call the model_fn and collect the export_outputs. estimator_spec = self._call_model_fn( features=serving_input_receiver.features, labels=None, mode=Modes.PREDICT) # Build the SignatureDefs from receivers and all outputs signature_def_map = build_all_signature_defs( serving_input_receiver.receiver_tensors, estimator_spec.export_outputs) if not checkpoint_path: # Locate the latest checkpoint checkpoint_path = saver.latest_checkpoint(self._model_dir) if not checkpoint_path: raise ValueError("Couldn't find trained model at %s." % self._model_dir) export_dir = get_timestamped_export_dir(export_dir_base) with tf_session.Session() as session: saver_for_restore = estimator_spec.scaffold.saver or saver.Saver( sharded=True) saver_for_restore.restore(session, checkpoint_path) local_init_op = ( estimator_spec.scaffold.local_init_op or monitored_session.Scaffold._default_local_init_op()) # Perform the export builder = saved_model_builder.SavedModelBuilder(export_dir) builder.add_meta_graph_and_variables( session, [tag_constants.SERVING], signature_def_map=signature_def_map, assets_collection=ops.get_collection( ops.GraphKeys.ASSET_FILEPATHS), legacy_init_op=local_init_op) builder.save(as_text) # Add the extra assets if assets_extra: assets_extra_path = os.path.join( compat.as_bytes(export_dir), compat.as_bytes('assets.extra')) for dest_relative, source in assets_extra.items(): dest_absolute = os.path.join( compat.as_bytes(assets_extra_path), compat.as_bytes(dest_relative)) dest_path = os.path.dirname(dest_absolute) gfile.MakeDirs(dest_path) gfile.Copy(source, dest_absolute) return export_dir
def test_model_equivalent_to_chained_model_chunk_size_one(self): numpy.random.seed(2) random_seed.set_random_seed(3) self._model_equivalent_to_chained_model_test_template(1)
def _evaluate_model(self, input_fn, hooks=None, checkpoint_path=None, name=''): """Evaluates the model using the training.evaluation library.""" # Check that model has been trained (if nothing has been set explicitly). if not checkpoint_path: latest_path = saver.latest_checkpoint(self._model_dir) if not latest_path: raise ValueError( 'Could not find trained model in model_dir: {}.'.format( self._model_dir)) checkpoint_path = latest_path # Setup output directory. eval_dir = os.path.join(self._model_dir, 'eval' if not name else 'eval_' + name) with ops.Graph().as_default() as g: random_seed.set_random_seed(self._config.tf_random_seed) global_step_tensor = self._create_and_assert_global_step(g) features, labels, input_hooks = ( self._get_features_and_labels_from_input_fn( input_fn, model_fn_lib.ModeKeys.EVAL)) estimator_spec = self._call_model_fn(features, labels, model_fn_lib.ModeKeys.EVAL, self.config) if model_fn_lib.LOSS_METRIC_KEY in estimator_spec.eval_metric_ops: raise ValueError( 'Metric with name "%s" is not allowed, because Estimator ' % (model_fn_lib.LOSS_METRIC_KEY) + 'already defines a default metric with the same name.') estimator_spec.eval_metric_ops[ model_fn_lib.LOSS_METRIC_KEY] = metrics_lib.mean( estimator_spec.loss) update_op, eval_dict = _extract_metric_update_ops( estimator_spec.eval_metric_ops) if ops.GraphKeys.GLOBAL_STEP in eval_dict: raise ValueError( 'Metric with name `global_step` is not allowed, because Estimator ' 'already defines a default metric with the same name.') eval_dict[ops.GraphKeys.GLOBAL_STEP] = global_step_tensor all_hooks = list(input_hooks) all_hooks.extend(hooks) all_hooks.extend(list(estimator_spec.evaluation_hooks or [])) eval_results = evaluation._evaluate_once( # pylint: disable=protected-access checkpoint_path=checkpoint_path, master=self._config.evaluation_master, scaffold=estimator_spec.scaffold, eval_ops=update_op, final_ops=eval_dict, hooks=all_hooks, config=self._session_config) _write_dict_to_summary( output_dir=eval_dir, dictionary=eval_results, current_global_step=eval_results[ops.GraphKeys.GLOBAL_STEP]) return eval_results
def _train_model(self, input_fn, hooks): all_hooks = [] self._graph = ops.Graph() with self._graph.as_default() as g, g.device(self._device_fn): random_seed.set_random_seed(self._config.tf_random_seed) global_step = training.get_or_create_global_step(g) with ops.device('/cpu:0'): features, labels = input_fn() estimator_spec = self._call_model_fn(features, labels, Modes.TRAIN) ops.add_to_collection(ops.GraphKeys.LOSSES, estimator_spec.loss) all_hooks.extend([ plx_hooks.NanTensorHook(estimator_spec.loss), plx_hooks.StepLoggingTensorHook( { 'loss': estimator_spec.loss, 'step': global_step }, every_n_iter=100) ]) all_hooks.extend(hooks) all_hooks.extend(estimator_spec.training_hooks) scaffold = estimator_spec.scaffold or monitored_session.Scaffold() if not (scaffold.saver or ops.get_collection(ops.GraphKeys.SAVERS)): ops.add_to_collection( ops.GraphKeys.SAVERS, # TODO remove non restorable vars saver.Saver( sharded=True, # TODO `var_list` max_to_keep=self._config.keep_checkpoint_max, defer_build=True)) chief_hooks = [] if self._config.save_checkpoints_secs or self._config.save_checkpoints_steps: saver_hook_exists = any([ isinstance(h, plx_hooks.StepCheckpointSaverHook) for h in (all_hooks + chief_hooks + list(estimator_spec.training_chief_hooks)) ]) if not saver_hook_exists: chief_hooks += [ plx_hooks.StepCheckpointSaverHook( self._model_dir, save_secs=self._config.save_checkpoints_secs, save_steps=self._config.save_checkpoints_steps, scaffold=scaffold) ] if self._config.save_summary_steps: saver_hook_exists = any([ isinstance(h, plx_hooks.StepSummarySaverHook) for h in (all_hooks + chief_hooks + list(estimator_spec.training_chief_hooks)) ]) if not saver_hook_exists: chief_hooks += [ plx_hooks.StepSummarySaverHook( scaffold=scaffold, save_steps=self._config.save_summary_steps, output_dir=self._model_dir, ) ] with monitored_session.MonitoredTrainingSession( master=self._config.master, is_chief=self._config.is_chief, checkpoint_dir=self._model_dir, scaffold=scaffold, hooks=all_hooks, chief_only_hooks=chief_hooks + list(estimator_spec.training_chief_hooks), save_checkpoint_secs= 0, # Saving checkpoint is handled by a hook. save_summaries_steps= 0, # Saving summaries is handled by a hook. config=self._session_config) as mon_sess: loss = None while not mon_sess.should_stop(): _, loss = mon_sess.run( [estimator_spec.train_op, estimator_spec.loss]) summary_io.SummaryWriterCache.clear() return loss
def predict(self, input_fn, predict_keys=None, hooks=None, checkpoint_path=None): """Yields predictions for given features. Args: input_fn: A function that constructs the features. Prediction continues until `input_fn` raises an end-of-input exception (`OutOfRangeError` or `StopIteration`). See @{$get_started/premade_estimators#create_input_functions} for more information. The function should construct and return one of the following: * A 'tf.data.Dataset' object: Outputs of `Dataset` object must have same constraints as below. * features: A `Tensor` or a dictionary of string feature name to `Tensor`. features are consumed by `model_fn`. They should satisfy the expectation of `model_fn` from inputs. * A tuple, in which case the first item is extracted as features. predict_keys: list of `str`, name of the keys to predict. It is used if the `EstimatorSpec.predictions` is a `dict`. If `predict_keys` is used then rest of the predictions will be filtered from the dictionary. If `None`, returns all. hooks: List of `SessionRunHook` subclass instances. Used for callbacks inside the prediction call. checkpoint_path: Path of a specific checkpoint to predict. If `None`, the latest checkpoint in `model_dir` is used. Yields: Evaluated values of `predictions` tensors. Raises: ValueError: Could not find a trained model in model_dir. ValueError: if batch length of predictions are not same. ValueError: If there is a conflict between `predict_keys` and `predictions`. For example if `predict_keys` is not `None` but `EstimatorSpec.predictions` is not a `dict`. """ hooks = _check_hooks_type(hooks) # Check that model has been trained. if not checkpoint_path: checkpoint_path = saver.latest_checkpoint(self._model_dir) if not checkpoint_path: raise ValueError( 'Could not find trained model in model_dir: {}.'.format( self._model_dir)) with ops.Graph().as_default() as g: random_seed.set_random_seed(self._config.tf_random_seed) self._create_and_assert_global_step(g) features, input_hooks = self._get_features_from_input_fn( input_fn, model_fn_lib.ModeKeys.PREDICT) estimator_spec = self._call_model_fn(features, None, model_fn_lib.ModeKeys.PREDICT, self.config) predictions = self._extract_keys(estimator_spec.predictions, predict_keys) with training.MonitoredSession( session_creator=training.ChiefSessionCreator( checkpoint_filename_with_path=checkpoint_path, scaffold=estimator_spec.scaffold, config=self._session_config), hooks=input_hooks + hooks) as mon_sess: while not mon_sess.should_stop(): preds_evaluated = mon_sess.run(predictions) if not isinstance(predictions, dict): for pred in preds_evaluated: yield pred else: for i in range( self._extract_batch_length(preds_evaluated)): yield { key: value[i] for key, value in six.iteritems( preds_evaluated) }
def testCtcLossV3(self, run_tf_func): """Testing GPU CTC loss. testing if GPU CTC loss will generate same result with CPU version """ if not test.is_gpu_available(): self.skipTest("Need GPU for testing.") random_seed.set_random_seed(5) batch_size = 8 num_labels = 6 max_label_length = 5 num_frames = 12 labels = random_ops.random_uniform([batch_size, max_label_length], minval=1, maxval=num_labels, dtype=dtypes.int64) logits = random_ops.random_uniform( [num_frames, batch_size, num_labels]) label_length = random_ops.random_uniform([batch_size], minval=2, maxval=max_label_length, dtype=dtypes.int64) label_mask = array_ops.sequence_mask(label_length, maxlen=max_label_length, dtype=label_length.dtype) labels *= label_mask logit_length = [num_frames] * batch_size def ctc_loss_cpu(labels, logits, label_length, logit_length): with test_util.device(use_gpu=False): sparse_labels = ctc_ops.dense_labels_to_sparse( labels, label_length) with backprop.GradientTape() as t: t.watch(logits) ref_loss = ctc_ops.ctc_loss_v3(labels=sparse_labels, logits=logits, label_length=label_length, logit_length=logit_length, blank_index=0) ref_grad = t.gradient(ref_loss, [logits]) return ref_loss, ref_grad def ctc_loss_gpu(labels, logits, label_length, logit_length): with test_util.device(use_gpu=True): sparse_labels = ctc_ops.dense_labels_to_sparse( labels, label_length) with backprop.GradientTape() as t: t.watch(logits) loss = ctc_ops.ctc_loss_v3(labels=sparse_labels, logits=logits, label_length=label_length, logit_length=logit_length, blank_index=0) grad = t.gradient(loss, [logits]) return loss, grad if run_tf_func: ctc_loss_cpu = def_function.function(ctc_loss_cpu) ctc_loss_gpu = def_function.function(ctc_loss_gpu) ref_loss, ref_grad = ctc_loss_cpu(labels, logits, label_length, logit_length) loss, grad = ctc_loss_gpu(labels, logits, label_length, logit_length) self.assertAllClose(loss, ref_loss, atol=1e-6) self.assertAllClose(grad, ref_grad, atol=2e-6)
def test_correctness(self, distribution, use_numpy, use_validation_data): with self.cached_session(): default_tolerance = 1e-5 tol_table = {} if isinstance(distribution, (mirrored_strategy.MirroredStrategy, mirrored_strategy.CoreMirroredStrategy, distribute_lib._DefaultDistributionStrategy)): # pylint: disable=protected-access # TODO(b/119257215): Weights are not exactly the same, so use larger # tolerance for now. Predict should be related to weights. tol_table = { 'weights_1': 1e-4, 'weights_2': 1e-4, 'predict_result_1': 1e-4, } keras.backend.set_image_data_format('channels_last') np.random.seed(_RANDOM_SEED) random_seed.set_random_seed(_RANDOM_SEED) # Train, eval, and predict datasets are created with the same input numpy # arrays. # TODO(xiejw): Change this back to 10000, once we support final partial # batch. num_samples = 9984 x_train = np.random.rand(num_samples, 1) y_train = 3 * x_train x_train = x_train.astype('float32') y_train = y_train.astype('float32') x_predict = [[1.], [2.], [3.], [4.]] # The model is built once and the initial weights are saved. # This is used to initialize the model for both the distribution and # non-distribution run. In addition, we add few non-linear layers to make # it non-trivial. def _create_model(): model = keras.Sequential() model.add( keras.layers.Dense(10, activation='relu', input_shape=(1, ))) model.add(keras.layers.Dense(10, activation='relu')) model.add(keras.layers.Dense(10, activation='relu')) model.add(keras.layers.Dense(1)) return model model = _create_model() initial_weights = model.get_weights() del model # avoid accident usage. def fit_eval_and_predict(with_distribution=None): model = _create_model() # We have initialized the model to the same weight for the distribution # and non-distribution run. model.set_weights(initial_weights) model.compile(loss=keras.losses.mean_squared_error, optimizer=gradient_descent_keras.SGD(0.5), metrics=['mse'], distribute=with_distribution) training_inputs, eval_inputs, predict_inputs = ( get_correctness_test_inputs(use_numpy, use_validation_data, with_distribution, x_train, y_train, x_predict)) result = {} result['training_history_1'] = model.fit( **training_inputs).history if eval_inputs is not None: result['eval_result_1'] = model.evaluate(**eval_inputs) result['weights_1'] = model.get_weights() result['predict_result_1'] = model.predict(**predict_inputs) # Train and eval again to mimic user's flow. result['training_history_2'] = model.fit( **training_inputs).history if eval_inputs is not None: result['eval_result_2'] = model.evaluate(**eval_inputs) result['weights_2'] = model.get_weights() return result results_with_ds = fit_eval_and_predict( with_distribution=distribution) results_without_ds = fit_eval_and_predict(with_distribution=None) # Verify that the weights, training history, eval results, predict outputs # are the same within some limits of tolerance. for key in results_with_ds: if (key.startswith('training_history') and isinstance( distribution, tpu_strategy.TPUStrategyV1) and distribution.extended.steps_per_run > 1): # TODO(b/119894254): Enable this test for all cases once the # underlying bug is fixed. continue tolerance = tol_table.get(key, default_tolerance) self.assertAllClose(results_with_ds[key], results_without_ds[key], atol=tolerance, rtol=tolerance, msg='Fail to assert {}.'.format(key))
def _train_model(self, input_fn, hooks, saving_listeners): worker_hooks = [] with ops.Graph().as_default() as g, g.device(self._device_fn): random_seed.set_random_seed(self._config.tf_random_seed) global_step_tensor = self._create_and_assert_global_step(g) training_util._get_or_create_global_step_read() # pylint: disable=protected-access features, labels, input_hooks = ( self._get_features_and_labels_from_input_fn( input_fn, model_fn_lib.ModeKeys.TRAIN)) worker_hooks.extend(input_hooks) estimator_spec = self._call_model_fn(features, labels, model_fn_lib.ModeKeys.TRAIN, self.config) # Check if the user created a loss summary, and add one if they didn't. # We assume here that the summary is called 'loss'. If it is not, we will # make another one with the name 'loss' to ensure it shows up in the right # graph in TensorBoard. if not any([ x.op.name == 'loss' for x in ops.get_collection(ops.GraphKeys.SUMMARIES) ]): summary.scalar('loss', estimator_spec.loss) ops.add_to_collection(ops.GraphKeys.LOSSES, estimator_spec.loss) worker_hooks.extend(hooks) worker_hooks.extend([ training.NanTensorHook(estimator_spec.loss), training.LoggingTensorHook( { 'loss': estimator_spec.loss, 'step': global_step_tensor }, every_n_iter=100) ]) worker_hooks.extend(estimator_spec.training_hooks) if not (estimator_spec.scaffold.saver or ops.get_collection(ops.GraphKeys.SAVERS)): ops.add_to_collection( ops.GraphKeys.SAVERS, training.Saver( sharded=True, max_to_keep=self._config.keep_checkpoint_max, keep_checkpoint_every_n_hours=( self._config.keep_checkpoint_every_n_hours), defer_build=True, save_relative_paths=True)) chief_hooks = [] all_hooks = worker_hooks + list( estimator_spec.training_chief_hooks) saver_hooks = [ h for h in all_hooks if isinstance(h, training.CheckpointSaverHook) ] if (self._config.save_checkpoints_secs or self._config.save_checkpoints_steps): if not saver_hooks: chief_hooks = [ training.CheckpointSaverHook( self._model_dir, save_secs=self._config.save_checkpoints_secs, save_steps=self._config.save_checkpoints_steps, scaffold=estimator_spec.scaffold) ] saver_hooks = [chief_hooks[0]] if saving_listeners: if not saver_hooks: raise ValueError( 'There should be a CheckpointSaverHook to use saving_listeners. ' 'Please set one of the RunConfig.save_checkpoints_steps or ' 'RunConfig.save_checkpoints_secs.') else: # It is expected to have one CheckpointSaverHook. If multiple, we pick # up the first one to add listener. saver_hooks[0]._listeners.extend(saving_listeners) # pylint: disable=protected-access with training.MonitoredTrainingSession( master=self._config.master, is_chief=self._config.is_chief, checkpoint_dir=self._model_dir, scaffold=estimator_spec.scaffold, hooks=worker_hooks, chief_only_hooks=( tuple(chief_hooks) + tuple(estimator_spec.training_chief_hooks)), save_checkpoint_secs=0, # Saving is handled by a hook. save_summaries_steps=self._config.save_summary_steps, config=self._session_config, log_step_count_steps=self._config.log_step_count_steps ) as mon_sess: loss = None while not mon_sess.should_stop(): _, loss = mon_sess.run( [estimator_spec.train_op, estimator_spec.loss]) return loss
def _train_model(self, input_fn, steps, feed_fn=None, init_op=None, init_feed_fn=None, init_fn=None, device_fn=None, monitors=None, log_every_steps=100, fail_on_nan_loss=True, max_steps=None): # TODO(wicke): Remove this once Model and associated code are gone. if hasattr(self._config, 'execution_mode'): if self._config.execution_mode not in ('all', 'train'): return # Stagger startup of worker sessions based on task id. sleep_secs = min( self._config.training_worker_max_startup_secs, self._config.task * self._config.training_worker_session_startup_stagger_secs) if sleep_secs: logging.info('Waiting %d secs before starting task %d.', sleep_secs, self._config.task) time.sleep(sleep_secs) # Device allocation device_fn = device_fn or self._device_fn self._graph = ops.Graph() with self._graph.as_default() as g, g.device(device_fn): random_seed.set_random_seed(self._config.tf_random_seed) global_step = contrib_framework.create_global_step(g) features, targets = input_fn() self._check_inputs(features, targets) train_op, loss_op = self._get_train_ops(features, targets) # Add default monitors. if monitors is None: monitors = [] is_chief = self._config.task == 0 if is_chief: monitors += monitors_lib.get_default_monitors( loss_op=loss_op, summary_op=logging_ops.get_summary_op(), save_summary_steps=self._config.save_summary_steps, summary_writer=graph_actions.get_summary_writer( self._model_dir)) else: monitors = [] # Setup monitors. for monitor in monitors: monitor.set_estimator(self) return graph_actions.train( graph=g, output_dir=self._model_dir, train_op=train_op, loss_op=loss_op, global_step_tensor=global_step, init_op=init_op, init_feed_dict=init_feed_fn() if init_feed_fn is not None else None, init_fn=init_fn, log_every_steps=log_every_steps, supervisor_is_chief=is_chief, supervisor_master=self._config.master, supervisor_save_model_secs=self._config.save_checkpoints_secs, keep_checkpoint_max=self._config.keep_checkpoint_max, feed_fn=feed_fn, steps=steps, fail_on_nan_loss=fail_on_nan_loss, monitors=monitors, max_steps=max_steps)
def train(parameters, train_ds, val_ds, wordvec, class_weights): tf.enable_eager_execution() tf.logging.set_verbosity(tf.logging.ERROR) random_seed.set_random_seed(parameters['seed']) (device, data_format) = ('/gpu:0', 'channels_first') if parameters['no_gpu'] > 0 or not tf.test.is_gpu_available(): (device, data_format) = ('/cpu:0', 'channels_last') print('Using device %s, and data format %s.' % (device, data_format)) model = HAN(wordvec, parameters) optimizer = AdamWeightDecayOptimizer( learning_rate=parameters['learning_rate'], weight_decay_rate=0.0, beta_1=0.9, beta_2=0.999, epsilon=1e-6, exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"]) timestamp = datetime.now().strftime(' %d%m%y %H%M%S') # Create and restore checkpoint (if one exists on the path) checkpoint_prefix = os.path.join(parameters['model_dir'], 'ckpt') step_counter = tf.train.get_or_create_global_step() checkpoint = tf.train.Checkpoint(model=model, optimizer=optimizer, step_counter=step_counter) best_acc_ep = (0.0, -1, float('inf')) # acc, epoch, loss patience = 0 with tf.device(device): for ep in range(parameters['train_epochs']): start = time.time() train_step(model, optimizer, train_ds, step_counter, ep, class_weights, parameters, parameters['log_interval']) val_acc, val_loss = test(model, val_ds, class_weights, ds_name='Val') end = time.time() print('\n Epoch: {} \tTime: {:.6f}'.format(ep + 1, end - start)) parameters['val_losses'].append(val_loss) if val_loss.numpy() < best_acc_ep[2]: best_acc_ep = (val_acc.numpy(), ep, val_loss.numpy()) print('Save checkpoint', checkpoint_prefix) checkpoint.save(checkpoint_prefix) # else: # if patience == parameters['patience']: # print('Apply early stopping') # break # patience += 1 # print('patience {}/{}'.format(patience, parameters['patience'])) print('Min loss {:.6f}, dev acc. {:.3f}%, ep {} \n'.format( best_acc_ep[2], best_acc_ep[0] * 100., best_acc_ep[1] + 1)) model._name = "Hybrid Attention Network" model.summary() plt.ylabel('Training/Validation Loss') plt.xlabel('Number of Epochs') plt.plot(parameters['train_losses'], label="Train Loss") plt.plot(parameters['val_losses'], label="Validation Loss") plt.legend() plt.show() plt.savefig('han_training_curve.png') plt.gcf().clear()
def setUp(self): np.random.seed(_RANDOM_SEED) random_seed.set_random_seed(_RANDOM_SEED) self._root_dir = 'base' super(TestSavedModelBase, self).setUp()
def testRegisterBlocks(self): with ops.Graph().as_default(): random_seed.set_random_seed(200) lc = layer_collection.LayerCollection() lc.register_fully_connected(array_ops.constant(1), array_ops.constant(2), array_ops.constant(3)) lc.register_fully_connected( array_ops.constant(1), array_ops.constant(2), array_ops.constant(3), approx=layer_collection.APPROX_DIAGONAL_NAME) lc.register_conv2d(params=array_ops.ones((2, 3, 4, 5)), strides=[1, 1, 1, 1], padding='SAME', inputs=array_ops.ones((1, 2, 3, 4)), outputs=array_ops.ones((1, 1, 1, 5))) lc.register_conv2d(params=array_ops.ones((2, 3, 4, 5)), strides=[1, 1, 1, 1], padding='SAME', inputs=array_ops.ones((1, 2, 3, 4)), outputs=array_ops.ones((1, 1, 1, 5)), approx=layer_collection.APPROX_DIAGONAL_NAME) lc.register_separable_conv2d(depthwise_params=array_ops.ones( (3, 3, 1, 2)), pointwise_params=array_ops.ones( (1, 1, 2, 4)), inputs=array_ops.ones((32, 5, 5, 1)), depthwise_outputs=array_ops.ones( (32, 5, 5, 2)), pointwise_outputs=array_ops.ones( (32, 5, 5, 4)), strides=[1, 1, 1, 1], padding='SAME') lc.register_convolution(params=array_ops.ones((3, 3, 1, 8)), inputs=array_ops.ones((32, 5, 5, 1)), outputs=array_ops.ones((32, 5, 5, 8)), padding='SAME') lc.register_generic(array_ops.constant(5), 16, approx=layer_collection.APPROX_FULL_NAME) lc.register_generic(array_ops.constant(6), 16, approx=layer_collection.APPROX_DIAGONAL_NAME) lc.register_fully_connected_multi( array_ops.constant(1), (array_ops.constant(2), array_ops.constant(3)), (array_ops.constant(4), array_ops.constant(5))) lc.register_conv2d_multi(params=array_ops.ones((2, 3, 4, 5)), strides=[1, 1, 1, 1], padding='SAME', inputs=(array_ops.ones((1, 2, 3, 4)), array_ops.ones((5, 6, 7, 8))), outputs=(array_ops.ones((1, 1, 1, 5)), array_ops.ones((2, 2, 2, 10)))) lc.register_embedding_multi( array_ops.constant( (1, )), (array_ops.constant(2), array_ops.constant(3)), (array_ops.constant(4), array_ops.constant(5))) self.assertEqual(12, len(lc.get_blocks()))
def _testOneSimpleTraining(self, rnn_mode, num_layers, num_units, input_size, batch_size, seq_length, dir_count, dropout, dtype, delta, tolerance): # Gradient checking runs two forward ops with almost the same input. Need to # make sure the drop patterns across the two runs are the same. old_env_state = os.environ.get("TF_CUDNN_RESET_RND_GEN_STATE", str(False)) os.environ["TF_CUDNN_RESET_RND_GEN_STATE"] = str(True) has_input_c = (rnn_mode == cudnn_rnn_ops.CUDNN_LSTM) random_seed.set_random_seed(1234) model = self._CreateModel(rnn_mode, num_layers, num_units, input_size, dtype=dtype, dropout=dropout) params_size_t = model.params_size() input_data = variables.Variable(random_ops.random_uniform( [seq_length, batch_size, input_size], dtype=dtype), dtype=dtype) input_h = variables.Variable(random_ops.random_uniform( [num_layers * dir_count, batch_size, num_units], dtype=dtype), dtype=dtype) params = variables.Variable(random_ops.random_uniform([params_size_t], dtype=dtype), validate_shape=False, dtype=dtype) if has_input_c: input_c = variables.Variable(random_ops.random_uniform( [num_layers * dir_count, batch_size, num_units], dtype=dtype), dtype=dtype) output, output_h, output_c = model(input_data=input_data, input_h=input_h, input_c=input_c, params=params) else: output, output_h = model(input_data=input_data, input_h=input_h, params=params) output_sum = math_ops.reduce_sum(output) output_h_sum = math_ops.reduce_sum(output_h) total_sum = output_sum + output_h_sum if has_input_c: output_c_sum = math_ops.reduce_sum(output_c) total_sum += output_c_sum with self.test_session(use_gpu=True) as sess: params_size_v = sess.run(params_size_t) inputs_and_shapes = [ (input_data, [seq_length, batch_size, input_size]), (input_h, [num_layers * dir_count, batch_size, num_units]), (params, [params_size_v]), ] if has_input_c: inputs_and_shapes.append( (input_c, [num_layers * dir_count, batch_size, num_units ]), ) sess.run(variables.global_variables_initializer()) all_inputs = [entry[0] for entry in inputs_and_shapes] all_shapes = [entry[1] for entry in inputs_and_shapes] err = gradient_checker.compute_gradient_error(all_inputs, all_shapes, total_sum, [1], delta=delta) self.assertLess(err, tolerance) os.environ["TF_CUDNN_RESET_RND_GEN_STATE"] = old_env_state
def setUp(self): super().setUp() random_seed.set_random_seed(None) config.enable_op_determinism()
def testTrainAllVarsHasLowerLossThanTrainSubsetOfVars(self): logdir = os.path.join(self.get_temp_dir(), 'tmp_logs3/') if gfile.Exists(logdir): # For running on jenkins. gfile.DeleteRecursively(logdir) # First, train only the weights of the model. with ops.Graph().as_default(): random_seed.set_random_seed(0) total_loss = self.ModelLoss() optimizer = gradient_descent.GradientDescentOptimizer( learning_rate=1.0) weights = variables_lib.get_variables_by_name('weights') train_op = training.create_train_op(total_loss, optimizer, variables_to_train=weights) saver = saver_lib.Saver() loss = training.train( train_op, logdir, hooks=[ basic_session_run_hooks.CheckpointSaverHook(logdir, save_steps=200, saver=saver), basic_session_run_hooks.StopAtStepHook(num_steps=200), ], save_checkpoint_secs=None, save_summaries_steps=None) self.assertGreater(loss, .015) self.assertLess(loss, .05) # Next, train the biases of the model. with ops.Graph().as_default(): random_seed.set_random_seed(1) total_loss = self.ModelLoss() optimizer = gradient_descent.GradientDescentOptimizer( learning_rate=1.0) biases = variables_lib.get_variables_by_name('biases') train_op = training.create_train_op(total_loss, optimizer, variables_to_train=biases) saver = saver_lib.Saver() loss = training.train( train_op, logdir, hooks=[ basic_session_run_hooks.CheckpointSaverHook(logdir, save_steps=300, saver=saver), basic_session_run_hooks.StopAtStepHook(num_steps=300), ], save_checkpoint_secs=None, save_summaries_steps=None) self.assertGreater(loss, .015) self.assertLess(loss, .05) # Finally, train both weights and bias to get lower loss. with ops.Graph().as_default(): random_seed.set_random_seed(2) total_loss = self.ModelLoss() optimizer = gradient_descent.GradientDescentOptimizer( learning_rate=1.0) train_op = training.create_train_op(total_loss, optimizer) saver = saver_lib.Saver() loss = training.train( train_op, logdir, hooks=[ basic_session_run_hooks.StopAtStepHook(num_steps=400), ], save_checkpoint_secs=None, save_summaries_steps=None) self.assertIsNotNone(loss) self.assertLess(loss, .015)
def _train_model(self, input_fn, hooks): all_hooks = [] with ops.Graph().as_default() as g, g.device(self._device_fn): random_seed.set_random_seed(self._config.tf_random_seed) global_step_tensor = self._create_and_assert_global_step(g) features, labels = self._call_input_fn(input_fn, model_fn_lib.ModeKeys.TRAIN) estimator_spec = self._call_model_fn(features, labels, model_fn_lib.ModeKeys.TRAIN) ops.add_to_collection(ops.GraphKeys.LOSSES, estimator_spec.loss) all_hooks.extend(hooks) all_hooks.extend([ training.NanTensorHook(estimator_spec.loss), training.LoggingTensorHook( { 'loss': estimator_spec.loss, 'step': global_step_tensor }, every_n_iter=100) ]) all_hooks.extend(estimator_spec.training_hooks) if not (estimator_spec.scaffold.saver or ops.get_collection(ops.GraphKeys.SAVERS)): ops.add_to_collection( ops.GraphKeys.SAVERS, training.Saver( sharded=True, max_to_keep=self._config.keep_checkpoint_max, keep_checkpoint_every_n_hours=( self._config.keep_checkpoint_every_n_hours), defer_build=True, save_relative_paths=True)) chief_hooks = [] if (self._config.save_checkpoints_secs or self._config.save_checkpoints_steps): saver_hook_exists = any([ isinstance(h, training.CheckpointSaverHook) for h in (all_hooks + chief_hooks + list(estimator_spec.training_chief_hooks)) ]) if not saver_hook_exists: chief_hooks = [ training.CheckpointSaverHook( self._model_dir, save_secs=self._config.save_checkpoints_secs, save_steps=self._config.save_checkpoints_steps, scaffold=estimator_spec.scaffold) ] with training.MonitoredTrainingSession( master=self._config.master, is_chief=self._config.is_chief, checkpoint_dir=self._model_dir, scaffold=estimator_spec.scaffold, hooks=all_hooks, chief_only_hooks=( tuple(chief_hooks) + tuple(estimator_spec.training_chief_hooks)), save_checkpoint_secs=0, # Saving is handled by a hook. save_summaries_steps=self._config.save_summary_steps, config=self._session_config, log_step_count_steps=self._config.log_step_count_steps ) as mon_sess: loss = None while not mon_sess.should_stop(): _, loss = mon_sess.run( [estimator_spec.train_op, estimator_spec.loss]) return loss
def _train_model(self, input_fn, steps, feed_fn=None, init_op=None, init_feed_fn=None, init_fn=None, device_fn=None, monitors=None, log_every_steps=100, fail_on_nan_loss=True): # TODO(wicke): This is a hack and needs to go. if self._config.execution_mode not in ('all', 'train'): return if not self._model_dir: raise ValueError('Estimator\'s model_dir should be non-empty.') # Stagger startup of worker sessions based on task id. sleep_secs = min(self._config.training_worker_max_startup_secs, self._config.task * self._config.training_worker_session_startup_stagger_secs) if sleep_secs: logging.info('Waiting %d secs before starting task %d.', sleep_secs, self._config.task) time.sleep(sleep_secs) # Device allocation device_fn = device_fn or self._device_fn self._graph = ops.Graph() with self._graph.as_default() as g, g.device(device_fn): random_seed.set_random_seed(self._config.tf_random_seed) global_step = contrib_framework.create_global_step(g) features, targets = input_fn() self._check_inputs(features, targets) train_op, loss_op = self._get_train_ops(features, targets) # Add default monitors. if monitors is None: monitors = [] monitors += monitors_lib.get_default_monitors( loss_op=loss_op, summary_op=logging_ops.get_summary_op(), save_summary_steps=100, summary_writer=graph_actions.get_summary_writer(self._model_dir)) is_chief = self._config.task == 0 if not is_chief: # Run monitors only on chief. monitors = [] # Setup monitors. for monitor in monitors: monitor.set_estimator(self) return graph_actions.train( graph=g, output_dir=self._model_dir, train_op=train_op, loss_op=loss_op, global_step_tensor=global_step, init_op=init_op, init_feed_dict=init_feed_fn() if init_feed_fn is not None else None, init_fn=init_fn, log_every_steps=log_every_steps, supervisor_is_chief=is_chief, supervisor_master=self._config.master, feed_fn=feed_fn, max_steps=steps, fail_on_nan_loss=fail_on_nan_loss, monitors=monitors)
def _testCudnnCompatibleRnnCells(self, num_layers, seq_length, num_units, input_size, batch_size, rnn_mode, use_block_cell): has_state_c = rnn_mode == cudnn_rnn_ops.CUDNN_LSTM np.random.seed(0) # Train graph with ops.Graph().as_default(): random_seed.set_random_seed(299) input_data = array_ops.placeholder( dtypes.float32, shape=[seq_length, batch_size, input_size]) output_tuple, cudnn_model, cudnn_params = self._build_forward_cudnn_model( rnn_mode, num_layers, num_units, input_data, is_training=True) target_output = array_ops.placeholder(dtype=dtypes.float32, shape=None) total_sum = sum(map(math_ops.reduce_sum, output_tuple)) loss_op = losses.log_loss(labels=target_output, predictions=total_sum) optimizer = gradient_descent.GradientDescentOptimizer( learning_rate=1e-2) train_op = optimizer.minimize(loss_op) saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2) # Train Cudnn model with self.test_session(use_gpu=True, graph=ops.get_default_graph()) as sess: sess.run(variables.global_variables_initializer()) # Train 128 steps num_steps = 128 for _ in range(num_steps): inputs = np.random.rand(seq_length, batch_size, input_size).astype(np.float32) targets = np.random.rand() sess.run(train_op, feed_dict={ input_data: inputs, target_output: targets }) save_path = os.path.join(self.get_temp_dir(), ("cudnn-rnn-%s-test" % rnn_mode)) save_v = saver.save(sess, save_path) self.assertEqual(save_path, save_v) cudnn_params_v = sess.run(cudnn_params) # cuDNN inference graph with ops.Graph().as_default(): random_seed.set_random_seed(299) cudnn_inputs = array_ops.placeholder( dtypes.float32, shape=[seq_length, batch_size, input_size]) (cudnn_output_tuple, cudnn_model, cudnn_params) = self._build_forward_cudnn_model(rnn_mode, num_layers, num_units, cudnn_inputs, is_training=False) saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2) inference_input = np.random.rand(seq_length, batch_size, input_size).astype(np.float32) with self.test_session(use_gpu=True, graph=ops.get_default_graph()) as sess: sess.run(variables.global_variables_initializer()) saver.restore(sess, save_path) restored_cudnn_params_v = sess.run(cudnn_params) self.assertAllEqual(cudnn_params_v, restored_cudnn_params_v) # Cudnn inference cudnn_output = sess.run( cudnn_output_tuple, feed_dict={cudnn_inputs: inference_input}) # Canonical RNN inference graph with ops.Graph().as_default(): random_seed.set_random_seed(299) cell_inputs = array_ops.placeholder( dtypes.float32, shape=[seq_length, batch_size, input_size]) (output, states) = _create_cudnn_compatible_canonical_rnn( cudnn_model, cell_inputs, use_block_cell) saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2) with self.test_session(use_gpu=True, graph=ops.get_default_graph()) as sess: saver.restore(sess, save_path) # BlockCell inference output_v, states_v = sess.run( [output, states], feed_dict={cell_inputs: inference_input}) # output across timestamps are packed into one tensor. self.assertAllClose(cudnn_output[0], output_v, atol=1e-6, rtol=1e-6) for i in range(num_layers): if has_state_c: # output_h self.assertAllClose(cudnn_output[1][i, :], states_v[i].h, atol=1e-6, rtol=1e-6) # output_c self.assertAllClose(cudnn_output[2][i, :], states_v[i].c, atol=1e-6, rtol=1e-6) else: self.assertAllClose(cudnn_output[1][i, :], states_v[i], atol=1e-6, rtol=1e-6)
def setUp(self): random_seed.set_random_seed(1)
def testTrainWithInitFromCheckpoint(self): logdir1 = os.path.join(self.get_temp_dir(), 'tmp_logs1/') logdir2 = os.path.join(self.get_temp_dir(), 'tmp_logs2/') if gfile.Exists(logdir1): # For running on jenkins. gfile.DeleteRecursively(logdir1) if gfile.Exists(logdir2): # For running on jenkins. gfile.DeleteRecursively(logdir2) # First, train the model one step (make sure the error is high). with ops.Graph().as_default(): random_seed.set_random_seed(0) train_op = self.create_train_op() saver = saver_lib.Saver() loss = training.train( train_op, logdir1, hooks=[ basic_session_run_hooks.CheckpointSaverHook(logdir1, save_steps=1, saver=saver), basic_session_run_hooks.StopAtStepHook(num_steps=1), ], save_checkpoint_secs=None, save_summaries_steps=None) self.assertGreater(loss, .5) # Next, train the model to convergence. with ops.Graph().as_default(): random_seed.set_random_seed(1) train_op = self.create_train_op() saver = saver_lib.Saver() loss = training.train( train_op, logdir1, hooks=[ basic_session_run_hooks.CheckpointSaverHook(logdir1, save_steps=300, saver=saver), basic_session_run_hooks.StopAtStepHook(num_steps=300), ], save_checkpoint_secs=None, save_summaries_steps=None) self.assertIsNotNone(loss) self.assertLess(loss, .02) # Finally, advance the model a single step and validate that the loss is # still low. with ops.Graph().as_default(): random_seed.set_random_seed(2) train_op = self.create_train_op() model_variables = variables_lib2.global_variables() model_path = saver_lib.latest_checkpoint(logdir1) assign_fn = variables_lib.assign_from_checkpoint_fn( model_path, model_variables) def init_fn(_, session): assign_fn(session) loss = training.train( train_op, None, scaffold=monitored_session.Scaffold(init_fn=init_fn), hooks=[basic_session_run_hooks.StopAtStepHook(num_steps=1)], save_checkpoint_secs=None, save_summaries_steps=None) self.assertIsNotNone(loss) self.assertLess(loss, .02)
def testConvOutputKroneckerFactorInitNotEnoughDims(self): with tf_ops.Graph().as_default(): random_seed.set_random_seed(200) tensor = array_ops.ones((2, 3), name='a/b/c') with self.assertRaises(IndexError): ff.ConvOutputKroneckerFactor((tensor, ))
def setUp(self): random_seed.set_random_seed(1) self._hash_key = 1