def testReadSnapshotParallelAfterWrite(self): with compat.forward_compatibility_horizon(2019, 8, 16): self.setUpTFRecord(10, 4000) filenames = self.test_filenames expected = [ b"Record %d of file %d" % (r, f) # pylint:disable=g-complex-comprehension for f in range(0, 10) for r in range(0, 4000) ] tmpdir = self.makeSnapshotDirectory() dataset = core_readers._TFRecordDataset(filenames) dataset = dataset.apply( snapshot.snapshot(tmpdir, shard_size_bytes=1024 * 1024, num_reader_threads=2, reader_buffer_size=10)) self.assertDatasetProduces(dataset, expected, assert_items_equal=True) # remove the original files and try to read the data back only from # snapshot. self.removeTFRecords() dataset2 = core_readers._TFRecordDataset(filenames) dataset2 = dataset2.apply( snapshot.snapshot(tmpdir, shard_size_bytes=1024 * 1024, num_reader_threads=2, reader_buffer_size=10)) self.assertDatasetProduces(dataset2, expected, assert_items_equal=True)
def test_conv_bn_dropout(self): """Test dropout precision of convolution batch norm graph.""" with compat.forward_compatibility_horizon(2019, 6, 7): if test.is_gpu_available(cuda_only=True): random_seed.set_random_seed(0) x = _input([2, 8, 8, 1]) y = _conv_bn(x) y = nn.dropout(y, rate=0.5) y = math_ops.add(y, 1, name='addition') y = _conv_bn(y) y = array_ops.identity(y) optimizer = gradient_descent.GradientDescentOptimizer( learning_rate=0.01) g = optimizer.compute_gradients(y, [x]) output = (y, g) output_val_ref, output_val, cost_graph = self._run(output) node_map = _build_node_map(cost_graph.node) self._assert_output_fp16(node_map, 'Conv2D') self._assert_output_fp16(node_map, 'FusedBatchNormV3') # We do not assert dropout's dtype because we do not want to rely on the # node names of dropout's internal implementation. self._assert_output_fp16(node_map, 'addition') self._assert_output_fp16(node_map, 'Conv2D_1') output_val_ref, output_val, cost_graph = self._run(output) self.assertAllClose(output_val_ref, output_val, atol=2e-3, rtol=2e-3)
def test_stft_and_inverse_stft(self, signal_length, frame_length, frame_step, fft_length, np_rtype, tol): """Test that spectral_ops.stft/inverse_stft match a NumPy implementation.""" # Enable float64 support for RFFTs. with compat.forward_compatibility_horizon(2019, 10, 13): signal = np.random.random(signal_length).astype(np_rtype) self._compare(signal, frame_length, frame_step, fft_length, tol)
def testBatchDimsMatchesPythonBatching(self, params_shape, indices_shape, batch_dims, axis, output_shape): """Checks that batch_dims matches multiple calls to tf.gather().""" # Generate a `params` tensor with the indicated shape. params_size = np.prod(params_shape) params = np.reshape(np.arange(params_size), params_shape) # Generate an `indices` tensor with the indicated shape, where each index # is within the appropriate range. indices_size = np.prod(indices_shape) indices = np.reshape(np.arange(indices_size), indices_shape) indices = indices % params_shape[axis] # Perform repeated (batched) gather operations with numpy, to find the # expected result. expected = self._batchNumpyGather(params, indices, axis, batch_dims) # On Windows, we get an exception if we pass in the transformed numpy # arrays ("Failed to convert numpy ndarray to a Tensor (Unsupported # feed type)."); so convert them back to lists before calling tf.gather. params = params.tolist() indices = indices.tolist() result = array_ops.gather(params, indices, axis=axis, batch_dims=batch_dims) self.assertAllEqual(output_shape, result.shape.as_list()) self.assertAllEqual(expected, result) with compat.forward_compatibility_horizon(2019, 8, 11): result = array_ops.gather( params, indices, axis=axis, batch_dims=batch_dims) self.assertAllEqual(output_shape, result.shape.as_list()) self.assertAllEqual(expected, result)
def testNMS128From1024(self): with compat.forward_compatibility_horizon(2018, 8, 8): num_boxes = 1024 boxes_np = np.random.normal(50, 10, (num_boxes, 4)).astype("f4") scores_np = np.random.normal(0.5, 0.1, (num_boxes,)).astype("f4") max_output_size = 128 iou_threshold_np = np.array(0.5, dtype=np.float32) score_threshold_np = np.array(0.0, dtype=np.float32) with self.cached_session() as sess: boxes = array_ops.placeholder(boxes_np.dtype, shape=boxes_np.shape) scores = array_ops.placeholder(scores_np.dtype, shape=scores_np.shape) iou_threshold = array_ops.placeholder(iou_threshold_np.dtype, iou_threshold_np.shape) score_threshold = array_ops.placeholder(score_threshold_np.dtype, score_threshold_np.shape) with self.test_scope(): selected_indices = image_ops.non_max_suppression_padded( boxes=boxes, scores=scores, max_output_size=max_output_size, iou_threshold=iou_threshold, score_threshold=score_threshold, pad_to_max_output_size=True) inputs_feed = { boxes: boxes_np, scores: scores_np, score_threshold: score_threshold_np, iou_threshold: iou_threshold_np } (indices_tf, _) = sess.run(selected_indices, feed_dict=inputs_feed) self.assertEqual(indices_tf.size, max_output_size)
def test_placeholder(self): def check(equation, *input_and_placeholder_shapes): r = np.random.RandomState(0) inputs = [] input_placeholders = [] for actual_shape, placeholder_shape in input_and_placeholder_shapes: input_np = np.array(r.randn(*actual_shape)) inputs.append(input_np) input_placeholders.append( array_ops.placeholder_with_default(input_np, placeholder_shape)) a = np.einsum(equation, *inputs) b = self.evaluate( special_math_ops.einsum(equation, *input_placeholders)) self.assertAllClose(a, b, atol=1e-4, rtol=1e-4) check('bijl,bjkm->bik', ((9, 2, 3, 5), (None, None, None, 5)), ((9, 3, 4, 7), (None, None, 4, None))) check('...ij,...->...i', ((4, 3, 1, 2), (None, 3, None, 2)), ((4, 3), (None, 3))) # Ellipsis with unknown rank. with compat.forward_compatibility_horizon(2019, 10, 19): check('bijl,bjkm->bik', ((9, 2, 3, 5), None), ((9, 3, 4, 7), None)) check('...ij,...jk->...ik', ((3, 1, 2, 3), None), ((1, 7, 3, 4), None))
def testBatchNormGradShape5(self): with compat.forward_compatibility_horizon(2019, 6, 7): for is_training in [True, False]: x_shape = [0, 7, 11, 4] for dtype in [np.float16, np.float32]: if test.is_gpu_available(cuda_only=True): self._test_gradient( x_shape, dtype, [7], np.float32, use_gpu=True, data_format='NCHW', is_training=is_training) self._test_gradient( x_shape, dtype, [4], np.float32, use_gpu=True, data_format='NHWC', is_training=is_training) self._test_gradient( x_shape, dtype, [4], np.float32, use_gpu=False, data_format='NHWC', is_training=is_training)
def test_conv_bn_dropout(self): """Test dropout precision of convolution batch norm graph.""" with compat.forward_compatibility_horizon(2019, 6, 7): if test.is_gpu_available(cuda_only=True): random_seed.set_random_seed(0) x = _input([2, 8, 8, 1]) y = _conv_bn(x) y = nn.dropout(y, rate=0.5) y = _conv_bn(y) y = array_ops.identity(y) optimizer = gradient_descent.GradientDescentOptimizer( learning_rate=0.01) g = optimizer.compute_gradients(y, [x]) output = (y, g) output_val_ref, output_val, cost_graph = self._run(output) node_map = _build_node_map(cost_graph.node) self._assert_output_fp16(node_map, 'Conv2D') self._assert_output_fp16(node_map, 'FusedBatchNormV3') self._assert_output_fp16(node_map, 'dropout/mul') self._assert_output_fp16(node_map, 'Conv2D_1') output_val_ref, output_val, cost_graph = self._run(output) self.assertAllClose(output_val_ref, output_val, atol=1e-3, rtol=1e-3)
def benchmarkBatchMatMulBroadcast(self): for (a_shape, b_shape) in self.shape_pairs: with compat.forward_compatibility_horizon(2019, 4, 26): with ops.Graph().as_default(), \ session.Session(config=benchmark.benchmark_config()) as sess, \ ops.device("/cpu:0"): matrix_a = variables.Variable( GetRandomNormalInput(a_shape, np.float32)) matrix_b = variables.Variable( GetRandomNormalInput(b_shape, np.float32)) variables.global_variables_initializer().run() # Use batch matmul op's internal broadcasting. self.run_op_benchmark( sess, math_ops.matmul(matrix_a, matrix_b), min_iters=50, name="batch_matmul_cpu_{}_{}".format(a_shape, b_shape)) # Manually broadcast the input matrices using the broadcast_to op. broadcasted_batch_shape = array_ops.broadcast_static_shape( matrix_a.shape[:-2], matrix_b.shape[:-2]) broadcasted_a_shape = broadcasted_batch_shape.concatenate( matrix_a.shape[-2:]) broadcasted_b_shape = broadcasted_batch_shape.concatenate( matrix_b.shape[-2:]) self.run_op_benchmark( sess, math_ops.matmul( array_ops.broadcast_to(matrix_a, broadcasted_a_shape), array_ops.broadcast_to(matrix_b, broadcasted_b_shape)), min_iters=50, name="batch_matmul_manual_broadcast_cpu_{}_{}".format( a_shape, b_shape))
def testExternalStatePolicyFail(self): with compat.forward_compatibility_horizon(2019, 11, 30): with ops.device(self._device0): dataset0 = dataset_ops.Dataset.range(100).map( lambda _: random_ops.random_uniform( # pylint:disable=g-long-lambda [], minval=1, maxval=10, dtype=dtypes.float32)) opt = dataset_ops.Options() opt.experimental_external_state_policy = ( dataset_ops.ExternalStatePolicy.FAIL) dataset0 = dataset0.with_options(opt) with self.assertRaises(errors.FailedPreconditionError): replicated_ds = distribute.replicate(dataset0, [self._device1, self._device2]) dataset1 = replicated_ds[self._device1] dataset2 = replicated_ds[self._device2] with ops.device(self._device0): get_next0 = self.getNext(dataset0) with ops.device(self._device1): get_next1 = self.getNext(dataset1) with ops.device(self._device2): get_next2 = self.getNext(dataset2) for _ in range(100): self.evaluate(get_next0()) self.evaluate(get_next1()) self.evaluate(get_next2())
def testAllowStatefulOp(self): with compat.forward_compatibility_horizon(2019, 9, 12): with ops.device(self._device0): dataset0 = dataset_ops.Dataset.range(100).map( lambda _: random_ops.random_uniform( # pylint:disable=g-long-lambda [], minval=1, maxval=10, dtype=dtypes.float32)) opt = dataset_ops.Options() opt.experimental_allow_stateful = True dataset0 = dataset0.with_options(opt) replicated_ds = distribute.replicate(dataset0, [self._device1, self._device2]) dataset1 = replicated_ds[self._device1] dataset2 = replicated_ds[self._device2] with ops.device(self._device0): get_next0 = self.getNext(dataset0) with ops.device(self._device1): get_next1 = self.getNext(dataset1) with ops.device(self._device2): get_next2 = self.getNext(dataset2) for _ in range(100): self.evaluate(get_next0()) self.evaluate(get_next1()) self.evaluate(get_next2())
def testCopyToDevicePingPongCPUGPU(self): if not test_util.is_gpu_available(): self.skipTest("No GPU available") with compat.forward_compatibility_horizon(2018, 8, 4): host_dataset = dataset_ops.Dataset.range(10) device_dataset = host_dataset.apply( prefetching_ops.copy_to_device("/gpu:0", source_device="/cpu:0")) back_to_cpu_dataset = device_dataset.apply( prefetching_ops.copy_to_device("/cpu:0", source_device="/gpu:0")) with ops.device("/cpu:0"): iterator = dataset_ops.make_initializable_iterator( back_to_cpu_dataset) next_element = iterator.get_next() with self.cached_session(config=config_pb2.ConfigProto( allow_soft_placement=False)): self.evaluate(iterator.initializer) for i in range(10): self.assertEqual(i, self.evaluate(next_element)) with self.assertRaises(errors.OutOfRangeError): self.evaluate(next_element)
def testSwitchingConditionalAccumulatorForV1(self): # Test into the future. with compat.forward_compatibility_horizon(2019, 8, 9): train_input_fn = _make_train_input_fn_dataset( is_classification=True) predict_input_fn = numpy_io.numpy_input_fn(x=FEATURES_DICT, y=None, batch_size=1, num_epochs=1, shuffle=False) est = boosted_trees.boosted_trees_classifier_train_in_memory( train_input_fn=train_input_fn, feature_columns=self._numeric_feature_columns, n_trees=1, max_depth=5, quantile_sketch_epsilon=0.33) # It will stop after 5 steps because of the max depth and num trees. self._assert_checkpoint(est.model_dir, global_step=5, finalized_trees=1, attempted_layers=5, bucket_boundaries=[[-2.001, -1.999, 12.5], [-3., 0.4995, 2.], [-100., 20., 102.75]]) eval_res = est.evaluate(input_fn=train_input_fn, steps=1) self.assertAllClose(eval_res['accuracy'], 1.0) predictions = list(est.predict(input_fn=predict_input_fn)) self.assertAllClose([[0], [1], [1], [0], [0]], [pred['class_ids'] for pred in predictions])
def test_empty(self): def check(equation, input_shapes, output_shape): # All these cases result in an output filled with zeros, so we don't call # np.einsum. Also np.einsum doesn't support generalized diagonals which # are needed for EinsumOp gradients. r = np.random.RandomState(0) inputs = [np.array(r.randn(*shape)) for shape in input_shapes] input_tensors = [ constant_op.constant(x, shape=x.shape) for x in inputs ] output = self.evaluate( special_math_ops.einsum(equation, *input_tensors)) self.assertAllClose(output, np.zeros(output_shape), atol=1e-4, rtol=1e-4) # Contractions along zero-sized dimensons. check('ab,bc->ac', [(0, 10), (10, 10)], (0, 10)) # From transformer xl. check('ibnd,ijbn->jnd', [(1, 0, 5, 10), (1, 1, 0, 5)], (1, 5, 10)) with compat.forward_compatibility_horizon(2019, 10, 19): # Generalized traces with zero-sized dimensions. check('aab,bc->ac', [(0, 0, 10), (10, 10)], (0, 10)) check('aaab,bc->c', [(0, 0, 0, 3), (3, 4)], (4, ))
def test_scope_disable(self): """Test graph with convolution followed by batch norm.""" with compat.forward_compatibility_horizon(2019, 11, 11): if test.is_gpu_available(cuda_only=True): random_seed.set_random_seed(0) y = _input([2, 8, 8, 1]) with auto_mixed_precision_scope(False): x = _conv_bn(y) with auto_mixed_precision_scope(True): x = _conv_bn(x) output = gradients.gradients(x, [y]) output_val_ref, output_val, cost_graph, partition_graphs = self._run( output) node_map = _build_node_map(cost_graph.node) num_to_fp16, num_to_fp32 = _count_casts(partition_graphs[0]) self._assert_output_fp32(node_map, 'Conv2D') self._assert_output_fp32(node_map, 'FusedBatchNormV3') self._assert_output_fp16(node_map, 'Conv2D_1') self._assert_output_fp32(node_map, 'FusedBatchNormV3_1') self._assert_output_fp32( node_map, 'gradients/Conv2D_grad/Conv2DBackpropInput') self._assert_output_fp16( node_map, 'gradients/Conv2D_1_grad/Conv2DBackpropInput') self.assertEqual(num_to_fp16, 2) # Before Conv2D_1:0, Conv2D_1:1 self.assertEqual(num_to_fp32, 2) # After Conv2D_1 and Conv2D_1_grad self.assertAllClose(output_val_ref, output_val, atol=1e-3, rtol=1e-3)
def test_conv3d_bn(self): """Test graph with convolution followed by batch norm.""" # TODO(nluehr) re-enable once nvbug 3098132 is fixed. self.skipTest('Test case should be skipped when cuDNN < 8.0.4') with compat.forward_compatibility_horizon(2019, 6, 7): if test.is_gpu_available(cuda_only=True): random_seed.set_random_seed(0) x = _input([2, 8, 8, 8, 1]) x = _conv3d_bn(x) output = _conv3d_bn(x) output_val_ref, output_val, cost_graph, partition_graphs = self._run( output) node_map = _build_node_map(cost_graph.node) num_to_fp16, num_to_fp32 = _count_casts(partition_graphs[0]) self._assert_output_fp16(node_map, 'Conv3D') self._assert_output_fp16(node_map, 'FusedBatchNormV3') self._assert_output_fp16(node_map, 'Conv3D_1') self.assertEqual(num_to_fp16, 3) # Before Conv3D:0, Conv3D:1, Conv3D_1:1 self.assertEqual(num_to_fp32, 1) # After FusedBatchNormV3:0 self.assertAllClose(output_val_ref, output_val, atol=1e-2, rtol=1e-2)
def test_gradients_numerical(self, signal_length, frame_length, frame_step, fft_length, np_rtype, forward_tol, backward_tol): # Enable float64 support for RFFTs. with compat.forward_compatibility_horizon(2019, 10, 13): # TODO(rjryan): Investigate why STFT gradient error is so high. signal = np.random.rand(signal_length).astype(np_rtype) * 2 - 1 def forward(signal): return spectral_ops.stft(signal, frame_length, frame_step, fft_length, pad_end=False) ((f_jacob_t, ), (f_jacob_n, )) = gradient_checker_v2.compute_gradient( forward, [signal]) self.assertAllClose(f_jacob_t, f_jacob_n, rtol=forward_tol, atol=forward_tol) def backward(stft): return spectral_ops.inverse_stft(stft, frame_length, frame_step, fft_length) stft = forward(signal) ((b_jacob_t, ), (b_jacob_n, )) = gradient_checker_v2.compute_gradient( backward, [stft]) self.assertAllClose(b_jacob_t, b_jacob_n, rtol=backward_tol, atol=backward_tol)
def testStaticRegexReplaceDelegation(self): with compat.forward_compatibility_horizon(2018, 10, 11): with self.test_session(): input_vector = constant_op.constant("foo", dtypes.string) pattern = "[a-z]" replace = "." op = string_ops.regex_replace(input_vector, pattern, replace) self.assertTrue(op.name.startswith("StaticRegexReplace"))
def test_repeated_indices(self): with compat.forward_compatibility_horizon(2019, 10, 19): # Repeated indices. self._check('ijj,k->ik', (2, 3, 3), (4, )) self._check('aba,a->b', (3, 4, 3), (3, )) # From https://github.com/dask/dask/pull/3412#discussion_r182413444 self._check('aab,bc->ac', (2, 2, 3), (3, 4)) self._check('aab,bcc->ac', (2, 2, 3), (3, 4, 4))
def test3DTensorAsInputNoReshape(self): with compat.forward_compatibility_horizon(2018, 8, 27): self._testSoftmax( np.array([[[1., 1., 1., 1.], [1., 2., 3., 4.]], [[2., 3., 4., 5.], [6., 7., 8., 9.]], [[5., 4., 3., 2.], [1., 2., 3., 4.]]]).astype(np.float32), use_gpu=False) self._testOverflow(use_gpu=False)
def testRegexReplaceDelegation(self, pattern_fn, rewrite_fn): with compat.forward_compatibility_horizon(2018, 10, 11): with self.test_session(): input_vector = constant_op.constant("foo", dtypes.string) pattern = pattern_fn("[a-z]") replace = rewrite_fn(".") op = string_ops.regex_replace(input_vector, pattern, replace) self.assertTrue(op.name.startswith("RegexReplace"))
def _forward_compat_context(np_dtype): @contextlib.contextmanager def null_context(): yield if np_dtype in (np.float64, np.complex128): return compat.forward_compatibility_horizon(2019, 10, 13) else: return null_context()
def test_decorator(self): compatibility_date = self._compatibility_date() one_day_after = self._n_days_after(1) with compat.forward_compatibility_horizon(*one_day_after): self.assertTrue(compat.forward_compatible(*compatibility_date)) self.assertFalse(compat.forward_compatible(*one_day_after)) # After exiting context manager, value should be reset. self.assertFalse(compat.forward_compatible(*compatibility_date))
def testGatherNdResourceVariable(self): with compat.forward_compatibility_horizon(2019, 4, 30): with self.cached_session(): v = resource_variable_ops.ResourceVariable( constant_op.constant([[1, 2], [3, 4], [5, 6]])) self.evaluate(variables.global_variables_initializer()) gather = array_ops.gather_nd(v, [[0, 1], [2, 0]]) if not context.executing_eagerly(): # .op doesn't make sense in Eager self.assertEqual("ResourceGatherNd", gather.op.inputs[0].op.type) self.assertAllEqual([2, 5], gather)
def testBatchDims(self, params, indices, batch_dims, expected=None, axis=None): result = array_ops.gather(params, indices, axis=axis, batch_dims=batch_dims) self.assertAllEqual(expected, result) with compat.forward_compatibility_horizon(2019, 6, 11): result = array_ops.gather( params, indices, axis=axis, batch_dims=batch_dims) self.assertAllEqual(expected, result)
def testStaticRegexFullMatchDelegation(self): with compat.forward_compatibility_horizon(2018, 11, 20): with self.cached_session(): input_tensor = constant_op.constant("foo", dtypes.string) pattern = "[a-z]*" op = string_ops.regex_full_match(input_tensor, pattern) self.assertTrue(op.name.startswith("StaticRegexFullMatch"), op.name) pattern_tensor = constant_op.constant("[a-z]*", dtypes.string) op_vec = string_ops.regex_full_match(input_tensor, pattern_tensor) self.assertTrue(op_vec.name.startswith("RegexFullMatch"), op.name)
def testStaticRegexFullMatchDelegation(self): with compat.forward_compatibility_horizon(2018, 11, 20): with self.test_session(): input_tensor = constant_op.constant("foo", dtypes.string) pattern = "[a-z]*" op = string_ops.regex_full_match(input_tensor, pattern) self.assertTrue(op.name.startswith("StaticRegexFullMatch"), op.name) pattern_tensor = constant_op.constant("[a-z]*", dtypes.string) op_vec = string_ops.regex_full_match(input_tensor, pattern_tensor) self.assertTrue(op_vec.name.startswith("RegexFullMatch"), op.name)
def _compute_stft_gradient(signal, frame_length=32, frame_step=16, fft_length=32): """Computes the gradient of the STFT with respect to `signal`.""" # Enable float64 support for RFFTs. with compat.forward_compatibility_horizon(2019, 10, 13): stft = spectral_ops.stft(signal, frame_length, frame_step, fft_length) magnitude_stft = math_ops.abs(stft) loss = math_ops.reduce_sum(magnitude_stft) return gradients_impl.gradients([loss], [signal])[0]
def test_broadcasting(self): with compat.forward_compatibility_horizon(2019, 10, 19): self._check_gradient('...ij,...jk->...ik', (3, 2), (2, 4)) self._check_gradient('ij...,jk...->ik...', (3, 2, 1), (2, 4)) self._check_gradient('...ij,...jk->...ik', (3, 1, 3, 2), (1, 5, 2, 4)) self._check_gradient('ij,jk...k->i...', (3, 2), (2, 4, 1, 4)) self._check_gradient('aab,b...c->a...c', (1, 1, 3), (3, 1, 1, 4)) # Tests from dask. self._check_gradient('...i,...j,...k->...ijk', (1, 4, 1, 2), (5, 1, 1, 3), (1, 1, 1, 1, 9)) self._check_gradient('...i,...j,...k->...ijk', (1, ), (1, ), (1, ))
def Test(self): def CheckGradients(self, a_shape, b_shape): self._compare(a_shape, b_shape, dtype, adjoint_a, adjoint_b) with compat.forward_compatibility_horizon(2019, 4, 19): CheckGradients(self, [1, 5, 2, 3], [7, 1, 3, 2]) CheckGradients(self, [2, 3], [1, 3, 5]) CheckGradients(self, [2, 3], [5, 3, 5]) CheckGradients(self, [5, 2, 5], [5, 3]) CheckGradients(self, [5, 2, 2, 3], [3, 5]) CheckGradients(self, [4, 5, 1, 2, 3], [1, 1, 3, 5]) CheckGradients(self, [1, 2, 1, 4, 2, 1, 3, 4], [3, 2, 1, 1, 1, 2, 4, 2])
def testUnary(self): for dtype in self.float_types: self._testUnary( lambda x: special_math_ops.einsum('ijk->kji', x), np.array([[[1, 3], [2, 5], [6, 8]]], dtype=dtype), expected=np.array([[[1], [2], [6]], [[3], [5], [8]]], dtype=dtype)) with compat.forward_compatibility_horizon(2019, 10, 19): self._testUnary( lambda x: special_math_ops.einsum('ijk->kji', x), np.array([[[1, 3], [2, 5], [6, 8]]], dtype=dtype), expected=np.array([[[1], [2], [6]], [[3], [5], [8]]], dtype=dtype))
def test_numpy_input(self): with compat.forward_compatibility_horizon(2019, 10, 19): # In addition to Tensors, we also support raw numpy arrays as inputs. r = np.random.RandomState(0) s = 'ijk,ijl,ikl->i' x = r.randn(1, 2, 3) y = r.randn(1, 2, 4) z = r.randn(1, 3, 4) a = np.einsum(s, x, y, z) b = self.evaluate(special_math_ops.einsum(s, x, y, z)) self.assertAllClose(a, b, atol=1e-4, rtol=1e-4)
def test_environment_override(self): var_name = 'TF_FORWARD_COMPATIBILITY_DELTA_DAYS' def remove_os_environment_var(): try: del os.environ[var_name] except KeyError: pass self.addCleanup(remove_os_environment_var) compatibility_date = self._compatibility_date() one_day_before = self._n_days_after(-1) one_day_after = self._n_days_after(1) ten_days_after = self._n_days_after(10) nine_days_after = self._n_days_after(9) self.assertTrue(compat.forward_compatible(*one_day_before)) self.assertFalse(compat.forward_compatible(*compatibility_date)) self.assertFalse(compat.forward_compatible(*one_day_after)) self.assertFalse(compat.forward_compatible(*nine_days_after)) self.assertFalse(compat.forward_compatible(*ten_days_after)) os.environ[var_name] = '10' compat._update_forward_compatibility_date_number() self.assertTrue(compat.forward_compatible(*one_day_before)) self.assertTrue(compat.forward_compatible(*compatibility_date)) self.assertTrue(compat.forward_compatible(*one_day_after)) self.assertTrue(compat.forward_compatible(*nine_days_after)) self.assertFalse(compat.forward_compatible(*ten_days_after)) del os.environ[var_name] compat._update_forward_compatibility_date_number() self.assertTrue(compat.forward_compatible(*one_day_before)) self.assertFalse(compat.forward_compatible(*compatibility_date)) self.assertFalse(compat.forward_compatible(*one_day_after)) self.assertFalse(compat.forward_compatible(*nine_days_after)) self.assertFalse(compat.forward_compatible(*ten_days_after)) # Now test interaction between environment variable and context func. os.environ[var_name] = '10' compat._update_forward_compatibility_date_number() self.assertTrue(compat.forward_compatible(*one_day_after)) with compat.forward_compatibility_horizon(*one_day_after): self.assertTrue(compat.forward_compatible(*one_day_before)) self.assertTrue(compat.forward_compatible(*compatibility_date)) self.assertFalse(compat.forward_compatible(*one_day_after)) self.assertFalse(compat.forward_compatible(*nine_days_after)) self.assertFalse(compat.forward_compatible(*ten_days_after)) self.assertTrue(compat.forward_compatible(*one_day_after))
def test_fused_batch_norm(self): with compat.forward_compatibility_horizon(2019, 6, 7): data_formats = ["NHWC"] if test.is_gpu_available(): data_formats.append("NCHW") for is_training in (True, False): for data_format in data_formats: with backprop.GradientTape(persistent=True) as g: if data_format == "NCHW": x = random_ops.random_uniform([3, 1, 2, 5, 5]) else: x = random_ops.random_uniform([3, 1, 5, 5, 2]) g.watch(x) scale = random_ops.random_uniform([2]) g.watch(scale) offset = random_ops.random_uniform([2]) g.watch(offset) mean = None if is_training else random_ops.random_uniform([2]) variance = None if is_training else random_ops.random_uniform([2]) # pylint: disable=cell-var-from-loop def loop_fn(i): with g: x1 = array_ops.gather(x, i) outputs = nn.fused_batch_norm( x1, scale, offset, mean=mean, variance=variance, epsilon=0.01, data_format=data_format, is_training=is_training) outputs = list(outputs) # We only test the first value of outputs when is_training is # False. It looks like CPU and GPU have different outputs for # batch_mean and batch_variance for this case. if not is_training: outputs[1] = constant_op.constant(0.) outputs[2] = constant_op.constant(0.) loss = nn.l2_loss(outputs[0]) if is_training: gradients = g.gradient(loss, [x1, scale, offset]) else: gradients = [constant_op.constant(0.)] * 3 return outputs + gradients # pylint: enable=cell-var-from-loop self._test_loop_fn(loop_fn, 3)
def test_decorator_with_failure(self): compatibility_date = self._compatibility_date() one_day_after = self._n_days_after(1) class DummyError(Exception): pass try: with compat.forward_compatibility_horizon(*one_day_after): raise DummyError() except DummyError: pass # silence DummyError # After exiting context manager, value should be reset. self.assertFalse(compat.forward_compatible(*compatibility_date))
def testNMS3Then2WithScoreThresh(self): # Three boxes are selected based on IOU. # One is filtered out by score threshold. # TODO(b/26783907): The Sort HLO is not implemented on CPU or GPU. if self.device in ["XLA_CPU", "XLA_GPU"]: return with compat.forward_compatibility_horizon(2018, 8, 8): boxes_data = [[0, 0, 1, 1], [0, 0.1, 1, 1.1], [0, -0.1, 1, 0.9], [0, 10, 1, 11], [0, 10.1, 1, 11.1], [0, 100, 1, 101]] boxes_np = np.array(boxes_data, dtype=np.float32) scores_data = [0.9, 0.75, 0.6, 0.95, 0.5, 0.3] scores_np = np.array(scores_data, dtype=np.float32) max_output_size = 3 iou_threshold_np = np.array(0.5, dtype=np.float32) score_threshold_np = np.array(0.4, dtype=np.float32) with self.cached_session() as sess: boxes = array_ops.placeholder(boxes_np.dtype, shape=boxes_np.shape) scores = array_ops.placeholder(scores_np.dtype, shape=scores_np.shape) iou_threshold = array_ops.placeholder(iou_threshold_np.dtype, iou_threshold_np.shape) score_threshold = array_ops.placeholder(score_threshold_np.dtype, score_threshold_np.shape) with self.test_scope(): selected_indices = image_ops.non_max_suppression_padded( boxes=boxes, scores=scores, max_output_size=max_output_size, iou_threshold=iou_threshold, score_threshold=score_threshold, pad_to_max_output_size=True) inputs_feed = { boxes: boxes_np, scores: scores_np, iou_threshold: iou_threshold_np, score_threshold: score_threshold_np } (indices_tf, num_valid) = sess.run( selected_indices, feed_dict=inputs_feed) self.assertEqual(indices_tf.size, max_output_size) self.assertEqual(num_valid, 2) self.assertAllClose(indices_tf[:num_valid], [3, 0])
def testGradGradFloat32(self): with compat.forward_compatibility_horizon(2018, 11, 2): with self.test_session(): x = constant_op.constant( [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], shape=[2, 5], name="x") y = nn_ops.leaky_relu(x, alpha=0.1, name="leaky_relu") z = gradients_impl.gradients(y, x) x_init = np.asarray( [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], dtype=np.float32, order="F") err = gradient_checker.compute_gradient_error( x, [2, 5], z[0], [2, 5], x_init_value=x_init) print("leaky_relu (float32) gradient of gradient err = ", err) self.assertLess(err, 1e-4)
def testBasicGpu(self): if not test_util.is_gpu_available(): self.skipTest("No GPU available") with compat.forward_compatibility_horizon(2018, 8, 4): dataset = dataset_ops.Dataset.range(10) multi_device_iterator = prefetching_ops.MultiDeviceIterator( dataset, ["/cpu:1", "/gpu:0"]) elem_on_1, elem_on_2 = multi_device_iterator.get_next() config = config_pb2.ConfigProto(device_count={"CPU": 2, "GPU": 1}) with self.test_session(config=config) as sess: sess.run(multi_device_iterator.initializer) for i in range(0, 10, 2): self.assertEqual(i, sess.run(elem_on_1)) self.assertEqual(i + 1, sess.run(elem_on_2)) with self.assertRaises(errors.OutOfRangeError): sess.run(elem_on_1) sess.run(elem_on_2)
def testGradGradFloat64(self): with compat.forward_compatibility_horizon(2018, 11, 2): with self.cached_session(): def f(x): assert x.dtype == dtypes.float64 with backprop.GradientTape() as tape: tape.watch(x) y = nn_ops.leaky_relu(x) return tape.gradient(y, x) x = np.asarray( [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], dtype=np.float64, order="F") err = gradient_checker_v2.max_error( *gradient_checker_v2.compute_gradient(f, [x])) print("leaky_relu (float64) gradient of gradient err = ", err) self.assertLess(err, 1e-10)
def testNMS3From6Boxes(self): with compat.forward_compatibility_horizon(2018, 8, 8): # Three boxes are selected based on IOU. boxes_data = [[0, 0, 1, 1], [0, 0.1, 1, 1.1], [0, -0.1, 1, 0.9], [0, 10, 1, 11], [0, 10.1, 1, 11.1], [0, 100, 1, 101]] boxes_np = np.array(boxes_data, dtype=np.float32) scores_data = [0.9, 0.75, 0.6, 0.95, 0.5, 0.3] scores_np = np.array(scores_data, dtype=np.float32) max_output_size = 3 iou_threshold_np = np.array(0.5, dtype=np.float32) score_threshold_np = np.array(0.0, dtype=np.float32) with self.cached_session() as sess: boxes = array_ops.placeholder(boxes_np.dtype, shape=boxes_np.shape) scores = array_ops.placeholder(scores_np.dtype, shape=scores_np.shape) iou_threshold = array_ops.placeholder(iou_threshold_np.dtype, iou_threshold_np.shape) score_threshold = array_ops.placeholder(score_threshold_np.dtype, score_threshold_np.shape) with self.test_scope(): selected_indices = image_ops.non_max_suppression_padded( boxes=boxes, scores=scores, max_output_size=max_output_size, iou_threshold=iou_threshold, score_threshold=score_threshold, pad_to_max_output_size=True) inputs_feed = { boxes: boxes_np, scores: scores_np, score_threshold: score_threshold_np, iou_threshold: iou_threshold_np } (indices_tf, num_valid) = sess.run( selected_indices, feed_dict=inputs_feed) self.assertEqual(indices_tf.size, max_output_size) self.assertEqual(num_valid, 3) self.assertAllClose(indices_tf[:num_valid], [3, 0, 5])
def testCopyToDevicePingPongCPUGPU(self): if not test_util.is_gpu_available(): self.skipTest("No GPU available") with compat.forward_compatibility_horizon(2018, 8, 4): host_dataset = dataset_ops.Dataset.range(10) device_dataset = host_dataset.apply( prefetching_ops.copy_to_device("/gpu:0", source_device="/cpu:0")) back_to_cpu_dataset = device_dataset.apply( prefetching_ops.copy_to_device("/cpu:0", source_device="/gpu:0")) with ops.device("/cpu:0"): iterator = back_to_cpu_dataset.make_initializable_iterator() next_element = iterator.get_next() with self.cached_session() as sess: sess.run(iterator.initializer) for i in range(10): self.assertEqual(i, sess.run(next_element)) with self.assertRaises(errors.OutOfRangeError): sess.run(next_element)
def test1DTensorAsInputNoReshape(self): with compat.forward_compatibility_horizon(2018, 8, 27): self._testSoftmax( np.array([3., 2., 3., 9.]).astype(np.float64), use_gpu=False) self._testOverflow(use_gpu=False)
def Test(self): with compat.forward_compatibility_horizon(2019, 4, 19): np.random.seed(42) self._testBroadcasting(dtype, adjoint_a, adjoint_b, use_static_shape)
def testIteratorStringHandleFuture(self): with forward_compat.forward_compatibility_horizon(2018, 8, 4): dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3]) dataset_4 = dataset_ops.Dataset.from_tensor_slices([10, 20, 30, 40]) iterator_3 = dataset_ops.make_one_shot_iterator(dataset_3) iterator_4 = dataset_ops.make_one_shot_iterator(dataset_4) handle_placeholder = array_ops.placeholder(dtypes.string, shape=[]) feedable_iterator = iterator_ops.Iterator.from_string_handle( handle_placeholder, dataset_ops.get_legacy_output_types(dataset_3), dataset_ops.get_legacy_output_shapes(dataset_3)) next_element = feedable_iterator.get_next() self.assertTrue(dataset_ops.get_structure(dataset_3).is_compatible_with( dataset_ops.get_structure(feedable_iterator))) self.assertTrue(dataset_ops.get_structure(dataset_4).is_compatible_with( dataset_ops.get_structure(feedable_iterator))) with self.cached_session() as sess: iterator_3_handle = sess.run(iterator_3.string_handle()) iterator_4_handle = sess.run(iterator_4.string_handle()) self.assertEqual( 10, sess.run( next_element, feed_dict={handle_placeholder: iterator_4_handle})) self.assertEqual( 1, sess.run( next_element, feed_dict={handle_placeholder: iterator_3_handle})) self.assertEqual( 20, sess.run( next_element, feed_dict={handle_placeholder: iterator_4_handle})) self.assertEqual( 2, sess.run( next_element, feed_dict={handle_placeholder: iterator_3_handle})) self.assertEqual( 30, sess.run( next_element, feed_dict={handle_placeholder: iterator_4_handle})) self.assertEqual( 3, sess.run( next_element, feed_dict={handle_placeholder: iterator_3_handle})) self.assertEqual( 40, sess.run( next_element, feed_dict={handle_placeholder: iterator_4_handle})) with self.assertRaises(errors.OutOfRangeError): sess.run( next_element, feed_dict={handle_placeholder: iterator_3_handle}) with self.assertRaises(errors.OutOfRangeError): sess.run( next_element, feed_dict={handle_placeholder: iterator_4_handle})