def approximate_hessian(self, grads_and_vars, name=None): """ I haven't tested this yet so I have no idea if it works, but even if it does it's probably super slow, and either way nothing else has been modified to deal with it. """ gv = 0 var_refs = [] for g_t, x_tm1 in grads_and_vars: var_refs.append(x_tm1.ref()) if g_t is None: continue with ops.name_scope('update_' + x_tm1.op.name), ops.device(x_tm1.device): if isinstance(g_t, ops.Tensor): gv += math_ops.reduce_sum(g_t * random_ops.random_normal(g_t.get_shape())) else: idxs, idxs_ = array_ops.unique(g_t.indices) g_t_ = math_ops.unsorted_segment_sum(g_t.values, idxs_, array_ops.size(idxs)) gv += math_ops.reduce_sum(g_t_ * random_ops.random_normal(g_t_.get_shape())) hesses = gradients.gradients(gv, var_refs, gate_gradients=(gate_gradients == Optimizer.GATE_OP), aggregation_method=aggregation_method, colocate_gradients_with_ops=colocate_gradients_with_ops) return zip([g_t for g_t, _ in grads_and_vars], [x_tm1 for _, x_tm1 in grads_and_vars], hesses)
def testConcurrentExecutesWithoutError(self): with self.test_session(use_gpu=True) as sess: all_ops = [] for compute_uv_ in True, False: for full_matrices_ in True, False: matrix1 = random_ops.random_normal([5, 5], seed=42) matrix2 = random_ops.random_normal([5, 5], seed=42) if compute_uv_: s1, u1, v1 = linalg_ops.svd( matrix1, compute_uv=compute_uv_, full_matrices=full_matrices_) s2, u2, v2 = linalg_ops.svd( matrix2, compute_uv=compute_uv_, full_matrices=full_matrices_) all_ops += [s1, u1, v1, s2, u2, v2] else: s1 = linalg_ops.svd( matrix1, compute_uv=compute_uv_, full_matrices=full_matrices_) s2 = linalg_ops.svd( matrix2, compute_uv=compute_uv_, full_matrices=full_matrices_) all_ops += [s1, s2] val = sess.run(all_ops) for i in range(2): s = 6 * i self.assertAllEqual(val[s], val[s + 3]) # s1 == s2 self.assertAllEqual(val[s + 1], val[s + 4]) # u1 == u2 self.assertAllEqual(val[s + 2], val[s + 5]) # v1 == v2 for i in range(2): s = 12 + 2 * i self.assertAllEqual(val[s], val[s + 1]) # s1 == s2
def testMultiplyInverseNotTupleWithBias(self): with ops.Graph().as_default(), self.test_session() as sess: random_seed.set_random_seed(200) params = [random_ops.random_normal((2, 2, 2, 2))] inputs = random_ops.random_normal((2, 2, 2, 2)) outputs = random_ops.random_normal((2, 2, 2, 2)) block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, (1, 1, 1, 1), 'SAME') block.register_additional_minibatch(inputs, outputs) self.assertTrue(block._has_bias) grads = outputs**2 block.instantiate_factors(((grads,),), 0.5) block._input_factor.instantiate_cov_variables() block._output_factor.instantiate_cov_variables() block.register_inverse() block._input_factor.instantiate_inv_variables() block._output_factor.instantiate_inv_variables() # Make sure our inverse is something other than the identity. sess.run(tf_variables.global_variables_initializer()) sess.run(block._input_factor.make_inverse_update_ops()) sess.run(block._output_factor.make_inverse_update_ops()) vector = np.arange(1, 19).reshape(9, 2).astype(np.float32) output = block.multiply_inverse(array_ops.constant(vector)) self.assertAllClose([0.136455, 0.27291], sess.run(output)[0])
def test_optimize(self): scalar = variables.Variable(random_ops.random_normal([]), 'scalar') vector = variables.Variable(random_ops.random_normal([2]), 'vector') matrix = variables.Variable(random_ops.random_normal([2, 3]), 'matrix') minimum_location = constant_op.constant(np.arange(9), dtype=dtypes.float32) loss = math_ops.reduce_sum( math_ops.square(vector - minimum_location[:2])) / 2. loss += math_ops.reduce_sum( math_ops.square(scalar - minimum_location[2])) / 2. loss += math_ops.reduce_sum( math_ops.square( matrix - array_ops.reshape(minimum_location[3:], [2, 3]))) / 2. optimizer = MockOptimizerInterface(loss) with self.test_session() as sess: sess.run(variables.global_variables_initializer()) optimizer.minimize(sess) self.assertAllClose(np.arange(2), sess.run(vector)) self.assertAllClose(np.arange(1) + 2, sess.run(scalar)) self.assertAllClose(np.arange(6).reshape(2, 3) + 3, sess.run(matrix))
def testDistributedOOM(self): if not test.is_gpu_available(): return ops.reset_default_graph() workers, _ = test_util.create_local_cluster(2, 0) with ops.device('/job:worker/replica:0/task:0/gpu:0'): a = random_ops.random_normal([1, 10000, 20000], name='test_random1') with ops.device('/job:worker/replica:0/task:1/gpu:0'): b = random_ops.random_normal([30000, 10000, 1], name='test_random2') c = a * b try: with session.Session(workers[1].target) as sess: sess.run(c, options=config_pb2.RunOptions( report_tensor_allocations_upon_oom=True)) except Exception as e: # pylint: disable=broad-except exception_str = '%s' % e # test_random2 is reported because it's allocated in worker 1. self.assertTrue('Current usage from device: ' '/job:worker/replica:0/task:1/device:GPU:0, ' 'allocator: GPU_0_bfc' in exception_str) mat = re.search('(.*)GiB from test_random2/RandomStandardNormal', exception_str) self.assertGreater(float(mat.group(1)), 0.0) # test_random1 is not reported because it's allocated in worker 0. mat = re.search('(.*)MiB from test_random1/RandomStandardNormal', exception_str) self.assertTrue(mat is None)
def testOOM(self): if not test.is_gpu_available(): return ops.reset_default_graph() with ops.device('/device:GPU:0'): a = random_ops.random_normal([1, 10000, 20000], name='test_random1') b = random_ops.random_normal([30000, 10000, 1], name='test_random2') c = a * b try: with session.Session() as sess: sess.run(c, options=config_pb2.RunOptions( report_tensor_allocations_upon_oom=True)) except Exception as e: # pylint: disable=broad-except exception_str = '%s' % e # This trace reports allocations for to random tensor. self.assertTrue( 'OOM when allocating tensor with shape[30000,10000,20000]' in exception_str) mat = re.search('(.*)GiB from test_random2/RandomStandardNormal', exception_str) self.assertGreater(float(mat.group(1)), 0.0) mat = re.search('(.*)MiB from test_random1/RandomStandardNormal', exception_str) self.assertGreater(float(mat.group(1)), 0.0)
def random_normal(shape, mean=0.0, stddev=1.0, dtype=dtypes.float32, seed=None): """Tensor with (possibly complex) Gaussian entries. Samples are distributed like ``` N(mean, stddev^2), if dtype is real, X + iY, where X, Y ~ N(mean, stddev^2) if dtype is complex. ``` Args: shape: `TensorShape` or Python list. Shape of the returned tensor. mean: `Tensor` giving mean of normal to sample from. stddev: `Tensor` giving stdev of normal to sample from. dtype: `TensorFlow` `dtype` or numpy dtype seed: Python integer seed for the RNG. Returns: `Tensor` with desired shape and dtype. """ dtype = dtypes.as_dtype(dtype) with ops.name_scope("random_normal"): samples = random_ops.random_normal( shape, mean=mean, stddev=stddev, dtype=dtype.real_dtype, seed=seed) if dtype.is_complex: if seed is not None: seed += 1234 more_samples = random_ops.random_normal( shape, mean=mean, stddev=stddev, dtype=dtype.real_dtype, seed=seed) samples = math_ops.complex(samples, more_samples) return samples
def testMultiplyInverse(self): with ops.Graph().as_default(), self.test_session() as sess: random_seed.set_random_seed(200) params = random_ops.random_normal((3, 3, 8, 2)) inputs = random_ops.random_normal((32, 5, 5, 8)) outputs = random_ops.random_normal((32, 5, 5, 16)) layer_collection = lc.LayerCollection() block = fb.DepthwiseConvKFCBasicFB( layer_collection, params=params, strides=[1, 1, 1, 1], padding='SAME') block.register_additional_tower(inputs, outputs) grads = outputs**2 block.instantiate_factors(([grads],), 0.5) block._input_factor.instantiate_cov_variables() block._output_factor.instantiate_cov_variables() block.register_inverse() block._input_factor.instantiate_inv_variables() block._output_factor.instantiate_inv_variables() # Ensure inverse update op doesn't crash. sess.run(tf_variables.global_variables_initializer()) sess.run([ factor.make_inverse_update_ops() for factor in layer_collection.get_factors() ]) # Ensure inverse-vector multiply doesn't crash. output = block.multiply_inverse(params) sess.run(output) # Ensure same shape. self.assertAllEqual(output.shape, params.shape)
def testMultiplyInverseTuple(self): with ops.Graph().as_default(), self.test_session() as sess: random_seed.set_random_seed(200) params = random_ops.random_normal((2, 2, 2, 2)) inputs = random_ops.random_normal((2, 2, 2, 2)) outputs = random_ops.random_normal((2, 2, 2, 2)) block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, (1, 1, 1, 1), 'SAME') block.register_additional_minibatch(inputs, outputs) grads = outputs**2 block.instantiate_factors(([grads],), 0.5) # Make sure our inverse is something other than the identity. sess.run(tf_variables.global_variables_initializer()) sess.run(block._input_factor.make_inverse_update_ops()) sess.run(block._output_factor.make_inverse_update_ops()) vector = (np.arange(1, 15).reshape(7, 2).astype(np.float32), np.arange(2, 4).reshape(2, 1).astype(np.float32)) output = block.multiply_inverse((array_ops.constant(vector[0]), array_ops.constant(vector[1]))) output = sess.run(output) self.assertAllClose([0.136455, 0.27291], output[0][0]) self.assertAllClose([0.27291, 0.409365], output[1])
def _testConfMatrixOnTensors(self, tf_dtype, np_dtype): with self.test_session() as sess: m_neg = array_ops.placeholder(dtype=dtypes.float32) m_pos = array_ops.placeholder(dtype=dtypes.float32) s = array_ops.placeholder(dtype=dtypes.float32) neg = random_ops.random_normal([20], mean=m_neg, stddev=s, dtype=dtypes.float32) pos = random_ops.random_normal([20], mean=m_pos, stddev=s, dtype=dtypes.float32) data = array_ops.concat([neg, pos], 0) data = math_ops.cast(math_ops.round(data), tf_dtype) data = math_ops.minimum(math_ops.maximum(data, 0), 1) lab = array_ops.concat([array_ops.zeros([20], dtype=tf_dtype), array_ops.ones([20], dtype=tf_dtype)], 0) cm = confusion_matrix.confusion_matrix(lab, data, dtype=tf_dtype, num_classes=2) d, l, cm_out = sess.run([data, lab, cm], {m_neg: 0.0, m_pos: 1.0, s: 1.0}) truth = np.zeros([2, 2], dtype=np_dtype) try: range_builder = xrange except NameError: # In Python 3. range_builder = range for i in range_builder(len(d)): truth[l[i], d[i]] += 1 self.assertEqual(cm_out.dtype, np_dtype) self.assertAllClose(cm_out, truth, atol=1e-10)
def test_virtual_statistics(self): """Check that `_virtual_statistics` gives same result as `nn.moments`.""" random_seed.set_random_seed(1234) batch_axis = 0 partial_batch = random_ops.random_normal([4, 5, 7, 3]) single_example = random_ops.random_normal([1, 5, 7, 3]) full_batch = array_ops.concat([partial_batch, single_example], axis=0) for reduction_axis in range(1, 4): # Get `nn.moments` on the full batch. reduction_axes = list(range(4)) del reduction_axes[reduction_axis] mom_mean, mom_variance = nn.moments(full_batch, reduction_axes) # Get virtual batch statistics. vb_reduction_axes = list(range(4)) del vb_reduction_axes[reduction_axis] del vb_reduction_axes[batch_axis] vbn = virtual_batchnorm.VBN(partial_batch, reduction_axis) vb_mean, mean_sq = vbn._virtual_statistics( single_example, vb_reduction_axes) vb_variance = mean_sq - math_ops.square(vb_mean) # Remove singleton batch dim for easy comparisons. vb_mean = array_ops.squeeze(vb_mean, batch_axis) vb_variance = array_ops.squeeze(vb_variance, batch_axis) with self.cached_session(use_gpu=True) as sess: vb_mean_np, vb_var_np, mom_mean_np, mom_var_np = sess.run([ vb_mean, vb_variance, mom_mean, mom_variance]) self.assertAllClose(mom_mean_np, vb_mean_np) self.assertAllClose(mom_var_np, vb_var_np)
def _run_model(): x = random_ops.random_normal(shape=[1, SIZE]) w = random_ops.random_normal(shape=[SIZE, 2 * SIZE]) y = math_ops.matmul(x, w) config = config_pb2.ConfigProto() config.graph_options.rewrite_options.arithmetic_optimization = ( rewriter_config_pb2.RewriterConfig.OFF) with session.Session(config=config) as sess: run_metadata = config_pb2.RunMetadata() opts = builder.time_and_memory() opts['min_micros'] = 0 opts['min_bytes'] = 0 opts['order_by'] = 'name' opts['output'] = 'none' _ = sess.run(y, options=config_pb2.RunOptions( trace_level=config_pb2.RunOptions.FULL_TRACE), run_metadata=run_metadata) tfprof_node = model_analyzer.profile( sess.graph, run_meta=run_metadata, options=opts) return tfprof_node, run_metadata
def _testSimpleModel(self, use_forward_func, use_resource=False): def _Model(x): w = variable_scope.get_variable( "w", (64, 64), initializer=init_ops.random_uniform_initializer(seed=312), use_resource=use_resource) b = variable_scope.get_variable( "b", (64), initializer=init_ops.zeros_initializer(), use_resource=use_resource), return math_ops.sigmoid(math_ops.matmul(x, w) + b) @function.Defun() def Model(x): return _Model(x) cvars = [] @function.Defun() def Grad(x, y0): if use_forward_func: y = Model(x) else: y = _Model(x) loss = math_ops.reduce_mean( math_ops.reduce_sum(y0 * math_ops.log(y), 1), 0) arg_w, arg_b = function.get_extra_args() self.assertEqual(arg_w.get_shape(), tensor_shape.TensorShape([64, 64])) self.assertEqual(arg_b.get_shape(), tensor_shape.TensorShape([64])) dw, db = gradients_impl.gradients(loss, [arg_w, arg_b]) cvars.extend(function.get_extra_vars()) return loss, dw, db g = ops.Graph() with g.as_default(): x = random_ops.random_normal([64, 64], seed=100) y0 = random_ops.random_normal([64, 64], seed=200) with variable_scope.variable_scope("Foo"): loss, dw, db = Grad(x, y0) self.assertEqual(2, len(cvars)) w, b = cvars[:2] self.assertEqual("Foo/w", w.op.name) self.assertEqual("Foo/b", b.op.name) with self.test_session(graph=g) as sess: sess.run(variables.global_variables_initializer()) w, b, x, y0, loss, dw, db = sess.run([w, b, x, y0, loss, dw, db]) self.assertAllEqual(w.shape, (64, 64)) self.assertAllClose(np.sum(w), 2050.44) self.assertAllEqual(b.shape, (64,)) self.assertAllClose(np.sum(b), 0.0) self.assertAllClose(loss, -2.27, rtol=1e-2) self.assertAllEqual(dw.shape, (64, 64)) self.assertAllClose(np.sum(dw), -1.04, rtol=1e-2) self.assertAllEqual(db.shape, (64,)) self.assertAllClose(np.sum(db), 0.509, rtol=1e-2)
def testConcurrentExecutesWithoutError(self): with self.session(use_gpu=True) as sess: matrix1 = random_ops.random_normal([5, 5], seed=42) matrix2 = random_ops.random_normal([5, 5], seed=42) det1 = linalg_ops.matrix_determinant(matrix1) det2 = linalg_ops.matrix_determinant(matrix2) det1_val, det2_val = sess.run([det1, det2]) self.assertEqual(det1_val, det2_val)
def testNoCSE(self): for use_gpu in [False, True]: with self.test_session(use_gpu=use_gpu): shape = [2, 3, 4] rnd1 = random_ops.random_normal(shape, 0.0, 1.0, dtypes.float32) rnd2 = random_ops.random_normal(shape, 0.0, 1.0, dtypes.float32) diff = rnd2 - rnd1 self.assertTrue(np.linalg.norm(diff.eval()) > 0.1)
def testConcurrentExecutesWithoutError(self): with self.test_session(use_gpu=True) as sess: matrix1 = random_ops.random_normal([5, 5], seed=42) matrix2 = random_ops.random_normal([5, 5], seed=42) expm1 = linalg_impl.matrix_exponential(matrix1) expm2 = linalg_impl.matrix_exponential(matrix2) expm = sess.run([expm1, expm2]) self.assertAllEqual(expm[0], expm[1])
def testConcurrentExecutesWithoutError(self): matrix1 = random_ops.random_normal([5, 5], seed=42) matrix2 = random_ops.random_normal([5, 5], seed=42) lu1, p1 = linalg_ops.lu(matrix1) lu2, p2 = linalg_ops.lu(matrix2) lu1_val, p1_val, lu2_val, p2_val = self.evaluate([lu1, p1, lu2, p2]) self.assertAllEqual(lu1_val, lu2_val) self.assertAllEqual(p1_val, p2_val)
def testTransposedStatistics(self): a = variables.Variable(random_ops.random_normal([16, 25])) b = variables.Variable(random_ops.random_normal([16, 9])) math_ops.matmul(a, b, transpose_a=True) g = ops.get_default_graph() for op in g.get_operations(): flops = ops.get_stats_for_node_def(g, op.node_def, "flops").value if op.name == "MatMul": self.assertEqual(7200, flops)
def testConcurrentExecutesWithoutError(self): with self.test_session(use_gpu=True) as sess: matrix1 = random_ops.random_normal([5, 5], seed=42) matrix2 = random_ops.random_normal([5, 5], seed=42) sqrt1 = gen_linalg_ops.matrix_square_root(matrix1) sqrt2 = gen_linalg_ops.matrix_square_root(matrix2) all_ops = [sqrt1, sqrt2] sqrt = sess.run(all_ops) self.assertAllEqual(sqrt[0], sqrt[1])
def testConcurrentExecutesWithoutError(self): with self.test_session(use_gpu=True) as sess: matrix1 = random_ops.random_normal([5, 5], seed=42) matrix2 = random_ops.random_normal([5, 5], seed=42) matrix1 = math_ops.matmul(matrix1, matrix1, adjoint_a=True) matrix2 = math_ops.matmul(matrix2, matrix2, adjoint_a=True) c1 = linalg_ops.cholesky(matrix1) c2 = linalg_ops.cholesky(matrix2) c1_val, c2_val = sess.run([c1, c2]) self.assertAllEqual(c1_val, c2_val)
def testSimpleStatistics(self): g = ops.Graph() with g.as_default(): a = variables.Variable(random_ops.random_normal([25, 16])) b = variables.Variable(random_ops.random_normal([16, 9])) math_ops.matmul(a, b) for op in g.get_operations(): flops = ops.get_stats_for_node_def(g, op.node_def, "flops").value if op.name == "MatMul": self.assertEqual(7200, flops)
def testConcurrentExecutesWithoutError(self): with self.session(use_gpu=True) as sess: matrix1 = math_ops.cast( random_ops.random_normal([5, 5], seed=42), dtypes.complex64) matrix2 = math_ops.cast( random_ops.random_normal([5, 5], seed=42), dtypes.complex64) logm1 = gen_linalg_ops.matrix_logarithm(matrix1) logm2 = gen_linalg_ops.matrix_logarithm(matrix2) logm = sess.run([logm1, logm2]) self.assertAllEqual(logm[0], logm[1])
def testEagerSeed(self): with context.eager_mode(): # Ensure a context has been created random_ops.random_normal([]) # Set the same seed twice and check that the values match context.set_global_seed(42) rnd1 = random_ops.random_normal([]) context.set_global_seed(42) rnd2 = random_ops.random_normal([]) self.assertAllEqual(rnd1, rnd2)
def testConcurrentExecutesWithoutError(self): with test_util.use_gpu(): matrix1 = random_ops.random_normal([5, 5], seed=42) matrix2 = random_ops.random_normal([5, 5], seed=42) square1 = math_ops.matmul(matrix1, matrix1) square2 = math_ops.matmul(matrix2, matrix2) sqrt1 = gen_linalg_ops.matrix_square_root(square1) sqrt2 = gen_linalg_ops.matrix_square_root(square2) all_ops = [sqrt1, sqrt2] sqrt = self.evaluate(all_ops) self.assertAllClose(sqrt[0], sqrt[1])
def testRandomNormal(self): # Fully known shape. rnd1 = random_ops.random_normal([1, 2, 3]) self.assertEqual([1, 2, 3], rnd1.get_shape()) # Partially known shape. rnd2 = random_ops.random_normal( array_ops.placeholder(dtypes.int32, shape=(3,))) self.assertEqual([None, None, None], rnd2.get_shape().as_list()) # Unknown shape. rnd3 = random_ops.random_normal(array_ops.placeholder(dtypes.int32)) self.assertIs(None, rnd3.get_shape().ndims)
def testInstantiateFactors(self): with ops.Graph().as_default(): random_seed.set_random_seed(200) params = random_ops.random_normal((3, 3, 8, 2)) inputs = random_ops.random_normal((32, 5, 5, 8)) outputs = random_ops.random_normal((32, 5, 5, 16)) layer_collection = lc.LayerCollection() block = fb.DepthwiseConvKFCBasicFB( layer_collection, params=params, strides=[1, 1, 1, 1], padding='SAME') block.register_additional_tower(inputs, outputs) grads = outputs**2 block.instantiate_factors(([grads],), 0.5)
def testConcurrentExecutesWithoutError(self): with self.session(use_gpu=True) as sess: all_ops = [] for adjoint_ in True, False: matrix1 = random_ops.random_normal([5, 5], seed=42) matrix2 = random_ops.random_normal([5, 5], seed=42) inv1 = linalg_ops.matrix_inverse(matrix1, adjoint=adjoint_) inv2 = linalg_ops.matrix_inverse(matrix2, adjoint=adjoint_) all_ops += [inv1, inv2] inv = sess.run(all_ops) self.assertAllEqual(inv[0], inv[1]) self.assertAllEqual(inv[2], inv[3])
def test_swd_mismatched(self): """Test the inputs mismatched shapes are detected.""" d1 = random_ops.random_uniform([256, 32, 32, 3]) d2 = random_ops.random_normal([256, 32, 31, 3]) d3 = random_ops.random_normal([256, 31, 32, 3]) d4 = random_ops.random_normal([255, 32, 32, 3]) with self.assertRaises(ValueError): swd.sliced_wasserstein_distance(d1, d2) with self.assertRaises(ValueError): swd.sliced_wasserstein_distance(d1, d3) with self.assertRaises(ValueError): swd.sliced_wasserstein_distance(d1, d4)
def _testConvKFCBasicFBInitParams(self, params): with ops.Graph().as_default(): random_seed.set_random_seed(200) if isinstance(params, (list, tuple)): params = [array_ops.constant(param) for param in params] else: params = array_ops.constant(params) inputs = random_ops.random_normal((2, 2, 2)) outputs = random_ops.random_normal((2, 2, 2)) block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, [1, 1, 1], 'SAME') block.register_additional_minibatch(inputs, outputs) self.assertAllEqual([outputs], block.tensors_to_compute_grads())
def testRandomSeed(self): a = random.randint(1, 1000) a_np_rand = np.random.rand(1) with self.test_session(): a_rand = random_ops.random_normal([1]).eval() # ensure that randomness in multiple testCases is deterministic. self.setUp() b = random.randint(1, 1000) b_np_rand = np.random.rand(1) with self.test_session(): b_rand = random_ops.random_normal([1]).eval() self.assertEqual(a, b) self.assertEqual(a_np_rand, b_np_rand) self.assertEqual(a_rand, b_rand)
def test_saved_model(self): def create_model(): inputs = keras.layers.Input(shape=(4, )) outputs = keras.layers.Dense(2)(inputs) model = keras.Model(inputs, outputs) model.compile(optimizer='adam', loss='mean_squared_error') return model with self.strategy.scope(): model = create_model() inputs = random_ops.random_normal(shape=(8, 4)) expect = model(inputs) saved_dir = self.get_temp_dir() model.save(saved_dir) loaded_model = keras.models.load_model(saved_dir) got = loaded_model(inputs) self.assertAllClose(got, expect) self.assertGreater(len(model.variables), len(loaded_model.variables)) with self.assertRaises(ValueError): with self.strategy.scope(): keras.models.load_model(saved_dir)
def __call__for_keras_init_v1(self, shape, dtype=None, partition_info=None): """ Making keras VarianceScaling initializers v1 support dynamic shape. """ if dtype is None: dtype = self.dtype scale = self.scale scale_shape = shape if partition_info is not None: scale_shape = partition_info.full_shape fan_in, fan_out = _compute_fans_for_keras_init_v1_v2(scale_shape) fan_in = math_ops.cast(fan_in, dtype=dtype) fan_out = math_ops.cast(fan_out, dtype=dtype) if self.mode == "fan_in": scale /= math_ops.maximum(1., fan_in) elif self.mode == "fan_out": scale /= math_ops.maximum(1., fan_out) else: scale /= math_ops.maximum(1., (fan_in + fan_out) / 2.) if self.distribution == "normal" or self.distribution == "truncated_normal": # constant taken from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.) stddev = math_ops.sqrt(scale) / .87962566103423978 return random_ops.truncated_normal(shape, 0.0, stddev, dtype, seed=self.seed) elif self.distribution == "untruncated_normal": stddev = math_ops.sqrt(scale) return random_ops.random_normal(shape, 0.0, stddev, dtype, seed=self.seed) else: limit = math_ops.sqrt(3.0 * scale) return random_ops.random_uniform(shape, -limit, limit, dtype, seed=self.seed)
def testEstimatorInitManualRegistration(self): with ops.Graph().as_default(): layer_collection = lc.LayerCollection() inputs = random_ops.random_normal((2, 2), dtype=dtypes.float32) weights = variable_scope.get_variable( 'w', shape=(2, 2), dtype=dtypes.float32) bias = variable_scope.get_variable( 'b', initializer=init_ops.zeros_initializer(), shape=(2, 1)) output = math_ops.matmul(inputs, weights) + bias # Only register the weights. layer_collection.register_fully_connected((weights,), inputs, output) outputs = math_ops.tanh(output) layer_collection.register_categorical_predictive_distribution(outputs) # We should be able to build an estimator for only the registered vars. estimator.FisherEstimator([weights], 0.1, 0.2, layer_collection) # Check that we throw an error if we try to build an estimator for vars # that were not manually registered. with self.assertRaises(ValueError): estimator.FisherEstimator([weights, bias], 0.1, 0.2, layer_collection)
def __call__(self, shape, dtype=None, partition_info=None): if dtype is None: dtype = self.dtype scale = self.scale scale_shape = shape if partition_info is not None: scale_shape = partition_info.full_shape fan_in, fan_out = _compute_fans(scale_shape) if self.mode == "fan_in": scale /= max(1., fan_in) elif self.mode == "fan_out": scale /= max(1., fan_out) else: scale /= max(1., (fan_in + fan_out) / 2.) if self.distribution == "normal" or self.distribution == "truncated_normal": # constant taken from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.) stddev = math.sqrt(scale) / .87962566103423978 return random_ops.truncated_normal(shape, 0.0, stddev, dtype, seed=self.seed) elif self.distribution == "untruncated_normal": stddev = math.sqrt(scale) return random_ops.random_normal(shape, 0.0, stddev, dtype, seed=self.seed) else: limit = math.sqrt(3.0 * scale) return random_ops.random_uniform(shape, -limit, limit, dtype, seed=self.seed)
def __call__(self, shape, dtype=None, partition_info=None): if dtype is None: dtype = self.dtype # Check the shape if len(shape) < 3 or len(shape) > 5: raise ValueError( "The tensor to initialize must be at least three-dimensional and at most five-dimensional" ) if shape[-2] > shape[-1]: raise ValueError("In_filters cannot be greater than out_filters.") # Generate a random matrix a = random_ops.random_normal([shape[-1], shape[-1]], dtype=dtype, seed=self.seed) # Compute the qr factorization q, r = gen_linalg_ops.qr(a, full_matrices=False) # Make Q uniform d = array_ops.diag_part(r) q *= math_ops.sign(d) q = q[:shape[-2], :] q *= math_ops.sqrt(math_ops.cast(self.gain, dtype=dtype)) if len(shape) == 3: weight = array_ops.scatter_nd([[(shape[0] - 1) // 2]], array_ops.expand_dims(q, 0), shape) elif len(shape) == 4: weight = array_ops.scatter_nd([[(shape[0] - 1) // 2, (shape[1] - 1) // 2]], array_ops.expand_dims(q, 0), shape) else: weight = array_ops.scatter_nd([[(shape[0] - 1) // 2, (shape[1] - 1) // 2, (shape[2] - 1) // 2]], array_ops.expand_dims(q, 0), shape) return weight
def testGetConstantOutOfResourceVariableBeforeWrite(self): with ops.device('device:{}:0'.format(self.device)): # Use floats to force device placement. a = variables.Variable(50.0) b = variables.Variable(2.0) @def_function.function(jit_compile=True) def f(x, val1, val2): out = array_ops.reshape(x, [ math_ops.cast(a, dtypes.int32), math_ops.cast(b, dtypes.int32) ]) a.assign(math_ops.cast(val1, dtypes.float32)) b.assign(math_ops.cast(val2, dtypes.float32)) return out val1 = constant_op.constant(2) val2 = constant_op.constant(50) # OK since the write happens after the reshape. out = f(random_ops.random_normal([10, 10]), val1, val2) self.assertEqual(out.shape[0], 50) self.assertEqual(out.shape[1], 2)
def _transform_feature(self, inputs): """Handles cross transformation.""" # Bucketize the source column. if not self.add_random: return bucketization_op.bucketize(inputs.get(self.source_column), boundaries=list(self.boundaries), name="bucketize") else: rawts = inputs.get(self.source_column) tbn = np.asarray(self.boundaries[1:]) if len(tbn) > 30: # noise = min(np.median(tbn)-tbn[0],tbn[20:-20].std())/2. noise = tbn[10:-10].std() / 10. rndts = rawts + random_normal(array_ops.shape(rawts), 0, noise) return bucketization_op.bucketize(rndts, boundaries=list( self.boundaries), name="bucketize") else: return bucketization_op.bucketize(rawts, boundaries=list( self.boundaries), name="bucketize")
def create_fc_per_eg_grad(batch_size, activation_size, num_layers): inp = random_ops.random_normal([batch_size, activation_size]) layers = [ tf_layers.Dense(activation_size, activation=nn.relu) for _ in range(num_layers) ] projection = tf_layers.Dense(1) def model_fn(activation): for layer in layers: activation = layer(activation) activation = projection(activation) activation = nn.l2_loss(activation) return gradient_ops.gradients(activation, variables.trainable_variables()) def loop_fn(i): return model_fn(array_ops.expand_dims(array_ops.gather(inp, i), 0)) pfor_outputs = control_flow_ops.pfor(loop_fn, batch_size) loop_fn_dtypes = [x.dtype for x in variables.trainable_variables()] while_outputs = control_flow_ops.for_loop(loop_fn, loop_fn_dtypes, batch_size) return pfor_outputs, while_outputs
def decayed_lr(learning_rate, global_step, decay_steps, initial_variance, variance_decay, num_periods, alpha, beta, name): """Helper to recompute learning rate; most helpful in eager-mode.""" with ops.name_scope(name, "NoisyLinearCosineDecay", [learning_rate, global_step]) as name: learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") dtype = learning_rate.dtype decay_steps = math_ops.cast(decay_steps, dtype) initial_variance = math_ops.cast(initial_variance, dtype) variance_decay = math_ops.cast(variance_decay, dtype) num_periods = math_ops.cast(num_periods, dtype) alpha = math_ops.cast(alpha, dtype) beta = math_ops.cast(beta, dtype) global_step_recomp = math_ops.cast(global_step, dtype) global_step_recomp = math_ops.minimum(global_step_recomp, decay_steps) linear_decayed = (decay_steps - global_step_recomp) / decay_steps variance = initial_variance / (math_ops.pow( 1.0 + global_step_recomp, variance_decay)) std = math_ops.sqrt(variance) noisy_linear_decayed = ( linear_decayed + random_ops.random_normal(linear_decayed.shape, stddev=std)) completed_fraction = global_step_recomp / decay_steps fraction = 2.0 * num_periods * completed_fraction cosine_decayed = 0.5 * ( 1.0 + math_ops.cos(constant_op.constant(math.pi) * fraction)) noisy_linear_cosine_decayed = ( (alpha + noisy_linear_decayed) * cosine_decayed + beta) return math_ops.multiply(learning_rate, noisy_linear_cosine_decayed, name=name)
def test_move_dimension_dynamic_shape(self): x_ = random_ops.random_normal(shape=[200, 30, 4, 1, 6]) x = array_ops.placeholder_with_default(input=x_, shape=None) x_perm = distribution_util.move_dimension(x, 1, 1) self.assertAllEqual(self.evaluate(array_ops.shape(x_perm)), [200, 30, 4, 1, 6]) x_perm = distribution_util.move_dimension(x, 0, 3) self.assertAllEqual(self.evaluate(array_ops.shape(x_perm)), [30, 4, 1, 200, 6]) x_perm = distribution_util.move_dimension(x, 0, -2) self.assertAllEqual(self.evaluate(array_ops.shape(x_perm)), [30, 4, 1, 200, 6]) x_perm = distribution_util.move_dimension(x, 4, 2) self.assertAllEqual(self.evaluate(array_ops.shape(x_perm)), [200, 30, 6, 4, 1]) x_perm = distribution_util.move_dimension(x, -1, 2) self.assertAllEqual(self.evaluate(array_ops.shape(x_perm)), [200, 30, 6, 4, 1])
def branch_fn(): # Use a random value so the adds can't be constant folded. return x + sum(random_ops.random_normal([]) for _ in range(self.NUM_INTERMEDIATES))
def rng(dtype): return random_ops.random_normal(shape=[2], dtype=dtype)
def computation(): return random_ops.random_normal([10])
def initializer(): if init == "random": return random_ops.random_normal([size, cols]) else: return init[i]
def sample_n(self, n, seed=None, name='sample'): # pylint: disable=line-too-long """Generate `n` samples. Complexity: O(nbk^3) The sampling procedure is based on the [Bartlett decomposition]( https://en.wikipedia.org/wiki/Wishart_distribution#Bartlett_decomposition) and [using a Gamma distribution to generate Chi2 random variates]( https://en.wikipedia.org/wiki/Chi-squared_distribution#Gamma.2C_exponential.2C_and_related_distributions). Args: n: `Scalar` `Tensor` of type `int32` or `int64`, the number of observations to sample. seed: Python integer; random number generator seed. name: The name of this op. Returns: samples: a `Tensor` of shape `(n,) + self.batch_shape + self.event_shape` with values of type `self.dtype`. """ with ops.name_scope(self.name): with ops.name_scope(name, values=[n] + list(self.inputs.values())): n = ops.convert_to_tensor(n, name='n') if n.dtype != dtypes.int32: raise TypeError('n.dtype=%s which is not int32' % n.dtype) batch_shape = self.batch_shape() event_shape = self.event_shape() batch_ndims = array_ops.shape(batch_shape)[0] ndims = batch_ndims + 3 # sample_ndims=1, event_ndims=2 shape = array_ops.concat(0, ((n,), batch_shape, event_shape)) # Complexity: O(nbk^2) x = random_ops.random_normal(shape=shape, mean=0., stddev=1., dtype=self.dtype, seed=seed) # Complexity: O(nbk) # This parametrization is equivalent to Chi2, i.e., # ChiSquared(k) == Gamma(alpha=k/2, beta=1/2) g = random_ops.random_gamma(shape=(n,), alpha=self._multi_gamma_sequence( 0.5 * self.df, self.dimension), beta=0.5, dtype=self.dtype, seed=seed) # Complexity: O(nbk^2) x = array_ops.batch_matrix_band_part(x, -1, 0) # Tri-lower. # Complexity: O(nbk) x = array_ops.batch_matrix_set_diag(x, math_ops.sqrt(g)) # Make batch-op ready. # Complexity: O(nbk^2) perm = array_ops.concat(0, (math_ops.range(1, ndims), (0,))) x = array_ops.transpose(x, perm) shape = array_ops.concat(0, (batch_shape, (event_shape[0], -1))) x = array_ops.reshape(x, shape) # Complexity: O(nbM) where M is the complexity of the operator solving a # vector system. E.g., for OperatorPDDiag, each matmul is O(k^2), so # this complexity is O(nbk^2). For OperatorPDCholesky, each matmul is # O(k^3) so this step has complexity O(nbk^3). x = self.scale_operator_pd.sqrt_matmul(x) # Undo make batch-op ready. # Complexity: O(nbk^2) shape = array_ops.concat(0, (batch_shape, event_shape, (n,))) x = array_ops.reshape(x, shape) perm = array_ops.concat(0, ((ndims-1,), math_ops.range(0, ndims-1))) x = array_ops.transpose(x, perm) if not self.cholesky_input_output_matrices: # Complexity: O(nbk^3) x = math_ops.batch_matmul(x, x, adj_y=True) # Set shape hints. if self.scale_operator_pd.get_shape().ndims is not None: x.set_shape(tensor_shape.TensorShape( [tensor_util.constant_value(n)] + self.scale_operator_pd.get_shape().as_list())) elif x.get_shape().ndims is not None: x.get_shape()[0].merge_with( tensor_shape.TensorDimension(tensor_util.constant_value(n))) return x
def create_callable_acgan_model(): return train.acgan_model(Generator(), ACGANDiscriminator(), real_data=array_ops.zeros([1, 2]), generator_inputs=random_ops.random_normal([1, 2]), one_hot_labels=array_ops.one_hot([0, 1, 2], 10))
def inner(z, shp): return z + random_ops.random_normal(shp)**2, shp
def g(x): return control_flow_ops.while_loop_v2( lambda *_: True, lambda y, shp: (y + random_ops.random_normal(shp)**2, shp), (x, array_ops.shape(x)), maximum_iterations=3)[0]
def f(x, d): if math_ops.reduce_all( math_ops.greater(x, random_ops.random_normal([10, 10]))): return array_ops.reshape(x * 2, constant_op.constant([100])) else: return array_ops.reshape(x * 3, d)
def _initializer(shape, dtype=_assert_float_dtype(dtype), partition_info=None): return random_ops.random_normal(shape, mean, stddev, dtype, seed=seed)
def computation(): return random_ops.random_normal(shape=[1, 2, 3])
def noisy_linear_cosine_decay(learning_rate, global_step, decay_steps, initial_variance=1.0, variance_decay=0.55, num_periods=0.5, alpha=0.0, beta=0.001, name=None): """Applies noisy linear cosine decay to the learning rate. See [Bello et al., ICML2017] Neural Optimizer Search with RL. https://arxiv.org/abs/1709.07417 For the idea of warm starts here controlled by `num_periods`, see [Loshchilov & Hutter, ICLR2016] SGDR: Stochastic Gradient Descent with Warm Restarts. https://arxiv.org/abs/1608.03983 Note that linear cosine decay is more aggressive than cosine decay and larger initial learning rates can typically be used. When training a model, it is often recommended to lower the learning rate as the training progresses. This function applies a noisy linear cosine decay function to a provided initial learning rate. It requires a `global_step` value to compute the decayed learning rate. You can just pass a TensorFlow variable that you increment at each training step. The function returns the decayed learning rate. It is computed as: ```python global_step = min(global_step, decay_steps) linear_decay = (decay_steps - global_step) / decay_steps) cosine_decay = 0.5 * ( 1 + cos(pi * 2 * num_periods * global_step / decay_steps)) decayed = (alpha + linear_decay + eps_t) * cosine_decay + beta decayed_learning_rate = learning_rate * decayed ``` where eps_t is 0-centered gaussian noise with variance initial_variance / (1 + global_step) ** variance_decay Example usage: ```python decay_steps = 1000 lr_decayed = noisy_linear_cosine_decay( learning_rate, global_step, decay_steps) ``` Args: learning_rate: A scalar `float32` or `float64` Tensor or a Python number. The initial learning rate. global_step: A scalar `int32` or `int64` `Tensor` or a Python number. Global step to use for the decay computation. decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number. Number of steps to decay over. initial_variance: initial variance for the noise. See computation above. variance_decay: decay for the noise's variance. See computation above. num_periods: Number of periods in the cosine part of the decay. See computation above. alpha: See computation above. beta: See computation above. name: String. Optional name of the operation. Defaults to 'NoisyLinearCosineDecay'. Returns: A scalar `Tensor` of the same type as `learning_rate`. The decayed learning rate. Raises: ValueError: if `global_step` is not supplied. """ if global_step is None: raise ValueError("noisy linear cosine decay requires global_step") with ops.name_scope(name, "NoisyLinearCosineDecay", [learning_rate, global_step]) as name: learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") dtype = learning_rate.dtype global_step = math_ops.cast(global_step, dtype) decay_steps = math_ops.cast(decay_steps, dtype) global_step = math_ops.minimum(global_step, decay_steps) initial_variance = math_ops.cast(initial_variance, dtype) variance_decay = math_ops.cast(variance_decay, dtype) num_periods = math_ops.cast(num_periods, dtype) alpha = math_ops.cast(alpha, dtype) beta = math_ops.cast(beta, dtype) linear_decayed = (decay_steps - global_step) / decay_steps variance = initial_variance / (math_ops.pow(1.0 + global_step, variance_decay)) std = math_ops.sqrt(variance) noisy_linear_decayed = ( linear_decayed + random_ops.random_normal(linear_decayed.shape, stddev=std)) completed_fraction = global_step / decay_steps fraction = 2.0 * num_periods * completed_fraction cosine_decayed = 0.5 * ( 1.0 + math_ops.cos(constant_op.constant(math.pi) * fraction)) noisy_linear_cosine_decayed = ( (alpha + noisy_linear_decayed) * cosine_decayed + beta) return math_ops.multiply(learning_rate, noisy_linear_cosine_decayed, name=name)
def _sample_n(self, n, seed=None): shape = array_ops.concat([[n], self.batch_shape_tensor()], 0) sampled = random_ops.random_normal( shape=shape, mean=0., stddev=1., dtype=self.loc.dtype, seed=seed) return sampled * self.scale + self.loc
def __call__(self, shape, dtype=None, partition_info=None): if dtype is None: dtype = self.dtype return random_ops.random_normal( shape, self.mean, self.stddev, dtype, seed=self.seed)
def create_callable_gan_model(): return train.gan_model(Generator(), Discriminator(), real_data=array_ops.zeros([1, 2]), generator_inputs=random_ops.random_normal([1, 2]))
def _sample_n(self, n, seed=None): shape = array_ops.concat_v2(([n], array_ops.shape(self.mean())), 0) sampled = random_ops.random_normal( shape=shape, mean=0, stddev=1, dtype=self.mu.dtype, seed=seed) return sampled * self.sigma + self.mu
def test_basic(self): """A basic test that the op runs on random input.""" with spectral_ops_test_util.fft_kernel_label_map(): with self.test_session(use_gpu=True): signal = random_ops.random_normal((2, 3, 5)) mfcc_ops.mfccs_from_log_mel_spectrograms(signal).eval()
def random_normal(mu, sigma, dims, name=None): mu = ops.convert_to_tensor(mu) return random_ops.random_normal( dims, mean=mu, stddev=sigma, dtype=mu.dtype, name=name)
def testCreluShape(self): f = random_ops.random_normal([50, 5, 7, 10]) t = nn_ops.crelu(f) self.assertEqual([50, 5, 7, 20], t.get_shape())
def fully_connected_model_fn(batch_size, activation_size, num_layers): model = FullyConnectedModel(activation_size, num_layers) inp = random_ops.random_normal([batch_size, activation_size]) return inp, model(inp)