def testReduceStd(self):
x = np.array([[0, 0, 0], [0, 0, 0]], "float32")
self.assertAllClose(self.evaluate(math_ops.reduce_std(x)), 0)
self.assertAllClose(
self.evaluate(math_ops.reduce_std(x, axis=0)), [0, 0, 0])
x = np.array([[1, 2, 1, 1], [1, 1, 0, 1]], "float32")
self.assertAllClose(self.evaluate(math_ops.reduce_std(x)), 0.5)
def testReduceStd(self):
x = np.array([[0, 0, 0], [0, 0, 0]], "float32")
self.assertAllClose(self.evaluate(math_ops.reduce_std(x)), 0)
self.assertAllClose(self.evaluate(math_ops.reduce_std(x, axis=0)),
[0, 0, 0])
x = np.array([[1, 2, 1, 1], [1, 1, 0, 1]], "float32")
self.assertAllClose(self.evaluate(math_ops.reduce_std(x)), 0.5)
def _gradient_normalization(self,
grad,
non_zero,
centralize_gradients=True,
normalize_gradients=True):
"""
substract the mean from the gradient and divide it by its standard deviation
`non_zero` is a function that takes an input and insures that it will not be zero or negative
"""
ndim = grad.shape.ndims
can_centralize = centralize_gradients and (ndim > 1)
size = 1
for i in range(ndim):
size *= grad.shape.dims[i].value # np.prod(grad.shape.dims)
can_normalize = normalize_gradients and (size > 2)
if can_centralize or can_normalize:
# takes into account the fact that the gradient might be 1D
keepdims = (ndim > 1)
axis = list(range(0, ndim - 1)) if keepdims else None
# substract the mean from the gradient
grad_mean = math_ops.reduce_mean(grad,
axis=axis,
keepdims=keepdims)
grad -= grad_mean
if can_normalize:
# divide the centralized gradient by its standard deviation
grad_std = math_ops.reduce_std(grad,
axis=axis,
keepdims=keepdims)
grad /= non_zero(
grad_std) # we divide *after* subtracting the mean
# add the mean back to the gradient if we don't want to centralize it
if not can_centralize:
grad += grad_mean
return grad
def call(self, y_true, y_pred):
y_pred = ops.convert_to_tensor_v2(y_pred)
y_true = math_ops.cast(y_true, y_pred.dtype)
y_pred = math_ops.round(y_pred)
y_true = math_ops.round(y_true)
y_pred_mean = math_ops.reduce_mean(y_pred, keepdims=True)
y_true_mean = math_ops.reduce_mean(y_true, keepdims=True)
pred_std = math_ops.reduce_std(y_pred, keepdims=True)
true_std = math_ops.reduce_std(y_true, keepdims=True)
pearson = self.pcc(y_true, y_pred)
ccc_n = (2.0 * pearson * pred_std * true_std)
ccc_d = (math_ops.square(pred_std) + math_ops.square(true_std) +
math_ops.square(y_pred_mean - y_true_mean))
ccc = ccc_n / (ccc_d + 1e-25)
return ccc
def testReduceStdComplex(self):
# Ensure that complex values are handled to be consistent with numpy
complex_ys = [([0 - 1j, 0 + 1j], dtypes.float64),
(np.array([0 - 1j, 0 + 1j], "complex64"), dtypes.float32),
(np.array([0 - 1j, 0 + 1j], "complex128"), dtypes.float64)]
for y, dtype in complex_ys:
y_result = math_ops.reduce_std(y)
self.assertEqual(np.std(y), 1.0)
self.assertEqual(self.evaluate(y_result), 1.0)
self.assertEqual(y_result.dtype, dtype)
def update_state(self, y_true, y_pred, sample_weight=None):
y_pred = ops.convert_to_tensor_v2(y_pred)
y_true = math_ops.cast(y_true, y_pred.dtype)
y_pred = math_ops.round(y_pred)
y_true = math_ops.round(y_true)
y_pred_mean = math_ops.reduce_mean(y_pred, keepdims=True)
y_true_mean = math_ops.reduce_mean(y_true, keepdims=True)
pred_std = math_ops.reduce_std(y_pred, keepdims=True)
true_std = math_ops.reduce_std(y_true, keepdims=True)
pearson = self.pcc(y_true, y_pred)
ccc_n = (2.0 * pearson * pred_std * true_std)
ccc_d = (math_ops.square(pred_std) + math_ops.square(true_std) + math_ops.square(
y_pred_mean - y_true_mean))
ccc = ccc_n / (ccc_d+ 1e-25)
self.ccc_r.assign_add(tf.reduce_sum(ccc))
self.total_count.assign_add(len(ccc))
def get_grow_tensor(self, weights, method):
"""Different ways to initialize new connections.
Args:
weights: tf.Tensor or Variable.
method: str, available options: 'zeros', 'random_normal', 'random_uniform'
and 'initial_value'
Returns:
tf.Tensor same shape and type as weights.
Raises:
ValueError, when the method is not valid.
"""
if not isinstance(method, six.string_types):
raise ValueError('Grow-Init: %s is not a string' % method)
if method == 'zeros':
grow_tensor = array_ops.zeros_like(weights, dtype=weights.dtype)
elif method.startswith('initial_dist'):
original_shape = weights.initial_value.shape
divisor = extract_number(method)
grow_tensor = array_ops.reshape(
random_ops.random_shuffle(
array_ops.reshape(weights.initial_value, [-1])),
original_shape) / divisor
elif method.startswith('random_normal'):
stddev = math_ops.reduce_std(weights)
divisor = extract_number(method)
grow_tensor = self._random_normal(
weights.shape,
stddev=stddev,
dtype=weights.dtype,
seed=hash(weights.name + 'grow_init_n')) / divisor
elif method.startswith('random_uniform'):
mean = math_ops.reduce_mean(math_ops.abs(weights))
divisor = extract_number(method)
grow_tensor = self._random_uniform(
weights.shape,
minval=-mean,
maxval=mean,
dtype=weights.dtype,
seed=hash(weights.name + 'grow_init_u')) / divisor
else:
raise ValueError('Grow-Init: %s is not a valid option.' % method)
return grow_tensor
def testReduceStd(self):
x = np.array([[0, 0, 0], [0, 0, 0]], "float32")
self.assertAllClose(self.evaluate(math_ops.reduce_std(x)), 0)
self.assertAllClose(
self.evaluate(math_ops.reduce_std(x, axis=0)), [0, 0, 0])
x = [[1, 2, 1, 1], [1, 1, 0, 1]]
with self.assertRaisesRegexp(TypeError, "must be either real or complex"):
math_ops.reduce_std(x)
x = [[1., 2., 1., 1.], [1., 1., 0., 1.]]
self.assertEqual(self.evaluate(math_ops.reduce_std(x)), 0.5)
x_np = np.array(x)
self.assertEqual(np.std(x_np), 0.5)
self.assertEqual(self.evaluate(math_ops.reduce_std(x_np)), 0.5)
def test_embedding_lookup_sparse_with_initializer(self):
id = 0
embed_dim = 8
elements_num = 262144
for initializer, target_mean, target_stddev in [
(init_ops.random_normal_initializer(0.0, 0.001), 0.0, 0.001),
(init_ops.truncated_normal_initializer(0.0, 0.001), 0.0, 0.00088),
(keras_init_ops.RandomNormalV2(mean=0.0,
stddev=0.001), 0.0, 0.001),
]:
with self.session(config=default_config,
use_gpu=test_util.is_gpu_available()):
id += 1
embedding_weights = de.get_variable(
"emb-init-bugfix-" + str(id),
key_dtype=dtypes.int64,
value_dtype=dtypes.float32,
devices=_get_devices() * 3,
initializer=initializer,
dim=embed_dim,
)
ids = np.random.randint(
-0x7FFFFFFFFFFFFFFF,
0x7FFFFFFFFFFFFFFF,
elements_num,
dtype=np.int64,
)
ids = np.unique(ids)
ids = constant_op.constant(ids, dtypes.int64)
vals_op = de.embedding_lookup(embedding_weights, ids,
None).eval()
mean = self.evaluate(math_ops.reduce_mean(vals_op))
stddev = self.evaluate(math_ops.reduce_std(vals_op))
rtol = 2e-5
atol = rtol
self.assertTrue(not (list(vals_op[0]) == list(vals_op[1])))
self.assertAllClose(target_mean, mean, rtol, atol)
self.assertAllClose(target_stddev, stddev, rtol, atol)
def test_variable_initializer(self):
id = 0
for initializer, target_mean, target_stddev in [
(-1.0, -1.0, 0.0),
(init_ops.random_normal_initializer(0.0, 0.01, seed=2), 0.0, 0.01),
]:
with self.session(config=default_config,
use_gpu=test_util.is_gpu_available()):
id += 1
keys = constant_op.constant(list(range(2**17)), dtypes.int64)
table = de.get_variable(
"t1" + str(id),
key_dtype=dtypes.int64,
value_dtype=dtypes.float32,
initializer=initializer,
dim=10,
)
vals_op = table.lookup(keys)
mean = self.evaluate(math_ops.reduce_mean(vals_op))
stddev = self.evaluate(math_ops.reduce_std(vals_op))
rtol = 2e-5
atol = rtol
self.assertAllClose(target_mean, mean, rtol, atol)
self.assertAllClose(target_stddev, stddev, rtol, atol)
def test_safe_embedding_lookup_sparse_with_initializer(self):
id = 0
embed_dim = 8
dense_shape = np.array([64, 128, 32])
total_space = 64 * 128 * 32
elements_num = int(total_space * 0.50)
for initializer, target_mean, target_stddev in [
(init_ops.random_normal_initializer(0.0, 0.001), 0.0, 0.00029),
(init_ops.truncated_normal_initializer(0.0, 0.001), 0.0, 0.00029),
(keras_init_ops.RandomNormalV2(mean=0.0, stddev=0.001), 0.0, 0.00029),
]:
with self.session(config=default_config,
use_gpu=test_util.is_gpu_available()):
id += 1
embedding_weights = de.get_variable(
"safe-init-bugfix-" + str(id),
key_dtype=dtypes.int64,
value_dtype=dtypes.float32,
devices=_get_devices() * 3,
initializer=initializer,
dim=embed_dim,
)
indices_1d = np.random.randint(0, total_space, elements_num)
indices_1d = np.unique(indices_1d)
indices_1d.sort()
indices_3d = []
for _i in range(indices_1d.size):
a_indice = []
quotient = int(indices_1d[_i] / (128 * 32))
remainder = indices_1d[_i] % (128 * 32)
a_indice.append(quotient)
quotient = int(remainder / 32)
remainder = remainder % 32
a_indice.extend([quotient, remainder])
indices_3d.extend([a_indice])
indices_3d = np.array(indices_3d)
ids = np.random.randint(
-0x7FFFFFFFFFFFFFFF,
0x7FFFFFFFFFFFFFFF,
indices_1d.size,
dtype=np.int64,
)
sparse_ids = sparse_tensor.SparseTensor(
constant_op.constant(indices_3d, dtypes.int64),
constant_op.constant(ids, dtypes.int64),
constant_op.constant(dense_shape, dtypes.int64),
)
vals_op = de.safe_embedding_lookup_sparse(embedding_weights,
sparse_ids,
None,
combiner="mean").eval()
mean = self.evaluate(math_ops.reduce_mean(vals_op))
stddev = self.evaluate(math_ops.reduce_std(vals_op))
rtol = 2e-4
atol = rtol
self.assertTrue(not (vals_op[0][0][0] == vals_op[0][0][1]))
self.assertAllClose(target_mean, mean, rtol, atol)
self.assertAllClose(target_stddev, stddev, rtol, atol)
