def testWrongDimensions(self):
# The input to self_adjoint_eig should be a tensor of
# at least rank 2.
scalar = tf.constant(1.)
with self.assertRaises(ValueError):
tf.self_adjoint_eig(scalar)
vector = tf.constant([1., 2.])
with self.assertRaises(ValueError):
tf.self_adjoint_eig(vector)
def Test(self):
np.random.seed(1)
n = shape_[-1]
batch_shape = shape_[:-2]
a = np.random.uniform(
low=-1.0, high=1.0, size=n * n).reshape([n, n]).astype(dtype_)
a += a.T
a = np.tile(a, batch_shape + (1, 1))
# Optimal stepsize for central difference is O(epsilon^{1/3}).
epsilon = np.finfo(dtype_).eps
delta = 0.1 * epsilon**(1.0 / 3.0)
# tolerance obtained by looking at actual differences using
# np.linalg.norm(theoretical-numerical, np.inf) on -mavx build
if dtype_ == np.float32:
tol = 1e-2
else:
tol = 1e-7
with self.test_session():
tf_a = tf.constant(a)
tf_e, tf_v = tf.self_adjoint_eig(tf_a)
for b in tf_e, tf_v:
x_init = np.random.uniform(
low=-1.0, high=1.0, size=n * n).reshape([n, n]).astype(dtype_)
x_init += x_init.T
x_init = np.tile(x_init, batch_shape + (1, 1))
theoretical, numerical = tf.test.compute_gradient(
tf_a,
tf_a.get_shape().as_list(),
b,
b.get_shape().as_list(),
x_init_value=x_init,
delta=delta)
self.assertAllClose(theoretical, numerical, atol=tol, rtol=tol)
def Test(self):
np.random.seed(1)
n = shape_[-1]
batch_shape = shape_[:-2]
a = np.random.uniform(
low=-1.0, high=1.0, size=n * n).reshape([n, n]).astype(dtype_)
a += a.T
a = np.tile(a, batch_shape + (1, 1))
if dtype_ == np.float32:
atol = 1e-4
else:
atol = 1e-12
for compute_v in False, True:
np_e, np_v = np.linalg.eig(a)
with self.test_session():
if compute_v:
tf_e, tf_v = tf.self_adjoint_eig(tf.constant(a))
# Check that V*diag(E)*V^T is close to A.
a_ev = tf.batch_matmul(
tf.batch_matmul(tf_v, tf.batch_matrix_diag(tf_e)),
tf_v,
adj_y=True)
self.assertAllClose(a_ev.eval(), a, atol=atol)
# Compare to numpy.linalg.eig.
CompareEigenDecompositions(self, np_e, np_v, tf_e.eval(), tf_v.eval(),
atol)
else:
tf_e = tf.self_adjoint_eigvals(tf.constant(a))
self.assertAllClose(
np.sort(np_e, -1), np.sort(tf_e.eval(), -1), atol=atol)
def r_loss(communities = 2, group_size = 10, seed=None, p=0.4, q=0.05, r=1.0, projection_dim=2):
"""testing to see if the loss will decrease backproping through very simple function"""
B = np.asarray(balanced_stochastic_blockmodel(communities, group_size, p, q, seed)).astype(np.double)
B = tf.cast(B, tf.float64)
Diag = tf.diag(tf.reduce_sum(B,0))
Diag = tf.cast(Diag, tf.float64)
#r_grid = tf.linspace(r_min, r_max, grid_size)
r = tf.cast(r, tf.float64)
BH = (tf.square(r)-1)*tf.diag(tf.ones(shape=[communities*group_size], dtype=tf.float64))-tf.mul(r, B)+Diag
with tf.Session() as sess:
eigenval, eigenvec = tf.self_adjoint_eig(BH)
eigenvec_proj = tf.slice(eigenvec, [0,0], [communities*group_size, projection_dim])
true_assignment_a = tf.concat(0, [-1*tf.ones([group_size], dtype=tf.float64),
tf.ones([group_size], dtype=tf.float64)])
true_assignment_b = -1*true_assignment_a
true_assignment_a = tf.expand_dims(true_assignment_a, 1)
true_assignment_b = tf.expand_dims(true_assignment_b, 1)
projected_a = tf.matmul(tf.matmul(eigenvec_proj, tf.transpose(eigenvec_proj)), true_assignment_a)#tf.transpose(true_assignment_a))
projected_b = tf.matmul(tf.matmul(eigenvec_proj, tf.transpose(eigenvec_proj)), true_assignment_b)#tf.transpose(true_assignment_b))
loss = tf.minimum(tf.reduce_sum(tf.square(tf.sub(projected_a, true_assignment_a))),
tf.reduce_sum(tf.square(tf.sub(projected_b, true_assignment_b))))
d = sess.run(loss)
return d
def test_SelfAdjointEig(self):
t = tf.self_adjoint_eig(np.array([3,2,1, 2,4,5, 1,5,6]).reshape(3, 3).astype("float32"))
# the order of eigen vectors and values may differ between tf and np, so only compare sum
# and mean
# also, different numerical algorithms are used, so account for difference in precision by
# comparing numbers with 4 digits
self.check(t, ndigits=4, stats=True, abs=True)
def _psd_mask(x):
"""Computes whether each square matrix in the input is positive semi-definite.
Args:
x: A floating-point `Tensor` of shape `[B1, ..., Bn, M, M]`.
Returns:
mask: A floating-point `Tensor` of shape `[B1, ... Bn]`. Each
scalar is 1 if the corresponding matrix was PSD, otherwise 0.
"""
# Allegedly
# https://scicomp.stackexchange.com/questions/12979/testing-if-a-matrix-is-positive-semi-definite
# it is more efficient to test for positive semi-definiteness by
# trying to compute the Cholesky decomposition -- the matrix is PSD
# if you succeed and not PSD if you fail. However, TensorFlow's
# Cholesky raises an exception if _any_ of the input matrices are
# not PSD, from which I don't know how to extract _which ones_, so I
# proceed by explicitly computing all the eigenvalues and checking
# whether they are all positive or not.
#
# Also, as was discussed in the answer, it is somewhat dangerous to
# treat SPD-ness as binary in floating-point arithmetic. Cholesky
# factorization can complete and 'look' like everything is fine
# (e.g., O(1) entries and a diagonal of all ones) but the matrix can
# have an exponential condition number.
eigenvalues, _ = tf.self_adjoint_eig(x)
return tf.cast(
tf.reduce_min(eigenvalues, axis=-1) >= 0, dtype=x.dtype)
def target_subspace(adj, groupsize, communities, diag, dim_proj):
normalizer = tf.cast(2.0*groupsize*communities, dtype=tf.float64)
total_degree = tf.cast(tf.reduce_sum(adj), dtype=tf.float64)
r = tf.sqrt(total_degree/normalizer)
BH_op = (tf.square(r)-1)*tf.diag(tf.ones(shape=[communities*groupsize], dtype=tf.float64))-r*adj+diag
val, vec = tf.self_adjoint_eig(BH_op) #this is already normalized so no need to normalize
subspace = tf.slice(vec, [0,0], [communities*groupsize, dim_proj])
return r, subspace
def _compareSelfAdjointEig(self, x, use_gpu=False):
with self.test_session() as sess:
tf_eig = tf.self_adjoint_eig(tf.constant(x))
tf_eig_out = sess.run([tf_eig])[0]
d, _ = x.shape
self.assertEqual([d+1, d], tf_eig.get_shape().dims)
self._testEigs(x, d, tf_eig_out, use_gpu)
def matrix_log(self):
eigenvalues, eigenvectors = tf.self_adjoint_eig(self.reduced_matrix1)
eigenvalues_log = tf.log(
tf.clip_by_value(eigenvalues, 1e-8, tf.reduce_max(eigenvalues)))
diag_eigenvalues_log = tf.matrix_diag(eigenvalues_log)
eigenvectors_inverse = tf.matrix_inverse(eigenvectors)
return tf.matmul(tf.matmul(eigenvectors, diag_eigenvalues_log),
eigenvectors_inverse)
def inverse_Hessian(self, H, thr=0.001):
e, v = tf.self_adjoint_eig(H)
thr_condition = tf.greater(tf.abs(e), thr)
sign_ = tf.cast(tf.greater(e, 0.0), tf.float32)
e_inv = tf.where(
thr_condition, tf.divide(1.0, e),
tf.multiply(tf.multiply(tf.ones_like(e), 100.0), sign_))
H_inv = tf.matmul(tf.matmul(v, tf.diag(e_inv)), tf.transpose(v))
return H_inv, e
def build_network(self,pointclouds_pl,is_training,is_eveluate,bn_decay = None):
with_bn = self.conf.get_bool('with_bn')
batch_size = pointclouds_pl.get_shape()[0].value
num_point = pointclouds_pl.get_shape()[1].value
if (self.conf['with_rotations']):
cov = self.tf_cov(pointclouds_pl)
_, axis = tf.self_adjoint_eig(cov)
axis = tf.where(tf.linalg.det(axis) < 0, tf.matmul(axis, tf.tile(
tf.constant([[[0, 1], [1, 0]]], dtype=tf.float32), multiples=[axis.get_shape()[0], 1, 1])), axis)
indicies = [[[b, 0, 0], [b, 2, 0], [b, 0, 2], [b, 2, 2]] for b in list(range(batch_size))]
updates = tf.reshape(axis, [batch_size, -1])
updates = tf.reshape(tf.matmul(
tf.tile(tf.constant([[[0, 1, 0, 0], [0, 0, 0, 1], [1, 0, 0, 0], [0, 0, 1, 0]]], dtype=tf.float32),
multiples=[batch_size, 1, 1]), tf.expand_dims(updates, axis=-1)), shape=[batch_size, -1])
alignment_transform = tf.scatter_nd(indices=indicies, updates=updates,
shape=[batch_size, 3, 3]) + tf.expand_dims(
tf.diag([0.0, 1.0, 0.0]), axis=0)
mean_points = tf.reduce_mean(pointclouds_pl, axis=1, keepdims=True)
pointclouds_pl = tf.matmul(pointclouds_pl - mean_points, alignment_transform) + mean_points
ps_function_pl = tf.concat([pointclouds_pl,tf.ones(shape=[batch_size,num_point,1],dtype=tf.float32)],axis=2)
pool_sizes_sigma = self.conf.get_list('pool_sizes_sigma')
spacing = self.conf.get_float('kernel_spacing')
network = ps_function_pl
input_channel = network.get_shape()[2].value
blocks = self.conf.get_list('blocks_out_channels')
for block_index,block in enumerate(blocks):
block_elm = ConvElements(pointclouds_pl, 1. * tf.reciprocal(tf.sqrt(tf.cast(pointclouds_pl.get_shape()[1].value,tf.float32))),spacing,self.conf.get_float('kernel_sigma_factor'))
for out_index,out_channel in enumerate(block):
network = ConvLayer(input_channel, block_elm, out_channel, '{0}_block_{1}'.format(block_index,out_index),is_training).get_layer(network,with_bn,bn_decay,self.conf.get_bool('interpolation'))
input_channel = out_channel
pointclouds_pl, network = PoolingLayer(block_elm, out_channel, out_channel,
int(pool_sizes_sigma[block_index + 1][0])).get_layer(network,is_subsampling=self.conf.get_bool('subsampling'),use_fps= tf.logical_or(is_training,is_eveluate))
network = tf.reshape(network, [batch_size, -1])
network = tf_util.fully_connected(network, self.conf.get_int('fc1.size'), bn=True, is_training=is_training,
scope='fc1', bn_decay=bn_decay)
network = tf_util.dropout(network, keep_prob=self.conf.get_float('dropout.keep_prob'), is_training=is_training,
scope='dp1')
network = tf_util.fully_connected(network, self.conf.get_int('fc2.size'), bn=True, is_training=is_training,
scope='fc2', bn_decay=bn_decay)
network = tf_util.dropout(network, keep_prob=self.conf.get_float('dropout.keep_prob'), is_training=is_training,
scope='dp2')
network = tf_util.fully_connected(network, 40, activation_fn=None, scope='fc3')
return network
def build_KL(self):
"""
The covariance of q(u) has a kronecker structure, so
appropriate reductions apply for the trace and logdet terms.
"""
# Mahalanobis term, m^T K^{-1} m
Kuu = [make_Kuu(kern, a, b, self.ms) for kern, a, b, in zip(self.kerns, self.a, self.b)]
Kim = kron_vec_apply(Kuu, self.q_mu, 'solve')
KL = 0.5*tf.reduce_sum(self.q_mu * Kim)
# Constant term
KL += -0.5*tf.cast(tf.size(self.q_mu), float_type)
# Log det term
Ls = [tf.matrix_band_part(q_sqrt_d, -1, 0) for q_sqrt_d in self.q_sqrt_kron]
N_others = [float(np.prod(self.Ms)) / M for M in self.Ms]
Q_logdets = [tf.reduce_sum(tf.log(tf.square(tf.diag_part(L)))) for L in Ls]
KL += -0.5 * reduce(tf.add, [N*logdet for N, logdet in zip(N_others, Q_logdets)])
# trace term tr(K^{-1} Sigma_q)
Ss = [tf.matmul(L, tf.transpose(L)) for L in Ls]
traces = [K.trace_KiX(S) for K, S, in zip(Kuu, Ss)]
KL += 0.5 * reduce(tf.mul, traces) # kron-trace is the produce of traces
# log det term Kuu
Kuu_logdets = [K.logdet() for K in Kuu]
KL += 0.5 * reduce(tf.add, [N*logdet for N, logdet in zip(N_others, Kuu_logdets)])
if self.use_two_krons:
# extra logdet terms:
Ls_2 = [tf.matrix_band_part(q_sqrt_d, -1, 0) for q_sqrt_d in self.q_sqrt_kron_2]
LiL = [tf.matrix_triangular_solve(L1, L2) for L1, L2 in zip(Ls, Ls_2)]
eigvals = [tf.self_adjoint_eig(tf.matmul(tf.transpose(mat), mat))[0] for mat in LiL] # discard eigenvectors
eigvals_kronned = kron([tf.reshape(e, [1, -1]) for e in eigvals])
KL += -0.5 * tf.reduce_sum(tf.log(1 + eigvals_kronned))
# extra trace terms
Ss = [tf.matmul(L, tf.transpose(L)) for L in Ls_2]
traces = [K.trace_KiX(S) for K, S, in zip(Kuu, Ss)]
KL += 0.5 * reduce(tf.mul, traces) # kron-trace is the produce of traces
elif self.use_extra_ranks:
# extra logdet terms
KiW = kron_mat_apply(Kuu, self.q_sqrt_W, 'solve', self.use_extra_ranks)
WTKiW = tf.matmul(tf.transpose(self.q_sqrt_W), KiW)
L_extra = tf.cholesky(np.eye(self.use_extra_ranks) + WTKiW)
KL += -0.5 * tf.reduce_sum(tf.log(tf.square(tf.diag_part(L_extra))))
# extra trace terms
KL += 0.5 * tf.reduce_sum(tf.diag_part(WTKiW))
return KL
def entropy_with_gram(self, norm_gram):
with tf.device('/cpu:0'):
eigvals, _ = tf.self_adjoint_eig(norm_gram)
# Fix possible numerical instabilities:
# Remove small negatives
eigvals = tf.nn.relu(eigvals)
# Ensure eigenvalues sum 1
eigvals = eigvals / tf.reduce_sum(eigvals)
# Compute entropy in the specified base
sum_term = tf.reduce_sum(eigvals**self.alpha)
entropy = tf.log(sum_term) / (1.0 - self.alpha)
entropy = entropy / np.log(self.log_base)
return entropy
def fg_mf_gamma(param1, param2):
mean1, var = param1[0], param1[1]
mean2, alpha, beta = param2[0], param2[1], param2[2]
ab_ratio = alpha / beta
n, p = tf.to_float(tf.shape(mean1)[0]), tf.to_float(tf.shape(mean1)[1])
e, _ = tf.self_adjoint_eig(var)
logdet_var = tf.reduce_sum(tf.log(e), [-1])
kl = -n * p * (tf.digamma(alpha) - tf.log(beta + 1e-10)) - logdet_var
kl = kl - n * p + ab_ratio * tf.trace(var)
kl = kl + ab_ratio * tf.reduce_sum(tf.square(mean2 - mean1), [-1, -2])
return kl * 0.5
def log_coral_loss(source_output, target_output, percent_lambda=0.5):
"""[Calculate LogCoral loss]
Args:
source_output ([tf tensor]): [Source feature map tensor]
target_output ([tf tensor]): [Target feature map tensor]
percent_lambda (weighting factor, optional): [Log CORAL loss weighting factor]. Defaults to 0.5.
Returns:
[tf tensor]: [Log CORAL loss per batch]
"""
# regularized covariances result in inf or nan
# First: subtract the mean from the data matrix
h_src = source_output
h_trg = target_output
batch_size = tf.cast(tf.shape(h_src)[0], tf.float32)
h_src = h_src - tf.reduce_mean(h_src, axis=0)
h_trg = h_trg - tf.reduce_mean(h_trg, axis=0)
cov_source = (1.0 / (batch_size - 1)) * tf.matmul(
h_src, h_src,
transpose_a=True) # + gamma * tf.eye(self.hidden_repr_size)
cov_target = (1.0 /
(batch_size - 1)) * tf.matmul(h_trg, h_trg, transpose_a=True)
eig_source = tf.self_adjoint_eig(cov_source)
eig_target = tf.self_adjoint_eig(cov_target)
log_cov_source = tf.matmul(
eig_source[1],
tf.matmul(tf.diag(tf.log(eig_source[0])),
eig_source[1],
transpose_b=True),
)
log_cov_target = tf.matmul(
eig_target[1],
tf.matmul(tf.diag(tf.log(eig_target[0])),
eig_target[1],
transpose_b=True),
)
return percent_lambda * tf.reduce_mean(
tf.square(tf.subtract(log_cov_source, log_cov_target)))
def prior_fn(latent_dimension):
cov_init = util.positive_definate_initializer([10] +
[latent_dimension] * 2)
eigvals = tf.self_adjoint_eig(
tf.divide(cov_init + tf.matrix_transpose(cov_init),
2.,
name='symmetrised'))[0]
cov_init = tf.Print(cov_init, [cov_init])
return parameterized_distributions.gmm.GMM(10,
latent_dimension,
cov_init=cov_init,
trainable=True).model
def test_matrixrelu():
input_shape = (4, 100, 10)
epsilon = 1e-4
data = np.random.randn(*input_shape).astype(K.floatx())
k_data = K.batch_dot(data, data.transpose(0, 2, 1))
sess = K.get_session()
with sess.as_default():
data = K.eval(k_data)
# print(data)
print(data.shape)
# traditional log
res = np.zeros(shape=(4, 100, 100))
from numpy.linalg import eig
for i in range(data.shape[0]):
tmp_d = data[i, :, :]
s, u = eig(tmp_d)
comp = np.zeros_like(s) + epsilon
s[np.where(s < epsilon)] = epsilon
s = np.diag(s)
tmp_res = np.matmul(u, np.matmul(s, u.transpose()))
res[i, :, :] = tmp_res
import tensorflow as tf
tf_input = tf.placeholder(K.floatx(), (4, 100, 100))
tf_s, tf_u = tf.self_adjoint_eig(tf_input)
comp = tf.zeros_like(tf_s) + epsilon
comp = tf.Print(comp, [comp], message='comp:')
tf_s = tf.Print(tf_s, [tf_s], message='tf_s:', summarize=400)
inner = tf.where(tf.less(tf_s, comp), comp, tf_s)
inner = tf.Print(inner, [inner], 'inner:')
inner = tf.matrix_diag(inner)
tf_relu = tf.matmul(tf_u, tf.matmul(inner, tf.transpose(tf_u, [0, 2, 1])))
with sess.as_default():
tf_result = tf_relu.eval({tf_input: data})
# log layer
a = layer_test(MatrixReLU,
input_data=data,
kwargs={
'epsilon': epsilon,
},
input_shape=(4, 100, 100),
input_dtype=K.floatx())
# a = tf_result
# print(a - res)
diff = np.linalg.norm(a - res)
print(diff)
assert_allclose(a, res, rtol=1e-4)
def __init__(self, phys_d, bond_d, A_matrices=None, symmetrize=True, hamiltonian=None, r_prec=1e-14):
"""
:param phys_d: Physical dimension of the state e.g. 2 for spin-1/2 systems.
:param bond_d: Bond dimension, the size of the A matrices.
:param A_matrices: Square matrices of size `bond_d` forming the Matrix Product State.
:param symmetrize: Boolean indicating A matrices are symmetrized.
:param hamiltonian: Tensor of shape [phys_d, phys_d, phys_d, phys_d] giving two site Hamiltonian
"""
self._session = tf.Session()
self.r_prec = r_prec
self.phys_d = phys_d
self.bond_d = bond_d
self.hamiltonian = hamiltonian
self.mps_manifold = pymanopt.manifolds.Stiefel(phys_d * bond_d, bond_d)
# Define the A
if A_matrices is None:
A_init = tf.reshape(self.mps_manifold.rand(), [phys_d, bond_d, bond_d])
else:
A_init = A_matrices
# Create Stiefel from the A
Stiefel_init = tf.reshape(A_init, [self.phys_d * self.bond_d, self.bond_d])
# Define the variational tensor variable Stiefel, and from there the A
self.Stiefel = tf.get_variable("Stiefel_matrix", initializer=Stiefel_init, trainable=True, dtype=tf.float64)
self.A = tf.reshape(self.Stiefel, [self.phys_d, self.bond_d, self.bond_d])
if symmetrize:
self.A = self._symmetrize(self.A)
self._transfer_matrix = None
self._right_eigenvector = None
self._all_eig = tf.self_adjoint_eig(self.transfer_matrix)
self._dominant_eig = None
self._variational_energy = None
if hamiltonian is not None:
if symmetrize:
self.variational_energy = self._add_variational_energy_symmetric_mps(hamiltonian)
else:
self.variational_energy = self._add_variational_energy_left_canonical_mps(hamiltonian)
def call(self, x, mask=None):
"""
Returns
-------
3D tensor with same shape as input
"""
if K.backend() == 'theano' or K.backend() == 'CNTK':
raise NotImplementedError(
"This is not implemented for theano anymore.")
else:
if self.built:
# import tensorflow as tf
# from kyu.tensorflow.ops import safe_truncated_sqrt, safe_sign_sqrt
# with tf.device('/cpu:0'):
# s, u = safe_matrix_eig_op(x)
# # s, u = tf.self_adjoint_eig(x)
# inner = safe_sign_sqrt(s)
# if self.norm == 'l2':
# inner /= tf.reduce_max(inner)
# elif self.norm == 'frob' or self.norm == 'Frob':
# inner /= tf.sqrt(tf.reduce_sum(s))
# elif self.norm is None:
# pass
# else:
# raise ValueError("PowTransform: Normalization not supported {}".format(self.norm))
# # inner = tf.Print(inner, [inner], message='power inner', summarize=65)
# inner = tf.matrix_diag(inner)
# tf_pow = tf.matmul(u, tf.matmul(inner, tf.transpose(u, [0, 2, 1])))
# return tf_pow
import tensorflow as tf
s, u = tf.self_adjoint_eig(x)
comp = tf.zeros_like(s) + self.eps
inner = tf.where(tf.less(s, comp), comp, s)
inner = inner + self.eps
inner = tf.sqrt(inner)
if self.norm == 'l2':
pass
elif self.norm == 'frob' or self.norm == 'Frob':
inner /= tf.norm(s)
# inner = tf.Print(inner, [inner], message='power inner', summarize=65)
inner = tf.matrix_diag(inner)
tf_pow = tf.matmul(
u, tf.matmul(inner, tf.transpose(u, [0, 2, 1])))
return tf_pow
else:
raise RuntimeError(
"PowTransform layer should be built before using")
def tf_smooth_eig_vec(self):
"""Function that returns smoothed version of min eigen vector."""
_, matrix_m = self.dual_object.get_full_psd_matrix()
# Easier to think in terms of max so negating the matrix
[eig_vals, eig_vectors] = tf.self_adjoint_eig(-matrix_m)
exp_eig_vals = tf.exp(tf.divide(eig_vals, self.smooth_placeholder))
scaling_factor = tf.reduce_sum(exp_eig_vals)
# Multiplying each eig vector by exponential of corresponding eig value
# Scaling factor normalizes the vector to be unit norm
eig_vec_smooth = tf.divide(
tf.matmul(eig_vectors, tf.diag(tf.sqrt(exp_eig_vals))),
tf.sqrt(scaling_factor))
return tf.reshape(tf.reduce_sum(eig_vec_smooth, axis=1),
shape=[eig_vec_smooth.shape[0].value, 1])
def _covariance_final_ops(sum_squares, total):
# http://www.johnloomis.org/ece563/notes/covar/covar.html
total = total - tf.constant(1.0)
total_3x3 = tf.reshape(tf.tile(tf.expand_dims(total, 0), [9]), [3, 3])
covariance = tf.div(sum_squares, total_3x3)
variance = tf.gather(tf.reshape(covariance, [-1]), [0, 4, 8])
# eigenvalues and eigenvectors for PCA
eigens = tf.self_adjoint_eig(covariance)
eigenvalues = tf.slice(eigens, [0, 0], [-1, 1])
eigenvectors = tf.slice(eigens, [1, 0], [-1, -1])
return tf.sqrt(variance), eigenvalues, eigenvectors
def update(self, block):
input_factor = block._input_factor
output_factor = block._output_factor
pi = _compute_pi_tracenorm(input_factor.get_cov(),
output_factor.get_cov())
coeff = self._coeff / block._renorm_coeff
coeff = coeff**0.5
damping = coeff / (self._eta**0.5)
ue, uv = tf.self_adjoint_eig(input_factor.get_cov() / pi + damping *
tf.eye(self._u_c.shape.as_list()[0]))
ve, vv = tf.self_adjoint_eig(output_factor.get_cov() * pi + damping *
tf.eye(self._v_c.shape.as_list()[0]))
ue = coeff / tf.maximum(ue, damping)
new_uc = uv * ue**0.5
ve = coeff / tf.maximum(ve, damping)
new_vc = vv * ve**0.5
updates_op = [self._u_c.assign(new_uc), self._v_c.assign(new_vc)]
return tf.group(*updates_op)
def _compute_normal_eig(self, pts):
""" Compute normal given per pixel local point clouds by eigen decompisition.
Args:
pts: NHWK3 tensorf of per pixel local point clouds.
Returns:
NHW3 tensor of per pixel normal.
"""
cov = self._compute_covariance(pts)
_, v = tf.self_adjoint_eig(cov)
# v has shape NHW33, we need to take the first slice (index 0) of the 4th dimension (the one behind W)
# which corresponds to the least eigen value
n = v[:, :, :, 0, :]
n = tf.nn.lrn(n, depth_radius=3, bias=1e-4)
return n
def create_loss_svd(f_out,
g_out,
clip_min=np.float32(-10000),
clip_max=np.float32(10000)):
"""
Create the loss function that will be minimized by the fg-net. Many options exist.
The implementation below uses the 1-Schatten norm from the derivation. It might slow.
Inputs: f_out and g_out, which are tensors of the same shape produced by the outut
of the f and g nets. Assumes that they are of the form (#batches, #output).
Outputs: returns objective
"""
# number of samples in batch
nBatch = tf.cast(tf.shape(f_out)[0], tf.float32)
# we clip f to avoid runaway arguments
f_clip = tf.clip_by_value(f_out, clip_min, clip_max)
#f_clip = f_out
# create correlation matrices
corrF = tf.matmul(tf.transpose(f_clip), f_clip) / nBatch
corrFG = tf.matmul(tf.transpose(f_clip), g_out) / nBatch
# Second moment of g
sqG = tf.reduce_sum(tf.reduce_mean(tf.square(g_out), 0))
# compute svd in objective
n = tf.shape(corrF)[0]
#correction term
epsilon = 1e-4
invCorrF = tf.matrix_inverse(corrF + epsilon * tf.eye(n),
adjoint=True) #check
prodGiFG = tf.matmul(tf.matmul(tf.transpose(corrFG), invCorrF), corrFG)
s, v = tf.self_adjoint_eig(prodGiFG)
#s,u,v = tf.svd(prodGiFG)
schatNorm = tf.reduce_sum(tf.sqrt(tf.abs(s)))
# define objective
objective = sqG - 2 * schatNorm #+ tf.trace(corrF)#.3*tf.reduce_sum(tf.square((corrF-tf.eye(n))))
#return objective
return objective, schatNorm
def giff_evals_evecs(S_b, S_w, n_rdims):
evals_b, evecs_b = tf.self_adjoint_eig(S_b)
evalsRest_b = tf.boolean_mask(evals_b, tf.greater(evals_b, eigenThreshold))
evecsRest_b = evecs_b[:, -tf.shape(evalsRest_b)[0]:]
evals_bh = tf.diag(tf.sqrt(evalsRest_b))
S_bh = tf.matmul(tf.matmul(evecsRest_b, evals_bh),
evecsRest_b,
transpose_b=True)
# S_w += tf.eye(tf.shape(S_w)[0], tf.shape(S_w)[1])*1e-4
S_wi = tf.matrix_inverse(S_w)
S_s = tf.matmul(tf.matmul(S_bh, S_wi),
S_bh) + tf.eye(tf.shape(S_b)[0],
tf.shape(S_b)[1]) * 1e-5
evals, evecs = tf.self_adjoint_eig(S_s)
evalsRest_t = tf.boolean_mask(evals, tf.greater(evals, eigenThreshold))
evalsRest_r = evals[-n_rdims:]
evalsRest = tf.cond(tf.greater(tf.shape(evalsRest_t)[0], n_rdims),
lambda: evalsRest_r, lambda: evalsRest_t)
evecsRest = evecs[:, -tf.shape(evalsRest)[0]:]
return evalsRest, evecsRest
def build_PCA():
#Hyperparameter
D = 3
#parameter
x_PCA = tf.placeholder(tf.float32, [None, D], name="x_PCA")
#mean
x_mean_PCA = tf.reduce_mean(x_PCA, 0)
#variance
S_PCA = compute_outer_product(x_PCA - x_mean_PCA) #compute outer product
eigen_val, eigen_vectors = tf.self_adjoint_eig(S_PCA, name="eigenvect")
#u_PCA
max_index = tf.argmax(eigen_val, 0)
u_PCA = tf.gather(eigen_vectors, max_index)
return x_PCA, S_PCA, eigen_val, eigen_vectors, u_PCA
def _make_positive_semidefinite_tf(convariance_mat,n=3):
e,v = tf.self_adjoint_eig(convariance_mat)
min_eig = e[0]
# e,v = tf.self_adjoint_eig(H)
# e_pos = tf.maximum(0.0,e)+1e-6 #make sure positive definite
# e_sqrt = tf.diag(tf.sqrt(e_pos))
# sq_H = tf.matmul(v,tf.matmul(e_sqrt,tf.transpose(v)))
def f1():
print('need to make the covariance matrix SPD, min_eig =',min_eig)
return convariance_mat-10*min_eig * tf.eye(n)
def f2(): return tf.reshape(tf.stack(convariance_mat),[n,n])
convariance_mat = tf.cond(tf.less(min_eig,tf.constant(0.0)), f1, f2)
return convariance_mat
def group_decorrelation(cov_matrix, group_index):
'''
For a given covariance matrix deducted for a vector_length random vector
and a group index like [... i-th, j-th, k-th ...],
return a transformation that the diagonal part is ones
and makes the [... i-th, j-th, k-th ...] no related with each others and random variables no in the list.
---------------------------------
| trans * cov_matrix * trans' |
_________________________________
:param cov_matrix: A tensor, sized [vector_length, vector_length], float
:param group_index: A tensor, sized [num, 1], int
:return: A tensor, sized [vector_length, vector_length], float
'''
vector_length = int(cov_matrix.get_shape().as_list()[0])
num = group_index.get_shape().as_list()[0]
index_proto = []
for i in range(num):
index_proto.append(i)
index_proto = tf.Variable(index_proto)
index_proto = tf.reshape(index_proto, [num, 1])
sparse_index = tf.concat([index_proto, group_index], axis=1)
take_matrix = tf.sparse_to_dense(sparse_index, [num, vector_length], 1.0)
c_inv = tf.matrix_inverse(cov_matrix)
c_sharp = tf.transpose(tf.matmul(tf.matmul(take_matrix, c_inv), tf.transpose(take_matrix)))
_, m = tf.self_adjoint_eig(c_sharp)
cm = tf.matmul(c_sharp, m)
cm_diag = tf.diag_part(cm)
cm_diag = tf.reshape(cm_diag, [num])
m_ = tf.transpose(tf.divide(m, cm_diag))
m_large = tf.matmul(tf.matmul(tf.transpose(take_matrix), m_), take_matrix)
trans = tf.matmul(m_large, c_inv)
one_diag = tf.diag_part(trans)
all_one = tf.ones([vector_length], tf.float32)
inv_diag = tf.subtract(all_one, one_diag)
com_matrix = tf.diag(inv_diag)
trans = tf.add(trans, com_matrix)
return trans
def post_process_and_get_tensor_list(self):
with tf.variable_scope('tear_apart'):
if len(self.return_list) == 0:
if self.built_method == 'MEAN':
print('Get the tensor list of gradients\' means.')
for acls in self.acls_list:
self.return_list.append(acls.variable_avg)
elif self.built_method == 'VAR':
print('Get the tensor list of vv, where vv*vv\' = E(GG\')')
for acls in self.acls_list:
e, v = tf.self_adjoint_eig(acls.variable_avg)
vv = tf.multiply(v, tf.sqrt(tf.maximum(e, 0)))
# vv*vv' = acls.variable_avg
self.return_list.append(tf.transpose(vv))
return self.return_list
def quat_avg(quats, weights=None):
'''
average
:param quats: ... x K x 4
:param dim:
:param weights: x K
:return:
'''
if weights is not None:
cov = tf.matmul(tf.matmul(quats, tf.linalg.diag(weights), transpose_a=True), quats)
else:
cov = tf.matmul(quats, quats, transpose_a=True)
_, v = tf.self_adjoint_eig(cov)
return v[..., -1]
def _build_graph(self, n_cells, time_window, lr, lam_l1, beta):
"""Build tensorflow graph.
Args :
n_cells : number of cells (int)
time_window : number of time bins in each response vector (int)
lr : learning rate (float)
lam_l1 : regularization on entires of A (float)
"""
# declare variables
self.dim = n_cells * time_window
self.lam_l1 = lam_l1
self.beta = beta
self.A_symm = tf.Variable(0.001 * np.eye(self.dim).astype(np.float32),
name='A_symm') # initialize Aij to be zero
# placeholders for anchor, pos and neg
self.anchor = tf.placeholder(dtype=tf.float32,
shape=[None, n_cells, time_window],
name='anchor')
self.pos = tf.placeholder(dtype=tf.float32,
shape=[None, n_cells, time_window],
name='pos')
self.neg = tf.placeholder(dtype=tf.float32,
shape=[None, n_cells, time_window],
name='neg')
self.score_anchor_pos = self.get_score(self.anchor, self.pos)
self.score_anchor_neg = self.get_score_all_pairs(self.anchor, self.neg)
self.get_loss()
self.train_step = tf.train.AdagradOptimizer(lr).minimize(self.loss)
# do projection to A_symm symmetric
project_A_symm = tf.assign(
self.A_symm, (self.A_symm + tf.transpose(self.A_symm)) / 2)
# remove negative eigen vectors.
with tf.control_dependencies([project_A_symm]):
e, v = tf.self_adjoint_eig(self.A_symm)
e_pos = tf.nn.relu(e)
A_symm_new = tf.matmul(tf.matmul(v, tf.diag(e_pos)),
tf.transpose(v))
project_A_PSD = tf.assign(self.A_symm, A_symm_new)
# combine all projection operators into one.
self.project_A = tf.group(project_A_symm, project_A_PSD)
def train_PCA(self):
m, w, h, k = self.trainX.shape
data = np.reshape(self.trainX, (m, w * h * k)).T
matrix = tf.placeholder(tf.float32, shape=data.shape)
mean_var = tf.reduce_mean(matrix, 1)
cov = tf.subtract(matrix, tf.reshape(mean_var, (w * h * k, 1)))
cov_var = tf.divide(tf.matmul(cov, tf.transpose(cov)),
tf.constant(m - 1, dtype=tf.float32))
e, v = tf.self_adjoint_eig(cov_var)
mean, eig_values, eig_vectors = self.sess.run((mean_var, e, v),
feed_dict={matrix: data})
eig_values = np.maximum(eig_values, 0)
a = np.argsort(eig_values).tolist()
a.reverse()
self.project_matrix = eig_vectors[:, a]
self.mu = np.reshape(mean, (w * h * k, 1))
def NUS(W_root, A):
'''
input the matrix A and matrix W_root
A is the one to be compute
W_root is the square-root of W
which will make W positive
'''
S, U = tf.self_adjoint_eig(A)
W = tf.multiply(W_root, W_root)
Sigma = tf.matrix_diag(S)
NUSresult = tf.matmul(U, W) # U * W
NUSresult = tf.matmul(NUSresult, Sigma) # U * W * Sigma
NUSresult = tf.matmul(NUSresult, U,
transpose_b=True) # U * W * Sigma * U.T
return NUSresult
def body(fock_matrix, density, delta, energy):
energy = self.nuclear_repulsion
fock_matrix = self.update_fock_matrix(density)
fock_prime = self.overlap_rsqrt_t @ (
fock_matrix @ self.overlap_rsqrt)
_, c_prime = tf.self_adjoint_eig(fock_prime)
coefficients = self.overlap_rsqrt @ c_prime
energy = energy + 0.5 * tf.reduce_sum(density *
(self.h_core + fock_matrix))
density, old_density = self.update_density(density, coefficients)
density = self.damping_factor * density + (
1 - self.damping_factor) * old_density
delta = self.get_density_change(density, old_density)
return [fock_matrix, density, delta, energy]
def main(_):
env = gym.make('Pong-v0')
observs = [env.reset()]
for _ in range(100):
observs.append(env.step(env.action_space.sample())[0])
observs = np.stack(observs)
observs = observs.astype(np.float32)
observs = tf.image.resize_images(observs, [80, 80])
observs = tf.reduce_mean(observs, 3)
# observs = tf.reduce_mean(observs, 0)
print(observs.shape)
eig_observs = tf.self_adjoint_eig(observs)
print(eig_observs)
def fewer_observations():
'''
Solve using Sherman-Morrison-Woodbury; also, compute
eigendecomposition for use when sampling \theta.
'''
# Feature-wise inner product
ZZt = tf.matmul(Z, Zt)
ZZt_nz = ZZt + sigma2 * tf.eye(tf.shape(Z)[0], dtype=Z.dtype)
if (Y is None): #same as Y = [0,...,0]
mu = tf.zeros([tf.shape(Z)[0], 1], dtype=Z.dtype)
else:
chol = tf.cholesky(ZZt_nz)
A = tf.matmul(Zt, 1 / sigma2 * Y)
beta = tf.cholesky_solve(chol, tf.matmul(Z, A))
mu = A - tf.matmul(Zt, beta)
eigen_vals, eigen_vects = tf.self_adjoint_eig(ZZt_nz)
return mu, (eigen_vals, eigen_vects)
def prop(prev_out, curr_in):
H = self.baseham.tensorflowH(curr_in) # [batch, i, j]
if self.taylorord == 'eig':
e, v = tf.self_adjoint_eig(H)
expe = tf.exp(-1j / self.timesteps * e)
next_out = tf.einsum(
'bij,bj->bi', v,
expe * tf.einsum('bji,bj->bi', tf.conj(v), prev_out))
return next_out
next_out = prev_out
for i in range(self.taylorord, 0, -1):
next_out = prev_out + (-1j / self.timesteps / i) * tf.einsum(
'bij,bj->bi', H, next_out)
if self.normalize:
next_out = next_out / tf.norm(
next_out, 2, axis=-1, keep_dims=True)
return next_out
def FM(A, B, a, n):
'''
input matrix A and matrix B and scalar a(lpha) and scalar n
n is the size of A (and B)
return the form of FM(A,B,a) in the paper
A * sqrt (A^-1 * B + (2a-1)/4*(I-A^-1 *B)^2 - (2a-1)/2*(I-A^-1 * B))
'''
AB = tf.matmul(tf.matrix_inverse(A), B) # A^-1 * B
IAB = tf.subtract(tf.eye(n), AB) # I - A^-1 * B
eta = (2 * a - 1) / 2 # (2a-1)/2
before_root = AB + eta * eta * tf.matmul(IAB, IAB) - eta * (
IAB) # (A^-1 * B + (2a-1)/4*(I-A^-1 *B)^2 - (2a-1)/2*(I-A^-1 * B))
S, U = tf.self_adjoint_eig(before_root)
Sigma_root = tf.matrix_diag(tf.sqrt(S))
after_root = tf.matmul(
tf.matmul(U, Sigma_root), U,
transpose_b=True) # calculate the square root by eig-decomponent
return tf.matmul(A, after_root)
def ComputeDPPrincipalProjection(data, projection_dims, sanitizer, eps_delta,
sigma):
"""Compute differentially private projection.
Args:
data: the input data, each row is a data vector.
projection_dims: the projection dimension.
sanitizer: the sanitizer used for acheiving privacy.
eps_delta: (eps, delta) pair.
sigma: if not None, use noise sigma; otherwise compute it using
eps_delta pair.
Returns:
A projection matrix with projection_dims columns.
"""
eps, delta = eps_delta
# Normalize each row.
normalized_data = tf.nn.l2_normalize(data, 1)
covar = tf.matmul(tf.transpose(normalized_data), normalized_data)
saved_shape = tf.shape(covar)
num_examples = tf.slice(tf.shape(data), [0], [1])
if eps > 0:
# Since the data is already normalized, there is no need to clip
# the covariance matrix.
assert delta > 0
saned_covar = sanitizer.sanitize(tf.reshape(covar, [1, -1]),
eps_delta,
sigma=sigma,
option=san.ClipOption(1.0, False),
num_examples=num_examples)
saned_covar = tf.reshape(saned_covar, saved_shape)
# Symmetrize saned_covar. This also reduces the noise variance.
saned_covar = 0.5 * (saned_covar + tf.transpose(saned_covar))
else:
saned_covar = covar
# Compute the eigen decomposition of the covariance matrix, and
# return the top projection_dims eigen vectors, represented as columns of
# the projection matrix.
eigvals, eigvecs = tf.self_adjoint_eig(saned_covar)
_, topk_indices = tf.nn.top_k(eigvals, projection_dims)
topk_indices = tf.reshape(topk_indices, [projection_dims])
# Gather and return the corresponding eigenvectors.
return tf.transpose(tf.gather(tf.transpose(eigvecs), topk_indices))
def ComputeDPPrincipalProjection(data, projection_dims,
sanitizer, eps_delta, sigma):
"""Compute differentially private projection.
Args:
data: the input data, each row is a data vector.
projection_dims: the projection dimension.
sanitizer: the sanitizer used for acheiving privacy.
eps_delta: (eps, delta) pair.
sigma: if not None, use noise sigma; otherwise compute it using
eps_delta pair.
Returns:
A projection matrix with projection_dims columns.
"""
eps, delta = eps_delta
# Normalize each row.
normalized_data = tf.nn.l2_normalize(data, 1)
covar = tf.matmul(tf.transpose(normalized_data), normalized_data)
saved_shape = tf.shape(covar)
num_examples = tf.slice(tf.shape(data), [0], [1])
if eps > 0:
# Since the data is already normalized, there is no need to clip
# the covariance matrix.
assert delta > 0
saned_covar = sanitizer.sanitize(
tf.reshape(covar, [1, -1]), eps_delta, sigma=sigma,
option=san.ClipOption(1.0, False), num_examples=num_examples)
saned_covar = tf.reshape(saned_covar, saved_shape)
# Symmetrize saned_covar. This also reduces the noise variance.
saned_covar = 0.5 * (saned_covar + tf.transpose(saned_covar))
else:
saned_covar = covar
# Compute the eigen decomposition of the covariance matrix, and
# return the top projection_dims eigen vectors, represented as columns of
# the projection matrix.
eigvals, eigvecs = tf.self_adjoint_eig(saned_covar)
_, topk_indices = tf.nn.top_k(eigvals, projection_dims)
topk_indices = tf.reshape(topk_indices, [projection_dims])
# Gather and return the corresponding eigenvectors.
return tf.transpose(tf.gather(tf.transpose(eigvecs), topk_indices))
def testWrongDimensions(self):
# The input to self_adjoint_eig should be 2-dimensional tensor.
scalar = tf.constant(1.)
with self.assertRaises(ValueError):
tf.self_adjoint_eig(scalar)
vector = tf.constant([1., 2.])
with self.assertRaises(ValueError):
tf.self_adjoint_eig(vector)
tensor = tf.constant([[[1., 2.], [3., 4.]], [[1., 2.], [3., 4.]]])
with self.assertRaises(ValueError):
tf.self_adjoint_eig(tensor)
# The input to batch_batch_self_adjoint_eig should be a tensor of
# at least rank 2.
scalar = tf.constant(1.)
with self.assertRaises(ValueError):
tf.batch_self_adjoint_eig(scalar)
vector = tf.constant([1., 2.])
with self.assertRaises(ValueError):
tf.batch_self_adjoint_eig(vector)
def testWrongDimensions(self):
tensor3 = tf.constant([1., 2.])
with self.assertRaises(ValueError):
tf.self_adjoint_eig(tensor3)
def learn_average_deg_variable(communities = 2, group_size = 10, seed_v=None, projection_dim=2, l_rate=0.00000001, mean=0.3, sd=0.1):
"""testing to see if the loss will decrease backproping through very simple function"""
#now p and q will be generated from a range of
X = tf.placeholder(dtype=tf.float64, shape=[communities*group_size, communities*group_size])
B = tf.cast(X, dtype = tf.float64)
Diag = tf.diag(tf.reduce_sum(B,0))
Diag = tf.cast(Diag, tf.float64)
#by symmetry I should make this a bit more constrained. so
v = tf.Variable(tf.random_normal(shape=[communities*group_size,1], mean=mean,
stddev=sd, dtype=tf.float64,
seed=seed_v, name=None))
degree = tf.cast(communities*group_size, dtype=tf.float64)
r_param = tf.div(tf.cast(1.0, dtype=tf.float64), degree)*tf.matmul(tf.transpose(v), tf.matmul(Diag, v))
BH = (tf.square(r_param)-1)*tf.diag(tf.ones(shape=[communities*group_size], dtype=tf.float64))-tf.mul(r_param, B)+Diag
with tf.Session() as sess:
g = tf.get_default_graph()
with g.gradient_override_map({'SelfAdjointEigV2': 'grassman_with_2d'}):
eigenval, eigenvec = tf.self_adjoint_eig(BH)
#we try to do svm in this subspace
#or we can project it down to 1 dimensions, do the clustering there via some threshold and check if it makes sense
#by computing the loss, if it is too big, we change the angle we project down to...
eigenvec_proj = tf.slice(eigenvec, [0,0], [communities*group_size, projection_dim])
true_assignment_a = tf.concat(0, [-1*tf.ones([group_size], dtype=tf.float64),
tf.ones([group_size], dtype=tf.float64)])
true_assignment_b = -1*true_assignment_a
true_assignment_a = tf.expand_dims(true_assignment_a, 1)
true_assignment_b = tf.expand_dims(true_assignment_b, 1)
projected_a = tf.matmul(tf.matmul(eigenvec_proj, tf.transpose(eigenvec_proj)), true_assignment_a)#tf.transpose(true_assignment_a))
projected_b = tf.matmul(tf.matmul(eigenvec_proj, tf.transpose(eigenvec_proj)), true_assignment_b)#tf.transpose(true_assignment_b))
loss = tf.minimum(tf.reduce_sum(tf.square(tf.sub(projected_a, true_assignment_a))),
tf.reduce_sum(tf.square(tf.sub(projected_b, true_assignment_b))))
optimizer = tf.train.AdamOptimizer(l_rate)
train = optimizer.minimize(loss, var_list=[v])
eigenvec_grad = tf.gradients(eigenvec, v)
loss_grad = tf.gradients(loss, v)
#using Laplacian
Laplacian = Diag-B
eigenval_baseline, eigenvec_baseline = tf.self_adjoint_eig(Laplacian)
project_baseline = tf.slice(eigenvec_baseline, [0,0], [communities*group_size, projection_dim])
projected_a_baseline = tf.matmul(tf.matmul(project_baseline, tf.transpose(project_baseline)), true_assignment_a)#tf.transpose(true_assignment_a))
projected_b_baseline = tf.matmul(tf.matmul(project_baseline, tf.transpose(project_baseline)), true_assignment_b)
loss_baseline = tf.minimum(tf.reduce_sum(tf.square(tf.sub(projected_a_baseline, true_assignment_a))),
tf.reduce_sum(tf.square(tf.sub(projected_b_baseline, true_assignment_b))))
r_op, target = target_subspace(adj=B, groupsize=group_size, communities=communities, diag=Diag, dim_proj=projection_dim)
r_diff = (r_op-r_param) #difference between r_op and r_param is how close we are to the average degree
init = tf.initialize_all_variables()
sess.run(init)
V, R_param, R_diff, avg_deg, Loss_baseline= sess.run([v, r_param, r_diff, r_op, loss_baseline], feed_dict={X:data[0]})
# V_lst = np.zeros(len(data), np.float64)
V_lst = [0.0]*len(data)
R_param_lst = np.zeros(len(data), np.float64)
R_diff_list = np.zeros(len(data), np.float64)
loss_list = [0.0]*len(data)
true_assignment_lst = np.zeros(len(data), np.float64)
avg_deg_lst = np.zeros(len(data), np.float64)
# data_lst = []
r_param_grad_lst = [0.0]*len(data)
loss_baseline = np.zeros(len(data), np.float64)
average_loss = np.zeros(len(data), np.float64)
V_lst[0]=V
R_param_lst[0]=R_param
R_diff_list[0]=R_diff
loss_list[0]=None
true_assignment_lst[0]=None
avg_deg_lst[0]=avg_deg
# data_lst.append(None)
r_param_grad_lst[0]=None
loss_baseline[0]=Loss_baseline
average_loss[0]=None
# print "initial v: {}. r_param: {}, difference between r_param and sqrt average deg {}.".format(a, r, b)
for i in range(1, len(data)):
try:
sess.run(train, feed_dict={X:data[i]})
#if i%print_ratio==0:
#print i
#try:
if i%2==0:
V, R_param, R_diff, Loss, Loss_baseline, r_param_grad, Tru_assign, avg_deg = sess.run([v,
r_param, r_diff, loss, loss_baseline, tf.gradients(loss, r_param),
tf.transpose(projected_a), r_op], feed_dict={X:data[i]})
#for j in range(len(data)):
# loss_tmp = np.zeros(len(data), np.float64)
# tmp = sess.run(loss, feed_dict={X:data[j]})
# loss_tmp[j](tmp)
#np.mean(loss_tmp)
#average_loss = np.mean()
V_lst[j]=V
R_param_lst[j]=R_param
R_diff_list[j]=R_diff
loss_list[j]=Loss
loss_baseline[j]=Loss_baseline
true_assignment_lst[j]=Tru_assign
r_param_grad_lst[j]=r_param_grad
avg_deg_lst[j]=avg_deg
#data_lst.append(data[i])
print "step {}: Loss:{}, Loss_baseline: {}, avg_deg: {}".format(j, Loss, Loss_baseline, avg_deg)
except:
V, R_param, R_diff, Loss, Loss_baseline, r_param_grad, Tru_assign, avg_deg = None, None, None, None, None, None, None, None
V_lst[i]=V
R_param_lst[i]=R_param
R_diff_list[i]=R_diff
loss_list[i]=Loss
loss_baseline[i]=Loss_baseline
true_assignment_lst[i]=Tru_assign
r_param_grad_lst[i]=r_param_grad
avg_deg_lst[i]=avg_deg
#data_lst.append(data[i])
print "step:{} not sucessful".format(i)
pass
d = {"v": V_lst, "r_param": R_param_lst, "r_diff": R_diff_list, "loss": loss_list, "loss_baseline": loss_baseline, "R_grad_list": r_param_grad_lst,
"true_assignmnet": true_assignment_lst, "avg_deg": avg_deg_lst}#,"data":data_lst }
d = pd.DataFrame(d)
easy_size = len(data)
#d.to_csv("mean{}l_rate{}data_size{}p_min{}p_max{}hard_ratio{}.csv".format(mean, l_rate, easy_size+hard_size, p_min, p_max, hard_ratio))
return d
# 矩阵转置
print(sess.run(tf.transpose(matrix_C)))
# 重新初始化会得到不同的值，
# 所以会与之前的matrix_C不是转置矩阵
print('- - - - - - - - - - - - - - - - - - - -')
# 矩阵行列式
print(sess.run(tf.matrix_determinant(matrix_D)))
print('- - - - - - - - - - - - - - - - - - - -')
# 矩阵的逆矩阵
print(sess.run(tf.matrix_inverse(matrix_D)))
# 矩阵的逆矩阵是用平方根法，
# 需要矩阵为对称正定矩阵或者可进行LU分解
print('- - - - - - - - - - - - - - - - - - - -')
# 矩阵分解法
print(sess.run(tf.cholesky(indentity_matrix)))
print('- - - - - - - - - - - - - - - - - - - -')
# 求矩阵特征值和特征向量
print(sess.run(tf.self_adjoint_eig(matrix_D)))
# 第一行 为特征值，
# 其余的向量是对应的特征向量
# 此方法又称矩阵的特征分解
def testNonSquareMatrix(self):
with self.assertRaises(ValueError):
tf.self_adjoint_eig(tf.constant(np.array([[1., 2., 3.], [3., 4., 5.]])))
input2_tensor = tf.Variable(tf.to_int32(input2_value))
# input2_tensor = tf.Variable(input2_value)
dot_tensor = tf.matmul(input_tensor, input2_tensor)
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
dot_tensorflow = sess.run(dot_tensor)
dot_tensorflow.must_close(dot_numpy)
with test("numpy.linalg.eig"):
input_value = numpy.diag((1, 2, 3))
eigenvalues, eigenvectors = numpy.linalg.eig(input_value)
eigenvalues.must_close([1,2,3])
eigenvectors.must_close(numpy.eye(3))
input_tensor = tf.Variable(tf.cast(input_value,tf.float32))
eigen_tensor = tf.self_adjoint_eig(input_tensor)
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
eigen_tensorflow = sess.run(eigen_tensor)
eigen_tensorflow[0,].must_close(eigenvalues)
eigen_tensorflow[1:4,].must_close(eigenvectors)
with test("add, same as matrix"):
input_value = numpy.diag((1, 2, 3))
add_one = input_value + 3
add_one.must_close([
[4, 3, 3],
[3, 5, 3],
[3, 3, 6]
# initialize and shape
with tf.Session() as sess:
sess.run(init)
sess.run(mps)
for i in range(L):
print mps[i].get_shape()
lenv = [0]*L
renv = [0]*L
# Also, deriable
mat = tf.reshape(mps[0],[n,D])
lenv[0] = tf.matmul(mat,mat,transpose_a=True)
# A Tensor. Has the same type as input. Shape is [M+1, M].
# the first row is eigenvalues, columns of other part are eignvectors.
res = tf.self_adjoint_eig(lenv[0])
import scipy.linalg
l0 = numpy.einsum('pi,pj->ij',mps0[0],mps0[0])
e,v = scipy.linalg.eigh(l0)
print e
print v
flt = tf.reshape(mps[0],[-1])
print 'shp',flt.get_shape()
tr0 = tf.reduce_sum(tf.diag_part(lenv[0]))
g0 = tf.gradients(tr0,mps[0])
tr1 = tf.reduce_sum(tf.mul(mps[0],mps[0]))
g1 = tf.gradients(tr1,mps[0])
# 3x2 random uniform matrix C = tf.random_uniform([3,2]) print(sess.run(C)) print(sess.run(C)) # Note that we are reinitializing, hence the new random variabels # Create matrix from np array D = tf.convert_to_tensor(np.array([[1., 2., 3.], [-3., -7., -1.], [0., 5., -2.]])) print(sess.run(D)) # Matrix addition/subtraction print(sess.run(A+B)) print(sess.run(B-B)) # Matrix Multiplication print(sess.run(tf.matmul(B, identity_matrix))) # Matrix Transpose print(sess.run(tf.transpose(C))) # Again, new random variables # Matrix Determinant print(sess.run(tf.matrix_determinant(D))) # Matrix Inverse print(sess.run(tf.matrix_inverse(D))) # Cholesky Decomposition print(sess.run(tf.cholesky(identity_matrix))) # Eigenvalues and Eigenvectors print(sess.run(tf.self_adjoint_eig(D)))
def test_svm_cluster(communities = 2, group_size = 10, seed=1, seed_r=1, p=0.8, q=0.05, name='test1', projection_dim=2, iterations=100,
print_ratio=10, l_rate=0.1, mean=2.0, sd=0.4):
"""testing to see if the loss will decrease backproping through very simple function"""
B = np.asarray(balanced_stochastic_blockmodel(communities, group_size, p, q, seed)).astype(np.double)
B = tf.cast(B, dtype = tf.float64)
Diag = tf.diag(tf.reduce_sum(B,0))
Diag = tf.cast(Diag, tf.float64)
r = tf.Variable(tf.random_normal(shape=[1], mean=mean,
stddev=sd, dtype=tf.float64,
seed=seed_r, name=None))
BH = (tf.square(r)-1)*tf.diag(tf.ones(shape=[communities*group_size], dtype=tf.float64))-tf.mul(r, B)+Diag
with tf.Session() as sess:
g = tf.get_default_graph()
with g.gradient_override_map({'SelfAdjointEigV2': name}):
eigenval, eigenvec = tf.self_adjoint_eig(BH)
#we try to do svm in this subspace
#or we can project it down to 1 dimensions, do the clustering there via some threshold and check if it makes sense
#by computing the loss, if it is too big, we change the angle we project down to...
eigenvec_proj = tf.slice(eigenvec, [0,0], [communities*group_size, projection_dim])
true_assignment_a = tf.concat(0, [-1*tf.ones([group_size], dtype=tf.float64),
tf.ones([group_size], dtype=tf.float64)])
true_assignment_b = -1*true_assignment_a
true_assignment_a = tf.expand_dims(true_assignment_a, 1)
true_assignment_b = tf.expand_dims(true_assignment_b, 1)
projected_a = tf.matmul(tf.matmul(eigenvec_proj, tf.transpose(eigenvec_proj)), true_assignment_a)#tf.transpose(true_assignment_a))
projected_b = tf.matmul(tf.matmul(eigenvec_proj, tf.transpose(eigenvec_proj)), true_assignment_b)#tf.transpose(true_assignment_b))
loss = tf.minimum(tf.reduce_sum(tf.square(tf.sub(projected_a, true_assignment_a))),
tf.reduce_sum(tf.square(tf.sub(projected_b, true_assignment_b))))
optimizer = tf.train.GradientDescentOptimizer(l_rate)
train = optimizer.minimize(loss, var_list=[r])
eigenvec_grad = tf.gradients(eigenvec, r)
loss_grad = tf.gradients(loss, r)
r_op, target = target_subspace(adj=B, groupsize=group_size, communities=communities, diag=Diag, dim_proj=projection_dim)
r_op_projection_a = tf.matmul(tf.matmul(target, tf.transpose(target)), true_assignment_a)
r_op_projection_b = tf.matmul(tf.matmul(target, tf.transpose(target)), true_assignment_b)
r_op_loss = tf.minimum(tf.reduce_sum(tf.square(tf.sub(r_op_projection_a, true_assignment_a))),
tf.reduce_sum(tf.square(tf.sub(r_op_projection_b, true_assignment_b))))
init = tf.initialize_all_variables()
sess.run(init)
a,b,c,d= sess.run([r, r_op, r_op_loss, tf.transpose(r_op_projection_a)])
a_lst = []
b_lst = []
c_lst = []
d_lst = []
a_lst.append(a)
b_lst.append(b)
c_lst.append(c)
d_lst.append(d)
print "initial r: {}. r_op = sqrt(average degree) : {} . Loss associated with r_op: {}. r_op assignments {}.".format(a, b, c, d)
for i in range(iterations):
try: sess.run(train)
except:
pass
if i%print_ratio==0:
#print i
try:
a,b,c,d = sess.run([r, loss, tf.gradients(loss, r), tf.transpose(projected_a)])
a_lst.append(a)
b_lst.append(b)
c_lst.append(c)
d_lst.append(d)
except:
a,b,c,d = 0, 0, 0, 0
a_lst.append(a)
b_lst.append(b)
c_lst.append(c)
d_lst.append(d)
#print "current r: {}, current loss: {}, gradient of loss/r is {} and current assignments (up to sign) {}.".format(a,b,c,d)
d = {"r_value": a_lst, "loss": b_lst, "gradient_loss_r": c_lst, "projection": d_lst}
d = pd.DataFrame(d)
d.to_csv("/Users/xiangli/Desktop/clusternet/plot_data/r{}rate{}p{}q{}iterations{}step{}.csv".format(mean, l_rate, p, q, iterations, print_ratio))
return d
A = tf.truncated_normal([2, 3]) print(sess.run(A)) # 2x3 constant matrix: B = tf.fill([2, 3], 5.0) print(sess.run(B)) # 3x2 random uniform matrix: C = tf.random_uniform([3, 2]) print(sess.run(C)) # Create matrix from np array: D = tf.convert_to_tensor(np.array([[1., 2., 3.], [-3., -7., -1.], [0., 5., -2.]])) print(sess.run(D)) # Matrix Operations Matrix addition/subtraction: print(sess.run(A + B)) print(sess.run(B - B)) # Matrix Multiplication: print(sess.run(tf.matmul(B, identity_matrix))) # Matrix Transpose: print(sess.run(tf.transpose(C))) # Matrix Determinant: print(sess.run(tf.matrix_determinant(D))) # Matrix Inverse: print(sess.run(tf.matrix_inverse(D))) # Cholesky Decomposition: print(sess.run(tf.cholesky(identity_matrix))) # Eigenvalues and Eigenvectors: We use tf.self_adjoint_eig() function, which returns two objects, # first one is an array of eigenvalues, the second is a matrix of the eigenvectors. eigenvalues, eigenvectors = sess.run(tf.self_adjoint_eig(D)) print(eigenvalues) print(eigenvectors)
def learn_average_deg_variable(communities = 2, group_size = 10, seed_v=None, projection_dim=2, l_rate=0.00000001, mean=0.3, sd=0.1):
"""testing to see if the loss will decrease backproping through very simple function"""
#now p and q will be generated from a range of
X = tf.placeholder(dtype=tf.float64, shape=[communities*group_size, communities*group_size])
B = tf.cast(X, dtype = tf.float64)
Diag = tf.diag(tf.reduce_sum(B,0))
Diag = tf.cast(Diag, tf.float64)
#by symmetry I should make this a bit more constrained. so
v = tf.Variable(tf.random_normal(shape=[communities*group_size,1], mean=mean,
stddev=sd, dtype=tf.float64,
seed=seed_v, name=None))
degree = tf.cast(communities*group_size, dtype=tf.float64)
r_param = tf.div(tf.cast(1.0, dtype=tf.float64), degree)*tf.matmul(tf.transpose(v), tf.matmul(Diag, v))
BH = (tf.square(r_param)-1)*tf.diag(tf.ones(shape=[communities*group_size], dtype=tf.float64))-tf.mul(r_param, B)+Diag
with tf.Session() as sess:
g = tf.get_default_graph()
with g.gradient_override_map({'SelfAdjointEigV2': 'grassman_with_2d'}):
eigenval, eigenvec = tf.self_adjoint_eig(BH)
#we try to do svm in this subspace
#or we can project it down to 1 dimensions, do the clustering there via some threshold and check if it makes sense
#by computing the loss, if it is too big, we change the angle we project down to...
eigenvec_proj = tf.slice(eigenvec, [0,0], [communities*group_size, projection_dim])
true_assignment_a = tf.concat(0, [-1*tf.ones([group_size], dtype=tf.float64),
tf.ones([group_size], dtype=tf.float64)])
true_assignment_b = -1*true_assignment_a
true_assignment_a = tf.expand_dims(true_assignment_a, 1)
true_assignment_b = tf.expand_dims(true_assignment_b, 1)
projected_a = tf.matmul(tf.matmul(eigenvec_proj, tf.transpose(eigenvec_proj)), true_assignment_a)#tf.transpose(true_assignment_a))
projected_b = tf.matmul(tf.matmul(eigenvec_proj, tf.transpose(eigenvec_proj)), true_assignment_b)#tf.transpose(true_assignment_b))
loss = tf.minimum(tf.reduce_sum(tf.square(tf.sub(projected_a, true_assignment_a))),
tf.reduce_sum(tf.square(tf.sub(projected_b, true_assignment_b))))
optimizer = tf.train.AdamOptimizer(l_rate)
train = optimizer.minimize(loss, var_list=[v])
eigenvec_grad = tf.gradients(eigenvec, v)
loss_grad = tf.gradients(loss, v)
r_op, target = target_subspace(adj=B, groupsize=group_size, communities=communities, diag=Diag, dim_proj=projection_dim)
r_diff = (r_op-r_param) #difference between r_op and r_param is how close we are to the average degree
init = tf.initialize_all_variables()
sess.run(init)
a,r, b, avg_deg= sess.run([v, r_param, r_diff, r_op], feed_dict={X:data[0]})
a_lst = []
r_lst = []
r_diff_list = []
b_lst = []
c_lst = []
d_lst = []
avg_deg_lst = []
data_lst = []
r_param_grad_lst = []
a_lst.append(a)
r_lst.append(r)
r_diff_list.append(b)
b_lst.append(None)
c_lst.append(None)
d_lst.append(None)
avg_deg_lst.append(avg_deg)
data_lst.append(None)
r_param_grad_lst.append(None)
print "initial v: {}. r_param: {}, difference between r_param and sqrt average deg {}.".format(a, r, b)
for i in range(len(data)):
try:
sess.run(train, feed_dict={X:data[i]})
#if i%print_ratio==0:
#print i
#try:
a,r, k, b,c,d, r_param_grad, avg_deg = sess.run([v, r_param, r_diff, loss, tf.gradients(loss, v), tf.transpose(projected_a), tf.gradients(loss, r_param), r_op], feed_dict={X:data[i]})
a_lst.append(a)
r_lst.append(r)
r_diff_list.append(k)
b_lst.append(b)
c_lst.append(c)
d_lst.append(d)
r_param_grad_lst.append(r_param_grad)
avg_deg_lst.append(avg_deg)
data_lst.append(data[i])
print "step: {}: loss: {}, avg_deg: {} r_param:{}, r_diff: {}, r_param gradient: {}".format(i, b, avg_deg, r, k, r_param_grad)
except:
a,r, k, b,c,d, r_param_grad, avg_deg = None, None, None, None, None, None, None, None, None
a_lst.append(a)
r_lst.append(r)
r_diff_list.append(k)
b_lst.append(b)
c_lst.append(c)
d_lst.append(d)
r_param_grad_lst.append(r_param_grad)
avg_deg_lst.append(avg_deg)
data_lst.append(data[i])
print "step:{} not sucessful".format(i)
pass
d = {"v": a_lst, "r_param": r_lst, "r_diff": r_diff_list, "loss": b_lst, "gradient_loss_v": c_lst, "projection": d_lst,
"r_param_grad": r_param_grad_lst, "avg_deg": avg_deg_lst, "data":data_lst }
d = pd.DataFrame(d)
easy_size = len(data)
d.to_csv("/Users/xiangli/Desktop/clusternet/Learning_r_matrix_data/mean{}l_rate{}data_size{}p_min{}p_max{}hard_ratio{}.csv".format(mean, l_rate, easy_size+hard_size, p_min, p_max, hard_ratio))
def computeStatsEigen(self):
""" compute the eigen decomp using copied var stats to avoid concurrent read/write from other queue """
# TO-DO: figure out why this op has delays (possibly moving
# eigenvectors around?)
with tf.device('/cpu:0'):
def removeNone(tensor_list):
local_list = []
for item in tensor_list:
if item is not None:
local_list.append(item)
return local_list
def copyStats(var_list):
print("copying stats to buffer tensors before eigen decomp")
redundant_stats = {}
copied_list = []
for item in var_list:
if item is not None:
if item not in redundant_stats:
if self._use_float64:
redundant_stats[item] = tf.cast(
tf.identity(item), tf.float64)
else:
redundant_stats[item] = tf.identity(item)
copied_list.append(redundant_stats[item])
else:
copied_list.append(None)
return copied_list
#stats = [copyStats(self.fStats), copyStats(self.bStats)]
#stats = [self.fStats, self.bStats]
stats_eigen = self.stats_eigen
computedEigen = {}
eigen_reverse_lookup = {}
updateOps = []
# sync copied stats
# with tf.control_dependencies(removeNone(stats[0]) +
# removeNone(stats[1])):
with tf.control_dependencies([]):
for stats_var in stats_eigen:
if stats_var not in computedEigen:
eigens = tf.self_adjoint_eig(stats_var)
e = eigens[0]
Q = eigens[1]
if self._use_float64:
e = tf.cast(e, tf.float32)
Q = tf.cast(Q, tf.float32)
updateOps.append(e)
updateOps.append(Q)
computedEigen[stats_var] = {'e': e, 'Q': Q}
eigen_reverse_lookup[e] = stats_eigen[stats_var]['e']
eigen_reverse_lookup[Q] = stats_eigen[stats_var]['Q']
self.eigen_reverse_lookup = eigen_reverse_lookup
self.eigen_update_list = updateOps
if KFAC_DEBUG:
self.eigen_update_list = [item for item in updateOps]
with tf.control_dependencies(updateOps):
updateOps.append(tf.Print(tf.constant(
0.), [tf.convert_to_tensor('computed factor eigen')]))
return updateOps