def testWrongDimensions(self): # The input to self_adjoint_eig should be a tensor of # at least rank 2. scalar = tf.constant(1.) with self.assertRaises(ValueError): tf.self_adjoint_eig(scalar) vector = tf.constant([1., 2.]) with self.assertRaises(ValueError): tf.self_adjoint_eig(vector)
def Test(self): np.random.seed(1) n = shape_[-1] batch_shape = shape_[:-2] a = np.random.uniform( low=-1.0, high=1.0, size=n * n).reshape([n, n]).astype(dtype_) a += a.T a = np.tile(a, batch_shape + (1, 1)) # Optimal stepsize for central difference is O(epsilon^{1/3}). epsilon = np.finfo(dtype_).eps delta = 0.1 * epsilon**(1.0 / 3.0) # tolerance obtained by looking at actual differences using # np.linalg.norm(theoretical-numerical, np.inf) on -mavx build if dtype_ == np.float32: tol = 1e-2 else: tol = 1e-7 with self.test_session(): tf_a = tf.constant(a) tf_e, tf_v = tf.self_adjoint_eig(tf_a) for b in tf_e, tf_v: x_init = np.random.uniform( low=-1.0, high=1.0, size=n * n).reshape([n, n]).astype(dtype_) x_init += x_init.T x_init = np.tile(x_init, batch_shape + (1, 1)) theoretical, numerical = tf.test.compute_gradient( tf_a, tf_a.get_shape().as_list(), b, b.get_shape().as_list(), x_init_value=x_init, delta=delta) self.assertAllClose(theoretical, numerical, atol=tol, rtol=tol)
def Test(self): np.random.seed(1) n = shape_[-1] batch_shape = shape_[:-2] a = np.random.uniform( low=-1.0, high=1.0, size=n * n).reshape([n, n]).astype(dtype_) a += a.T a = np.tile(a, batch_shape + (1, 1)) if dtype_ == np.float32: atol = 1e-4 else: atol = 1e-12 for compute_v in False, True: np_e, np_v = np.linalg.eig(a) with self.test_session(): if compute_v: tf_e, tf_v = tf.self_adjoint_eig(tf.constant(a)) # Check that V*diag(E)*V^T is close to A. a_ev = tf.batch_matmul( tf.batch_matmul(tf_v, tf.batch_matrix_diag(tf_e)), tf_v, adj_y=True) self.assertAllClose(a_ev.eval(), a, atol=atol) # Compare to numpy.linalg.eig. CompareEigenDecompositions(self, np_e, np_v, tf_e.eval(), tf_v.eval(), atol) else: tf_e = tf.self_adjoint_eigvals(tf.constant(a)) self.assertAllClose( np.sort(np_e, -1), np.sort(tf_e.eval(), -1), atol=atol)
def r_loss(communities = 2, group_size = 10, seed=None, p=0.4, q=0.05, r=1.0, projection_dim=2): """testing to see if the loss will decrease backproping through very simple function""" B = np.asarray(balanced_stochastic_blockmodel(communities, group_size, p, q, seed)).astype(np.double) B = tf.cast(B, tf.float64) Diag = tf.diag(tf.reduce_sum(B,0)) Diag = tf.cast(Diag, tf.float64) #r_grid = tf.linspace(r_min, r_max, grid_size) r = tf.cast(r, tf.float64) BH = (tf.square(r)-1)*tf.diag(tf.ones(shape=[communities*group_size], dtype=tf.float64))-tf.mul(r, B)+Diag with tf.Session() as sess: eigenval, eigenvec = tf.self_adjoint_eig(BH) eigenvec_proj = tf.slice(eigenvec, [0,0], [communities*group_size, projection_dim]) true_assignment_a = tf.concat(0, [-1*tf.ones([group_size], dtype=tf.float64), tf.ones([group_size], dtype=tf.float64)]) true_assignment_b = -1*true_assignment_a true_assignment_a = tf.expand_dims(true_assignment_a, 1) true_assignment_b = tf.expand_dims(true_assignment_b, 1) projected_a = tf.matmul(tf.matmul(eigenvec_proj, tf.transpose(eigenvec_proj)), true_assignment_a)#tf.transpose(true_assignment_a)) projected_b = tf.matmul(tf.matmul(eigenvec_proj, tf.transpose(eigenvec_proj)), true_assignment_b)#tf.transpose(true_assignment_b)) loss = tf.minimum(tf.reduce_sum(tf.square(tf.sub(projected_a, true_assignment_a))), tf.reduce_sum(tf.square(tf.sub(projected_b, true_assignment_b)))) d = sess.run(loss) return d
def test_SelfAdjointEig(self): t = tf.self_adjoint_eig(np.array([3,2,1, 2,4,5, 1,5,6]).reshape(3, 3).astype("float32")) # the order of eigen vectors and values may differ between tf and np, so only compare sum # and mean # also, different numerical algorithms are used, so account for difference in precision by # comparing numbers with 4 digits self.check(t, ndigits=4, stats=True, abs=True)
def _psd_mask(x): """Computes whether each square matrix in the input is positive semi-definite. Args: x: A floating-point `Tensor` of shape `[B1, ..., Bn, M, M]`. Returns: mask: A floating-point `Tensor` of shape `[B1, ... Bn]`. Each scalar is 1 if the corresponding matrix was PSD, otherwise 0. """ # Allegedly # https://scicomp.stackexchange.com/questions/12979/testing-if-a-matrix-is-positive-semi-definite # it is more efficient to test for positive semi-definiteness by # trying to compute the Cholesky decomposition -- the matrix is PSD # if you succeed and not PSD if you fail. However, TensorFlow's # Cholesky raises an exception if _any_ of the input matrices are # not PSD, from which I don't know how to extract _which ones_, so I # proceed by explicitly computing all the eigenvalues and checking # whether they are all positive or not. # # Also, as was discussed in the answer, it is somewhat dangerous to # treat SPD-ness as binary in floating-point arithmetic. Cholesky # factorization can complete and 'look' like everything is fine # (e.g., O(1) entries and a diagonal of all ones) but the matrix can # have an exponential condition number. eigenvalues, _ = tf.self_adjoint_eig(x) return tf.cast( tf.reduce_min(eigenvalues, axis=-1) >= 0, dtype=x.dtype)
def target_subspace(adj, groupsize, communities, diag, dim_proj): normalizer = tf.cast(2.0*groupsize*communities, dtype=tf.float64) total_degree = tf.cast(tf.reduce_sum(adj), dtype=tf.float64) r = tf.sqrt(total_degree/normalizer) BH_op = (tf.square(r)-1)*tf.diag(tf.ones(shape=[communities*groupsize], dtype=tf.float64))-r*adj+diag val, vec = tf.self_adjoint_eig(BH_op) #this is already normalized so no need to normalize subspace = tf.slice(vec, [0,0], [communities*groupsize, dim_proj]) return r, subspace
def _compareSelfAdjointEig(self, x, use_gpu=False): with self.test_session() as sess: tf_eig = tf.self_adjoint_eig(tf.constant(x)) tf_eig_out = sess.run([tf_eig])[0] d, _ = x.shape self.assertEqual([d+1, d], tf_eig.get_shape().dims) self._testEigs(x, d, tf_eig_out, use_gpu)
def matrix_log(self): eigenvalues, eigenvectors = tf.self_adjoint_eig(self.reduced_matrix1) eigenvalues_log = tf.log( tf.clip_by_value(eigenvalues, 1e-8, tf.reduce_max(eigenvalues))) diag_eigenvalues_log = tf.matrix_diag(eigenvalues_log) eigenvectors_inverse = tf.matrix_inverse(eigenvectors) return tf.matmul(tf.matmul(eigenvectors, diag_eigenvalues_log), eigenvectors_inverse)
def inverse_Hessian(self, H, thr=0.001): e, v = tf.self_adjoint_eig(H) thr_condition = tf.greater(tf.abs(e), thr) sign_ = tf.cast(tf.greater(e, 0.0), tf.float32) e_inv = tf.where( thr_condition, tf.divide(1.0, e), tf.multiply(tf.multiply(tf.ones_like(e), 100.0), sign_)) H_inv = tf.matmul(tf.matmul(v, tf.diag(e_inv)), tf.transpose(v)) return H_inv, e
def build_network(self,pointclouds_pl,is_training,is_eveluate,bn_decay = None): with_bn = self.conf.get_bool('with_bn') batch_size = pointclouds_pl.get_shape()[0].value num_point = pointclouds_pl.get_shape()[1].value if (self.conf['with_rotations']): cov = self.tf_cov(pointclouds_pl) _, axis = tf.self_adjoint_eig(cov) axis = tf.where(tf.linalg.det(axis) < 0, tf.matmul(axis, tf.tile( tf.constant([[[0, 1], [1, 0]]], dtype=tf.float32), multiples=[axis.get_shape()[0], 1, 1])), axis) indicies = [[[b, 0, 0], [b, 2, 0], [b, 0, 2], [b, 2, 2]] for b in list(range(batch_size))] updates = tf.reshape(axis, [batch_size, -1]) updates = tf.reshape(tf.matmul( tf.tile(tf.constant([[[0, 1, 0, 0], [0, 0, 0, 1], [1, 0, 0, 0], [0, 0, 1, 0]]], dtype=tf.float32), multiples=[batch_size, 1, 1]), tf.expand_dims(updates, axis=-1)), shape=[batch_size, -1]) alignment_transform = tf.scatter_nd(indices=indicies, updates=updates, shape=[batch_size, 3, 3]) + tf.expand_dims( tf.diag([0.0, 1.0, 0.0]), axis=0) mean_points = tf.reduce_mean(pointclouds_pl, axis=1, keepdims=True) pointclouds_pl = tf.matmul(pointclouds_pl - mean_points, alignment_transform) + mean_points ps_function_pl = tf.concat([pointclouds_pl,tf.ones(shape=[batch_size,num_point,1],dtype=tf.float32)],axis=2) pool_sizes_sigma = self.conf.get_list('pool_sizes_sigma') spacing = self.conf.get_float('kernel_spacing') network = ps_function_pl input_channel = network.get_shape()[2].value blocks = self.conf.get_list('blocks_out_channels') for block_index,block in enumerate(blocks): block_elm = ConvElements(pointclouds_pl, 1. * tf.reciprocal(tf.sqrt(tf.cast(pointclouds_pl.get_shape()[1].value,tf.float32))),spacing,self.conf.get_float('kernel_sigma_factor')) for out_index,out_channel in enumerate(block): network = ConvLayer(input_channel, block_elm, out_channel, '{0}_block_{1}'.format(block_index,out_index),is_training).get_layer(network,with_bn,bn_decay,self.conf.get_bool('interpolation')) input_channel = out_channel pointclouds_pl, network = PoolingLayer(block_elm, out_channel, out_channel, int(pool_sizes_sigma[block_index + 1][0])).get_layer(network,is_subsampling=self.conf.get_bool('subsampling'),use_fps= tf.logical_or(is_training,is_eveluate)) network = tf.reshape(network, [batch_size, -1]) network = tf_util.fully_connected(network, self.conf.get_int('fc1.size'), bn=True, is_training=is_training, scope='fc1', bn_decay=bn_decay) network = tf_util.dropout(network, keep_prob=self.conf.get_float('dropout.keep_prob'), is_training=is_training, scope='dp1') network = tf_util.fully_connected(network, self.conf.get_int('fc2.size'), bn=True, is_training=is_training, scope='fc2', bn_decay=bn_decay) network = tf_util.dropout(network, keep_prob=self.conf.get_float('dropout.keep_prob'), is_training=is_training, scope='dp2') network = tf_util.fully_connected(network, 40, activation_fn=None, scope='fc3') return network
def build_KL(self): """ The covariance of q(u) has a kronecker structure, so appropriate reductions apply for the trace and logdet terms. """ # Mahalanobis term, m^T K^{-1} m Kuu = [make_Kuu(kern, a, b, self.ms) for kern, a, b, in zip(self.kerns, self.a, self.b)] Kim = kron_vec_apply(Kuu, self.q_mu, 'solve') KL = 0.5*tf.reduce_sum(self.q_mu * Kim) # Constant term KL += -0.5*tf.cast(tf.size(self.q_mu), float_type) # Log det term Ls = [tf.matrix_band_part(q_sqrt_d, -1, 0) for q_sqrt_d in self.q_sqrt_kron] N_others = [float(np.prod(self.Ms)) / M for M in self.Ms] Q_logdets = [tf.reduce_sum(tf.log(tf.square(tf.diag_part(L)))) for L in Ls] KL += -0.5 * reduce(tf.add, [N*logdet for N, logdet in zip(N_others, Q_logdets)]) # trace term tr(K^{-1} Sigma_q) Ss = [tf.matmul(L, tf.transpose(L)) for L in Ls] traces = [K.trace_KiX(S) for K, S, in zip(Kuu, Ss)] KL += 0.5 * reduce(tf.mul, traces) # kron-trace is the produce of traces # log det term Kuu Kuu_logdets = [K.logdet() for K in Kuu] KL += 0.5 * reduce(tf.add, [N*logdet for N, logdet in zip(N_others, Kuu_logdets)]) if self.use_two_krons: # extra logdet terms: Ls_2 = [tf.matrix_band_part(q_sqrt_d, -1, 0) for q_sqrt_d in self.q_sqrt_kron_2] LiL = [tf.matrix_triangular_solve(L1, L2) for L1, L2 in zip(Ls, Ls_2)] eigvals = [tf.self_adjoint_eig(tf.matmul(tf.transpose(mat), mat))[0] for mat in LiL] # discard eigenvectors eigvals_kronned = kron([tf.reshape(e, [1, -1]) for e in eigvals]) KL += -0.5 * tf.reduce_sum(tf.log(1 + eigvals_kronned)) # extra trace terms Ss = [tf.matmul(L, tf.transpose(L)) for L in Ls_2] traces = [K.trace_KiX(S) for K, S, in zip(Kuu, Ss)] KL += 0.5 * reduce(tf.mul, traces) # kron-trace is the produce of traces elif self.use_extra_ranks: # extra logdet terms KiW = kron_mat_apply(Kuu, self.q_sqrt_W, 'solve', self.use_extra_ranks) WTKiW = tf.matmul(tf.transpose(self.q_sqrt_W), KiW) L_extra = tf.cholesky(np.eye(self.use_extra_ranks) + WTKiW) KL += -0.5 * tf.reduce_sum(tf.log(tf.square(tf.diag_part(L_extra)))) # extra trace terms KL += 0.5 * tf.reduce_sum(tf.diag_part(WTKiW)) return KL
def entropy_with_gram(self, norm_gram): with tf.device('/cpu:0'): eigvals, _ = tf.self_adjoint_eig(norm_gram) # Fix possible numerical instabilities: # Remove small negatives eigvals = tf.nn.relu(eigvals) # Ensure eigenvalues sum 1 eigvals = eigvals / tf.reduce_sum(eigvals) # Compute entropy in the specified base sum_term = tf.reduce_sum(eigvals**self.alpha) entropy = tf.log(sum_term) / (1.0 - self.alpha) entropy = entropy / np.log(self.log_base) return entropy
def fg_mf_gamma(param1, param2): mean1, var = param1[0], param1[1] mean2, alpha, beta = param2[0], param2[1], param2[2] ab_ratio = alpha / beta n, p = tf.to_float(tf.shape(mean1)[0]), tf.to_float(tf.shape(mean1)[1]) e, _ = tf.self_adjoint_eig(var) logdet_var = tf.reduce_sum(tf.log(e), [-1]) kl = -n * p * (tf.digamma(alpha) - tf.log(beta + 1e-10)) - logdet_var kl = kl - n * p + ab_ratio * tf.trace(var) kl = kl + ab_ratio * tf.reduce_sum(tf.square(mean2 - mean1), [-1, -2]) return kl * 0.5
def log_coral_loss(source_output, target_output, percent_lambda=0.5): """[Calculate LogCoral loss] Args: source_output ([tf tensor]): [Source feature map tensor] target_output ([tf tensor]): [Target feature map tensor] percent_lambda (weighting factor, optional): [Log CORAL loss weighting factor]. Defaults to 0.5. Returns: [tf tensor]: [Log CORAL loss per batch] """ # regularized covariances result in inf or nan # First: subtract the mean from the data matrix h_src = source_output h_trg = target_output batch_size = tf.cast(tf.shape(h_src)[0], tf.float32) h_src = h_src - tf.reduce_mean(h_src, axis=0) h_trg = h_trg - tf.reduce_mean(h_trg, axis=0) cov_source = (1.0 / (batch_size - 1)) * tf.matmul( h_src, h_src, transpose_a=True) # + gamma * tf.eye(self.hidden_repr_size) cov_target = (1.0 / (batch_size - 1)) * tf.matmul(h_trg, h_trg, transpose_a=True) eig_source = tf.self_adjoint_eig(cov_source) eig_target = tf.self_adjoint_eig(cov_target) log_cov_source = tf.matmul( eig_source[1], tf.matmul(tf.diag(tf.log(eig_source[0])), eig_source[1], transpose_b=True), ) log_cov_target = tf.matmul( eig_target[1], tf.matmul(tf.diag(tf.log(eig_target[0])), eig_target[1], transpose_b=True), ) return percent_lambda * tf.reduce_mean( tf.square(tf.subtract(log_cov_source, log_cov_target)))
def prior_fn(latent_dimension): cov_init = util.positive_definate_initializer([10] + [latent_dimension] * 2) eigvals = tf.self_adjoint_eig( tf.divide(cov_init + tf.matrix_transpose(cov_init), 2., name='symmetrised'))[0] cov_init = tf.Print(cov_init, [cov_init]) return parameterized_distributions.gmm.GMM(10, latent_dimension, cov_init=cov_init, trainable=True).model
def test_matrixrelu(): input_shape = (4, 100, 10) epsilon = 1e-4 data = np.random.randn(*input_shape).astype(K.floatx()) k_data = K.batch_dot(data, data.transpose(0, 2, 1)) sess = K.get_session() with sess.as_default(): data = K.eval(k_data) # print(data) print(data.shape) # traditional log res = np.zeros(shape=(4, 100, 100)) from numpy.linalg import eig for i in range(data.shape[0]): tmp_d = data[i, :, :] s, u = eig(tmp_d) comp = np.zeros_like(s) + epsilon s[np.where(s < epsilon)] = epsilon s = np.diag(s) tmp_res = np.matmul(u, np.matmul(s, u.transpose())) res[i, :, :] = tmp_res import tensorflow as tf tf_input = tf.placeholder(K.floatx(), (4, 100, 100)) tf_s, tf_u = tf.self_adjoint_eig(tf_input) comp = tf.zeros_like(tf_s) + epsilon comp = tf.Print(comp, [comp], message='comp:') tf_s = tf.Print(tf_s, [tf_s], message='tf_s:', summarize=400) inner = tf.where(tf.less(tf_s, comp), comp, tf_s) inner = tf.Print(inner, [inner], 'inner:') inner = tf.matrix_diag(inner) tf_relu = tf.matmul(tf_u, tf.matmul(inner, tf.transpose(tf_u, [0, 2, 1]))) with sess.as_default(): tf_result = tf_relu.eval({tf_input: data}) # log layer a = layer_test(MatrixReLU, input_data=data, kwargs={ 'epsilon': epsilon, }, input_shape=(4, 100, 100), input_dtype=K.floatx()) # a = tf_result # print(a - res) diff = np.linalg.norm(a - res) print(diff) assert_allclose(a, res, rtol=1e-4)
def __init__(self, phys_d, bond_d, A_matrices=None, symmetrize=True, hamiltonian=None, r_prec=1e-14): """ :param phys_d: Physical dimension of the state e.g. 2 for spin-1/2 systems. :param bond_d: Bond dimension, the size of the A matrices. :param A_matrices: Square matrices of size `bond_d` forming the Matrix Product State. :param symmetrize: Boolean indicating A matrices are symmetrized. :param hamiltonian: Tensor of shape [phys_d, phys_d, phys_d, phys_d] giving two site Hamiltonian """ self._session = tf.Session() self.r_prec = r_prec self.phys_d = phys_d self.bond_d = bond_d self.hamiltonian = hamiltonian self.mps_manifold = pymanopt.manifolds.Stiefel(phys_d * bond_d, bond_d) # Define the A if A_matrices is None: A_init = tf.reshape(self.mps_manifold.rand(), [phys_d, bond_d, bond_d]) else: A_init = A_matrices # Create Stiefel from the A Stiefel_init = tf.reshape(A_init, [self.phys_d * self.bond_d, self.bond_d]) # Define the variational tensor variable Stiefel, and from there the A self.Stiefel = tf.get_variable("Stiefel_matrix", initializer=Stiefel_init, trainable=True, dtype=tf.float64) self.A = tf.reshape(self.Stiefel, [self.phys_d, self.bond_d, self.bond_d]) if symmetrize: self.A = self._symmetrize(self.A) self._transfer_matrix = None self._right_eigenvector = None self._all_eig = tf.self_adjoint_eig(self.transfer_matrix) self._dominant_eig = None self._variational_energy = None if hamiltonian is not None: if symmetrize: self.variational_energy = self._add_variational_energy_symmetric_mps(hamiltonian) else: self.variational_energy = self._add_variational_energy_left_canonical_mps(hamiltonian)
def call(self, x, mask=None): """ Returns ------- 3D tensor with same shape as input """ if K.backend() == 'theano' or K.backend() == 'CNTK': raise NotImplementedError( "This is not implemented for theano anymore.") else: if self.built: # import tensorflow as tf # from kyu.tensorflow.ops import safe_truncated_sqrt, safe_sign_sqrt # with tf.device('/cpu:0'): # s, u = safe_matrix_eig_op(x) # # s, u = tf.self_adjoint_eig(x) # inner = safe_sign_sqrt(s) # if self.norm == 'l2': # inner /= tf.reduce_max(inner) # elif self.norm == 'frob' or self.norm == 'Frob': # inner /= tf.sqrt(tf.reduce_sum(s)) # elif self.norm is None: # pass # else: # raise ValueError("PowTransform: Normalization not supported {}".format(self.norm)) # # inner = tf.Print(inner, [inner], message='power inner', summarize=65) # inner = tf.matrix_diag(inner) # tf_pow = tf.matmul(u, tf.matmul(inner, tf.transpose(u, [0, 2, 1]))) # return tf_pow import tensorflow as tf s, u = tf.self_adjoint_eig(x) comp = tf.zeros_like(s) + self.eps inner = tf.where(tf.less(s, comp), comp, s) inner = inner + self.eps inner = tf.sqrt(inner) if self.norm == 'l2': pass elif self.norm == 'frob' or self.norm == 'Frob': inner /= tf.norm(s) # inner = tf.Print(inner, [inner], message='power inner', summarize=65) inner = tf.matrix_diag(inner) tf_pow = tf.matmul( u, tf.matmul(inner, tf.transpose(u, [0, 2, 1]))) return tf_pow else: raise RuntimeError( "PowTransform layer should be built before using")
def tf_smooth_eig_vec(self): """Function that returns smoothed version of min eigen vector.""" _, matrix_m = self.dual_object.get_full_psd_matrix() # Easier to think in terms of max so negating the matrix [eig_vals, eig_vectors] = tf.self_adjoint_eig(-matrix_m) exp_eig_vals = tf.exp(tf.divide(eig_vals, self.smooth_placeholder)) scaling_factor = tf.reduce_sum(exp_eig_vals) # Multiplying each eig vector by exponential of corresponding eig value # Scaling factor normalizes the vector to be unit norm eig_vec_smooth = tf.divide( tf.matmul(eig_vectors, tf.diag(tf.sqrt(exp_eig_vals))), tf.sqrt(scaling_factor)) return tf.reshape(tf.reduce_sum(eig_vec_smooth, axis=1), shape=[eig_vec_smooth.shape[0].value, 1])
def _covariance_final_ops(sum_squares, total): # http://www.johnloomis.org/ece563/notes/covar/covar.html total = total - tf.constant(1.0) total_3x3 = tf.reshape(tf.tile(tf.expand_dims(total, 0), [9]), [3, 3]) covariance = tf.div(sum_squares, total_3x3) variance = tf.gather(tf.reshape(covariance, [-1]), [0, 4, 8]) # eigenvalues and eigenvectors for PCA eigens = tf.self_adjoint_eig(covariance) eigenvalues = tf.slice(eigens, [0, 0], [-1, 1]) eigenvectors = tf.slice(eigens, [1, 0], [-1, -1]) return tf.sqrt(variance), eigenvalues, eigenvectors
def update(self, block): input_factor = block._input_factor output_factor = block._output_factor pi = _compute_pi_tracenorm(input_factor.get_cov(), output_factor.get_cov()) coeff = self._coeff / block._renorm_coeff coeff = coeff**0.5 damping = coeff / (self._eta**0.5) ue, uv = tf.self_adjoint_eig(input_factor.get_cov() / pi + damping * tf.eye(self._u_c.shape.as_list()[0])) ve, vv = tf.self_adjoint_eig(output_factor.get_cov() * pi + damping * tf.eye(self._v_c.shape.as_list()[0])) ue = coeff / tf.maximum(ue, damping) new_uc = uv * ue**0.5 ve = coeff / tf.maximum(ve, damping) new_vc = vv * ve**0.5 updates_op = [self._u_c.assign(new_uc), self._v_c.assign(new_vc)] return tf.group(*updates_op)
def _compute_normal_eig(self, pts): """ Compute normal given per pixel local point clouds by eigen decompisition. Args: pts: NHWK3 tensorf of per pixel local point clouds. Returns: NHW3 tensor of per pixel normal. """ cov = self._compute_covariance(pts) _, v = tf.self_adjoint_eig(cov) # v has shape NHW33, we need to take the first slice (index 0) of the 4th dimension (the one behind W) # which corresponds to the least eigen value n = v[:, :, :, 0, :] n = tf.nn.lrn(n, depth_radius=3, bias=1e-4) return n
def create_loss_svd(f_out, g_out, clip_min=np.float32(-10000), clip_max=np.float32(10000)): """ Create the loss function that will be minimized by the fg-net. Many options exist. The implementation below uses the 1-Schatten norm from the derivation. It might slow. Inputs: f_out and g_out, which are tensors of the same shape produced by the outut of the f and g nets. Assumes that they are of the form (#batches, #output). Outputs: returns objective """ # number of samples in batch nBatch = tf.cast(tf.shape(f_out)[0], tf.float32) # we clip f to avoid runaway arguments f_clip = tf.clip_by_value(f_out, clip_min, clip_max) #f_clip = f_out # create correlation matrices corrF = tf.matmul(tf.transpose(f_clip), f_clip) / nBatch corrFG = tf.matmul(tf.transpose(f_clip), g_out) / nBatch # Second moment of g sqG = tf.reduce_sum(tf.reduce_mean(tf.square(g_out), 0)) # compute svd in objective n = tf.shape(corrF)[0] #correction term epsilon = 1e-4 invCorrF = tf.matrix_inverse(corrF + epsilon * tf.eye(n), adjoint=True) #check prodGiFG = tf.matmul(tf.matmul(tf.transpose(corrFG), invCorrF), corrFG) s, v = tf.self_adjoint_eig(prodGiFG) #s,u,v = tf.svd(prodGiFG) schatNorm = tf.reduce_sum(tf.sqrt(tf.abs(s))) # define objective objective = sqG - 2 * schatNorm #+ tf.trace(corrF)#.3*tf.reduce_sum(tf.square((corrF-tf.eye(n)))) #return objective return objective, schatNorm
def giff_evals_evecs(S_b, S_w, n_rdims): evals_b, evecs_b = tf.self_adjoint_eig(S_b) evalsRest_b = tf.boolean_mask(evals_b, tf.greater(evals_b, eigenThreshold)) evecsRest_b = evecs_b[:, -tf.shape(evalsRest_b)[0]:] evals_bh = tf.diag(tf.sqrt(evalsRest_b)) S_bh = tf.matmul(tf.matmul(evecsRest_b, evals_bh), evecsRest_b, transpose_b=True) # S_w += tf.eye(tf.shape(S_w)[0], tf.shape(S_w)[1])*1e-4 S_wi = tf.matrix_inverse(S_w) S_s = tf.matmul(tf.matmul(S_bh, S_wi), S_bh) + tf.eye(tf.shape(S_b)[0], tf.shape(S_b)[1]) * 1e-5 evals, evecs = tf.self_adjoint_eig(S_s) evalsRest_t = tf.boolean_mask(evals, tf.greater(evals, eigenThreshold)) evalsRest_r = evals[-n_rdims:] evalsRest = tf.cond(tf.greater(tf.shape(evalsRest_t)[0], n_rdims), lambda: evalsRest_r, lambda: evalsRest_t) evecsRest = evecs[:, -tf.shape(evalsRest)[0]:] return evalsRest, evecsRest
def build_PCA(): #Hyperparameter D = 3 #parameter x_PCA = tf.placeholder(tf.float32, [None, D], name="x_PCA") #mean x_mean_PCA = tf.reduce_mean(x_PCA, 0) #variance S_PCA = compute_outer_product(x_PCA - x_mean_PCA) #compute outer product eigen_val, eigen_vectors = tf.self_adjoint_eig(S_PCA, name="eigenvect") #u_PCA max_index = tf.argmax(eigen_val, 0) u_PCA = tf.gather(eigen_vectors, max_index) return x_PCA, S_PCA, eigen_val, eigen_vectors, u_PCA
def _make_positive_semidefinite_tf(convariance_mat,n=3): e,v = tf.self_adjoint_eig(convariance_mat) min_eig = e[0] # e,v = tf.self_adjoint_eig(H) # e_pos = tf.maximum(0.0,e)+1e-6 #make sure positive definite # e_sqrt = tf.diag(tf.sqrt(e_pos)) # sq_H = tf.matmul(v,tf.matmul(e_sqrt,tf.transpose(v))) def f1(): print('need to make the covariance matrix SPD, min_eig =',min_eig) return convariance_mat-10*min_eig * tf.eye(n) def f2(): return tf.reshape(tf.stack(convariance_mat),[n,n]) convariance_mat = tf.cond(tf.less(min_eig,tf.constant(0.0)), f1, f2) return convariance_mat
def group_decorrelation(cov_matrix, group_index): ''' For a given covariance matrix deducted for a vector_length random vector and a group index like [... i-th, j-th, k-th ...], return a transformation that the diagonal part is ones and makes the [... i-th, j-th, k-th ...] no related with each others and random variables no in the list. --------------------------------- | trans * cov_matrix * trans' | _________________________________ :param cov_matrix: A tensor, sized [vector_length, vector_length], float :param group_index: A tensor, sized [num, 1], int :return: A tensor, sized [vector_length, vector_length], float ''' vector_length = int(cov_matrix.get_shape().as_list()[0]) num = group_index.get_shape().as_list()[0] index_proto = [] for i in range(num): index_proto.append(i) index_proto = tf.Variable(index_proto) index_proto = tf.reshape(index_proto, [num, 1]) sparse_index = tf.concat([index_proto, group_index], axis=1) take_matrix = tf.sparse_to_dense(sparse_index, [num, vector_length], 1.0) c_inv = tf.matrix_inverse(cov_matrix) c_sharp = tf.transpose(tf.matmul(tf.matmul(take_matrix, c_inv), tf.transpose(take_matrix))) _, m = tf.self_adjoint_eig(c_sharp) cm = tf.matmul(c_sharp, m) cm_diag = tf.diag_part(cm) cm_diag = tf.reshape(cm_diag, [num]) m_ = tf.transpose(tf.divide(m, cm_diag)) m_large = tf.matmul(tf.matmul(tf.transpose(take_matrix), m_), take_matrix) trans = tf.matmul(m_large, c_inv) one_diag = tf.diag_part(trans) all_one = tf.ones([vector_length], tf.float32) inv_diag = tf.subtract(all_one, one_diag) com_matrix = tf.diag(inv_diag) trans = tf.add(trans, com_matrix) return trans
def post_process_and_get_tensor_list(self): with tf.variable_scope('tear_apart'): if len(self.return_list) == 0: if self.built_method == 'MEAN': print('Get the tensor list of gradients\' means.') for acls in self.acls_list: self.return_list.append(acls.variable_avg) elif self.built_method == 'VAR': print('Get the tensor list of vv, where vv*vv\' = E(GG\')') for acls in self.acls_list: e, v = tf.self_adjoint_eig(acls.variable_avg) vv = tf.multiply(v, tf.sqrt(tf.maximum(e, 0))) # vv*vv' = acls.variable_avg self.return_list.append(tf.transpose(vv)) return self.return_list
def quat_avg(quats, weights=None): ''' average :param quats: ... x K x 4 :param dim: :param weights: x K :return: ''' if weights is not None: cov = tf.matmul(tf.matmul(quats, tf.linalg.diag(weights), transpose_a=True), quats) else: cov = tf.matmul(quats, quats, transpose_a=True) _, v = tf.self_adjoint_eig(cov) return v[..., -1]
def _build_graph(self, n_cells, time_window, lr, lam_l1, beta): """Build tensorflow graph. Args : n_cells : number of cells (int) time_window : number of time bins in each response vector (int) lr : learning rate (float) lam_l1 : regularization on entires of A (float) """ # declare variables self.dim = n_cells * time_window self.lam_l1 = lam_l1 self.beta = beta self.A_symm = tf.Variable(0.001 * np.eye(self.dim).astype(np.float32), name='A_symm') # initialize Aij to be zero # placeholders for anchor, pos and neg self.anchor = tf.placeholder(dtype=tf.float32, shape=[None, n_cells, time_window], name='anchor') self.pos = tf.placeholder(dtype=tf.float32, shape=[None, n_cells, time_window], name='pos') self.neg = tf.placeholder(dtype=tf.float32, shape=[None, n_cells, time_window], name='neg') self.score_anchor_pos = self.get_score(self.anchor, self.pos) self.score_anchor_neg = self.get_score_all_pairs(self.anchor, self.neg) self.get_loss() self.train_step = tf.train.AdagradOptimizer(lr).minimize(self.loss) # do projection to A_symm symmetric project_A_symm = tf.assign( self.A_symm, (self.A_symm + tf.transpose(self.A_symm)) / 2) # remove negative eigen vectors. with tf.control_dependencies([project_A_symm]): e, v = tf.self_adjoint_eig(self.A_symm) e_pos = tf.nn.relu(e) A_symm_new = tf.matmul(tf.matmul(v, tf.diag(e_pos)), tf.transpose(v)) project_A_PSD = tf.assign(self.A_symm, A_symm_new) # combine all projection operators into one. self.project_A = tf.group(project_A_symm, project_A_PSD)
def train_PCA(self): m, w, h, k = self.trainX.shape data = np.reshape(self.trainX, (m, w * h * k)).T matrix = tf.placeholder(tf.float32, shape=data.shape) mean_var = tf.reduce_mean(matrix, 1) cov = tf.subtract(matrix, tf.reshape(mean_var, (w * h * k, 1))) cov_var = tf.divide(tf.matmul(cov, tf.transpose(cov)), tf.constant(m - 1, dtype=tf.float32)) e, v = tf.self_adjoint_eig(cov_var) mean, eig_values, eig_vectors = self.sess.run((mean_var, e, v), feed_dict={matrix: data}) eig_values = np.maximum(eig_values, 0) a = np.argsort(eig_values).tolist() a.reverse() self.project_matrix = eig_vectors[:, a] self.mu = np.reshape(mean, (w * h * k, 1))
def NUS(W_root, A): ''' input the matrix A and matrix W_root A is the one to be compute W_root is the square-root of W which will make W positive ''' S, U = tf.self_adjoint_eig(A) W = tf.multiply(W_root, W_root) Sigma = tf.matrix_diag(S) NUSresult = tf.matmul(U, W) # U * W NUSresult = tf.matmul(NUSresult, Sigma) # U * W * Sigma NUSresult = tf.matmul(NUSresult, U, transpose_b=True) # U * W * Sigma * U.T return NUSresult
def body(fock_matrix, density, delta, energy): energy = self.nuclear_repulsion fock_matrix = self.update_fock_matrix(density) fock_prime = self.overlap_rsqrt_t @ ( fock_matrix @ self.overlap_rsqrt) _, c_prime = tf.self_adjoint_eig(fock_prime) coefficients = self.overlap_rsqrt @ c_prime energy = energy + 0.5 * tf.reduce_sum(density * (self.h_core + fock_matrix)) density, old_density = self.update_density(density, coefficients) density = self.damping_factor * density + ( 1 - self.damping_factor) * old_density delta = self.get_density_change(density, old_density) return [fock_matrix, density, delta, energy]
def main(_): env = gym.make('Pong-v0') observs = [env.reset()] for _ in range(100): observs.append(env.step(env.action_space.sample())[0]) observs = np.stack(observs) observs = observs.astype(np.float32) observs = tf.image.resize_images(observs, [80, 80]) observs = tf.reduce_mean(observs, 3) # observs = tf.reduce_mean(observs, 0) print(observs.shape) eig_observs = tf.self_adjoint_eig(observs) print(eig_observs)
def fewer_observations(): ''' Solve using Sherman-Morrison-Woodbury; also, compute eigendecomposition for use when sampling \theta. ''' # Feature-wise inner product ZZt = tf.matmul(Z, Zt) ZZt_nz = ZZt + sigma2 * tf.eye(tf.shape(Z)[0], dtype=Z.dtype) if (Y is None): #same as Y = [0,...,0] mu = tf.zeros([tf.shape(Z)[0], 1], dtype=Z.dtype) else: chol = tf.cholesky(ZZt_nz) A = tf.matmul(Zt, 1 / sigma2 * Y) beta = tf.cholesky_solve(chol, tf.matmul(Z, A)) mu = A - tf.matmul(Zt, beta) eigen_vals, eigen_vects = tf.self_adjoint_eig(ZZt_nz) return mu, (eigen_vals, eigen_vects)
def prop(prev_out, curr_in): H = self.baseham.tensorflowH(curr_in) # [batch, i, j] if self.taylorord == 'eig': e, v = tf.self_adjoint_eig(H) expe = tf.exp(-1j / self.timesteps * e) next_out = tf.einsum( 'bij,bj->bi', v, expe * tf.einsum('bji,bj->bi', tf.conj(v), prev_out)) return next_out next_out = prev_out for i in range(self.taylorord, 0, -1): next_out = prev_out + (-1j / self.timesteps / i) * tf.einsum( 'bij,bj->bi', H, next_out) if self.normalize: next_out = next_out / tf.norm( next_out, 2, axis=-1, keep_dims=True) return next_out
def FM(A, B, a, n): ''' input matrix A and matrix B and scalar a(lpha) and scalar n n is the size of A (and B) return the form of FM(A,B,a) in the paper A * sqrt (A^-1 * B + (2a-1)/4*(I-A^-1 *B)^2 - (2a-1)/2*(I-A^-1 * B)) ''' AB = tf.matmul(tf.matrix_inverse(A), B) # A^-1 * B IAB = tf.subtract(tf.eye(n), AB) # I - A^-1 * B eta = (2 * a - 1) / 2 # (2a-1)/2 before_root = AB + eta * eta * tf.matmul(IAB, IAB) - eta * ( IAB) # (A^-1 * B + (2a-1)/4*(I-A^-1 *B)^2 - (2a-1)/2*(I-A^-1 * B)) S, U = tf.self_adjoint_eig(before_root) Sigma_root = tf.matrix_diag(tf.sqrt(S)) after_root = tf.matmul( tf.matmul(U, Sigma_root), U, transpose_b=True) # calculate the square root by eig-decomponent return tf.matmul(A, after_root)
def ComputeDPPrincipalProjection(data, projection_dims, sanitizer, eps_delta, sigma): """Compute differentially private projection. Args: data: the input data, each row is a data vector. projection_dims: the projection dimension. sanitizer: the sanitizer used for acheiving privacy. eps_delta: (eps, delta) pair. sigma: if not None, use noise sigma; otherwise compute it using eps_delta pair. Returns: A projection matrix with projection_dims columns. """ eps, delta = eps_delta # Normalize each row. normalized_data = tf.nn.l2_normalize(data, 1) covar = tf.matmul(tf.transpose(normalized_data), normalized_data) saved_shape = tf.shape(covar) num_examples = tf.slice(tf.shape(data), [0], [1]) if eps > 0: # Since the data is already normalized, there is no need to clip # the covariance matrix. assert delta > 0 saned_covar = sanitizer.sanitize(tf.reshape(covar, [1, -1]), eps_delta, sigma=sigma, option=san.ClipOption(1.0, False), num_examples=num_examples) saned_covar = tf.reshape(saned_covar, saved_shape) # Symmetrize saned_covar. This also reduces the noise variance. saned_covar = 0.5 * (saned_covar + tf.transpose(saned_covar)) else: saned_covar = covar # Compute the eigen decomposition of the covariance matrix, and # return the top projection_dims eigen vectors, represented as columns of # the projection matrix. eigvals, eigvecs = tf.self_adjoint_eig(saned_covar) _, topk_indices = tf.nn.top_k(eigvals, projection_dims) topk_indices = tf.reshape(topk_indices, [projection_dims]) # Gather and return the corresponding eigenvectors. return tf.transpose(tf.gather(tf.transpose(eigvecs), topk_indices))
def ComputeDPPrincipalProjection(data, projection_dims, sanitizer, eps_delta, sigma): """Compute differentially private projection. Args: data: the input data, each row is a data vector. projection_dims: the projection dimension. sanitizer: the sanitizer used for acheiving privacy. eps_delta: (eps, delta) pair. sigma: if not None, use noise sigma; otherwise compute it using eps_delta pair. Returns: A projection matrix with projection_dims columns. """ eps, delta = eps_delta # Normalize each row. normalized_data = tf.nn.l2_normalize(data, 1) covar = tf.matmul(tf.transpose(normalized_data), normalized_data) saved_shape = tf.shape(covar) num_examples = tf.slice(tf.shape(data), [0], [1]) if eps > 0: # Since the data is already normalized, there is no need to clip # the covariance matrix. assert delta > 0 saned_covar = sanitizer.sanitize( tf.reshape(covar, [1, -1]), eps_delta, sigma=sigma, option=san.ClipOption(1.0, False), num_examples=num_examples) saned_covar = tf.reshape(saned_covar, saved_shape) # Symmetrize saned_covar. This also reduces the noise variance. saned_covar = 0.5 * (saned_covar + tf.transpose(saned_covar)) else: saned_covar = covar # Compute the eigen decomposition of the covariance matrix, and # return the top projection_dims eigen vectors, represented as columns of # the projection matrix. eigvals, eigvecs = tf.self_adjoint_eig(saned_covar) _, topk_indices = tf.nn.top_k(eigvals, projection_dims) topk_indices = tf.reshape(topk_indices, [projection_dims]) # Gather and return the corresponding eigenvectors. return tf.transpose(tf.gather(tf.transpose(eigvecs), topk_indices))
def testWrongDimensions(self): # The input to self_adjoint_eig should be 2-dimensional tensor. scalar = tf.constant(1.) with self.assertRaises(ValueError): tf.self_adjoint_eig(scalar) vector = tf.constant([1., 2.]) with self.assertRaises(ValueError): tf.self_adjoint_eig(vector) tensor = tf.constant([[[1., 2.], [3., 4.]], [[1., 2.], [3., 4.]]]) with self.assertRaises(ValueError): tf.self_adjoint_eig(tensor) # The input to batch_batch_self_adjoint_eig should be a tensor of # at least rank 2. scalar = tf.constant(1.) with self.assertRaises(ValueError): tf.batch_self_adjoint_eig(scalar) vector = tf.constant([1., 2.]) with self.assertRaises(ValueError): tf.batch_self_adjoint_eig(vector)
def testWrongDimensions(self): tensor3 = tf.constant([1., 2.]) with self.assertRaises(ValueError): tf.self_adjoint_eig(tensor3)
def learn_average_deg_variable(communities = 2, group_size = 10, seed_v=None, projection_dim=2, l_rate=0.00000001, mean=0.3, sd=0.1): """testing to see if the loss will decrease backproping through very simple function""" #now p and q will be generated from a range of X = tf.placeholder(dtype=tf.float64, shape=[communities*group_size, communities*group_size]) B = tf.cast(X, dtype = tf.float64) Diag = tf.diag(tf.reduce_sum(B,0)) Diag = tf.cast(Diag, tf.float64) #by symmetry I should make this a bit more constrained. so v = tf.Variable(tf.random_normal(shape=[communities*group_size,1], mean=mean, stddev=sd, dtype=tf.float64, seed=seed_v, name=None)) degree = tf.cast(communities*group_size, dtype=tf.float64) r_param = tf.div(tf.cast(1.0, dtype=tf.float64), degree)*tf.matmul(tf.transpose(v), tf.matmul(Diag, v)) BH = (tf.square(r_param)-1)*tf.diag(tf.ones(shape=[communities*group_size], dtype=tf.float64))-tf.mul(r_param, B)+Diag with tf.Session() as sess: g = tf.get_default_graph() with g.gradient_override_map({'SelfAdjointEigV2': 'grassman_with_2d'}): eigenval, eigenvec = tf.self_adjoint_eig(BH) #we try to do svm in this subspace #or we can project it down to 1 dimensions, do the clustering there via some threshold and check if it makes sense #by computing the loss, if it is too big, we change the angle we project down to... eigenvec_proj = tf.slice(eigenvec, [0,0], [communities*group_size, projection_dim]) true_assignment_a = tf.concat(0, [-1*tf.ones([group_size], dtype=tf.float64), tf.ones([group_size], dtype=tf.float64)]) true_assignment_b = -1*true_assignment_a true_assignment_a = tf.expand_dims(true_assignment_a, 1) true_assignment_b = tf.expand_dims(true_assignment_b, 1) projected_a = tf.matmul(tf.matmul(eigenvec_proj, tf.transpose(eigenvec_proj)), true_assignment_a)#tf.transpose(true_assignment_a)) projected_b = tf.matmul(tf.matmul(eigenvec_proj, tf.transpose(eigenvec_proj)), true_assignment_b)#tf.transpose(true_assignment_b)) loss = tf.minimum(tf.reduce_sum(tf.square(tf.sub(projected_a, true_assignment_a))), tf.reduce_sum(tf.square(tf.sub(projected_b, true_assignment_b)))) optimizer = tf.train.AdamOptimizer(l_rate) train = optimizer.minimize(loss, var_list=[v]) eigenvec_grad = tf.gradients(eigenvec, v) loss_grad = tf.gradients(loss, v) #using Laplacian Laplacian = Diag-B eigenval_baseline, eigenvec_baseline = tf.self_adjoint_eig(Laplacian) project_baseline = tf.slice(eigenvec_baseline, [0,0], [communities*group_size, projection_dim]) projected_a_baseline = tf.matmul(tf.matmul(project_baseline, tf.transpose(project_baseline)), true_assignment_a)#tf.transpose(true_assignment_a)) projected_b_baseline = tf.matmul(tf.matmul(project_baseline, tf.transpose(project_baseline)), true_assignment_b) loss_baseline = tf.minimum(tf.reduce_sum(tf.square(tf.sub(projected_a_baseline, true_assignment_a))), tf.reduce_sum(tf.square(tf.sub(projected_b_baseline, true_assignment_b)))) r_op, target = target_subspace(adj=B, groupsize=group_size, communities=communities, diag=Diag, dim_proj=projection_dim) r_diff = (r_op-r_param) #difference between r_op and r_param is how close we are to the average degree init = tf.initialize_all_variables() sess.run(init) V, R_param, R_diff, avg_deg, Loss_baseline= sess.run([v, r_param, r_diff, r_op, loss_baseline], feed_dict={X:data[0]}) # V_lst = np.zeros(len(data), np.float64) V_lst = [0.0]*len(data) R_param_lst = np.zeros(len(data), np.float64) R_diff_list = np.zeros(len(data), np.float64) loss_list = [0.0]*len(data) true_assignment_lst = np.zeros(len(data), np.float64) avg_deg_lst = np.zeros(len(data), np.float64) # data_lst = [] r_param_grad_lst = [0.0]*len(data) loss_baseline = np.zeros(len(data), np.float64) average_loss = np.zeros(len(data), np.float64) V_lst[0]=V R_param_lst[0]=R_param R_diff_list[0]=R_diff loss_list[0]=None true_assignment_lst[0]=None avg_deg_lst[0]=avg_deg # data_lst.append(None) r_param_grad_lst[0]=None loss_baseline[0]=Loss_baseline average_loss[0]=None # print "initial v: {}. r_param: {}, difference between r_param and sqrt average deg {}.".format(a, r, b) for i in range(1, len(data)): try: sess.run(train, feed_dict={X:data[i]}) #if i%print_ratio==0: #print i #try: if i%2==0: V, R_param, R_diff, Loss, Loss_baseline, r_param_grad, Tru_assign, avg_deg = sess.run([v, r_param, r_diff, loss, loss_baseline, tf.gradients(loss, r_param), tf.transpose(projected_a), r_op], feed_dict={X:data[i]}) #for j in range(len(data)): # loss_tmp = np.zeros(len(data), np.float64) # tmp = sess.run(loss, feed_dict={X:data[j]}) # loss_tmp[j](tmp) #np.mean(loss_tmp) #average_loss = np.mean() V_lst[j]=V R_param_lst[j]=R_param R_diff_list[j]=R_diff loss_list[j]=Loss loss_baseline[j]=Loss_baseline true_assignment_lst[j]=Tru_assign r_param_grad_lst[j]=r_param_grad avg_deg_lst[j]=avg_deg #data_lst.append(data[i]) print "step {}: Loss:{}, Loss_baseline: {}, avg_deg: {}".format(j, Loss, Loss_baseline, avg_deg) except: V, R_param, R_diff, Loss, Loss_baseline, r_param_grad, Tru_assign, avg_deg = None, None, None, None, None, None, None, None V_lst[i]=V R_param_lst[i]=R_param R_diff_list[i]=R_diff loss_list[i]=Loss loss_baseline[i]=Loss_baseline true_assignment_lst[i]=Tru_assign r_param_grad_lst[i]=r_param_grad avg_deg_lst[i]=avg_deg #data_lst.append(data[i]) print "step:{} not sucessful".format(i) pass d = {"v": V_lst, "r_param": R_param_lst, "r_diff": R_diff_list, "loss": loss_list, "loss_baseline": loss_baseline, "R_grad_list": r_param_grad_lst, "true_assignmnet": true_assignment_lst, "avg_deg": avg_deg_lst}#,"data":data_lst } d = pd.DataFrame(d) easy_size = len(data) #d.to_csv("mean{}l_rate{}data_size{}p_min{}p_max{}hard_ratio{}.csv".format(mean, l_rate, easy_size+hard_size, p_min, p_max, hard_ratio)) return d
# 矩阵转置 print(sess.run(tf.transpose(matrix_C))) # 重新初始化会得到不同的值, # 所以会与之前的matrix_C不是转置矩阵 print('- - - - - - - - - - - - - - - - - - - -') # 矩阵行列式 print(sess.run(tf.matrix_determinant(matrix_D))) print('- - - - - - - - - - - - - - - - - - - -') # 矩阵的逆矩阵 print(sess.run(tf.matrix_inverse(matrix_D))) # 矩阵的逆矩阵是用平方根法, # 需要矩阵为对称正定矩阵或者可进行LU分解 print('- - - - - - - - - - - - - - - - - - - -') # 矩阵分解法 print(sess.run(tf.cholesky(indentity_matrix))) print('- - - - - - - - - - - - - - - - - - - -') # 求矩阵特征值和特征向量 print(sess.run(tf.self_adjoint_eig(matrix_D))) # 第一行 为特征值, # 其余的向量是对应的特征向量 # 此方法又称矩阵的特征分解
def testNonSquareMatrix(self): with self.assertRaises(ValueError): tf.self_adjoint_eig(tf.constant(np.array([[1., 2., 3.], [3., 4., 5.]])))
input2_tensor = tf.Variable(tf.to_int32(input2_value)) # input2_tensor = tf.Variable(input2_value) dot_tensor = tf.matmul(input_tensor, input2_tensor) with tf.Session() as sess: sess.run(tf.initialize_all_variables()) dot_tensorflow = sess.run(dot_tensor) dot_tensorflow.must_close(dot_numpy) with test("numpy.linalg.eig"): input_value = numpy.diag((1, 2, 3)) eigenvalues, eigenvectors = numpy.linalg.eig(input_value) eigenvalues.must_close([1,2,3]) eigenvectors.must_close(numpy.eye(3)) input_tensor = tf.Variable(tf.cast(input_value,tf.float32)) eigen_tensor = tf.self_adjoint_eig(input_tensor) with tf.Session() as sess: sess.run(tf.initialize_all_variables()) eigen_tensorflow = sess.run(eigen_tensor) eigen_tensorflow[0,].must_close(eigenvalues) eigen_tensorflow[1:4,].must_close(eigenvectors) with test("add, same as matrix"): input_value = numpy.diag((1, 2, 3)) add_one = input_value + 3 add_one.must_close([ [4, 3, 3], [3, 5, 3], [3, 3, 6]
# initialize and shape with tf.Session() as sess: sess.run(init) sess.run(mps) for i in range(L): print mps[i].get_shape() lenv = [0]*L renv = [0]*L # Also, deriable mat = tf.reshape(mps[0],[n,D]) lenv[0] = tf.matmul(mat,mat,transpose_a=True) # A Tensor. Has the same type as input. Shape is [M+1, M]. # the first row is eigenvalues, columns of other part are eignvectors. res = tf.self_adjoint_eig(lenv[0]) import scipy.linalg l0 = numpy.einsum('pi,pj->ij',mps0[0],mps0[0]) e,v = scipy.linalg.eigh(l0) print e print v flt = tf.reshape(mps[0],[-1]) print 'shp',flt.get_shape() tr0 = tf.reduce_sum(tf.diag_part(lenv[0])) g0 = tf.gradients(tr0,mps[0]) tr1 = tf.reduce_sum(tf.mul(mps[0],mps[0])) g1 = tf.gradients(tr1,mps[0])
# 3x2 random uniform matrix C = tf.random_uniform([3,2]) print(sess.run(C)) print(sess.run(C)) # Note that we are reinitializing, hence the new random variabels # Create matrix from np array D = tf.convert_to_tensor(np.array([[1., 2., 3.], [-3., -7., -1.], [0., 5., -2.]])) print(sess.run(D)) # Matrix addition/subtraction print(sess.run(A+B)) print(sess.run(B-B)) # Matrix Multiplication print(sess.run(tf.matmul(B, identity_matrix))) # Matrix Transpose print(sess.run(tf.transpose(C))) # Again, new random variables # Matrix Determinant print(sess.run(tf.matrix_determinant(D))) # Matrix Inverse print(sess.run(tf.matrix_inverse(D))) # Cholesky Decomposition print(sess.run(tf.cholesky(identity_matrix))) # Eigenvalues and Eigenvectors print(sess.run(tf.self_adjoint_eig(D)))
def test_svm_cluster(communities = 2, group_size = 10, seed=1, seed_r=1, p=0.8, q=0.05, name='test1', projection_dim=2, iterations=100, print_ratio=10, l_rate=0.1, mean=2.0, sd=0.4): """testing to see if the loss will decrease backproping through very simple function""" B = np.asarray(balanced_stochastic_blockmodel(communities, group_size, p, q, seed)).astype(np.double) B = tf.cast(B, dtype = tf.float64) Diag = tf.diag(tf.reduce_sum(B,0)) Diag = tf.cast(Diag, tf.float64) r = tf.Variable(tf.random_normal(shape=[1], mean=mean, stddev=sd, dtype=tf.float64, seed=seed_r, name=None)) BH = (tf.square(r)-1)*tf.diag(tf.ones(shape=[communities*group_size], dtype=tf.float64))-tf.mul(r, B)+Diag with tf.Session() as sess: g = tf.get_default_graph() with g.gradient_override_map({'SelfAdjointEigV2': name}): eigenval, eigenvec = tf.self_adjoint_eig(BH) #we try to do svm in this subspace #or we can project it down to 1 dimensions, do the clustering there via some threshold and check if it makes sense #by computing the loss, if it is too big, we change the angle we project down to... eigenvec_proj = tf.slice(eigenvec, [0,0], [communities*group_size, projection_dim]) true_assignment_a = tf.concat(0, [-1*tf.ones([group_size], dtype=tf.float64), tf.ones([group_size], dtype=tf.float64)]) true_assignment_b = -1*true_assignment_a true_assignment_a = tf.expand_dims(true_assignment_a, 1) true_assignment_b = tf.expand_dims(true_assignment_b, 1) projected_a = tf.matmul(tf.matmul(eigenvec_proj, tf.transpose(eigenvec_proj)), true_assignment_a)#tf.transpose(true_assignment_a)) projected_b = tf.matmul(tf.matmul(eigenvec_proj, tf.transpose(eigenvec_proj)), true_assignment_b)#tf.transpose(true_assignment_b)) loss = tf.minimum(tf.reduce_sum(tf.square(tf.sub(projected_a, true_assignment_a))), tf.reduce_sum(tf.square(tf.sub(projected_b, true_assignment_b)))) optimizer = tf.train.GradientDescentOptimizer(l_rate) train = optimizer.minimize(loss, var_list=[r]) eigenvec_grad = tf.gradients(eigenvec, r) loss_grad = tf.gradients(loss, r) r_op, target = target_subspace(adj=B, groupsize=group_size, communities=communities, diag=Diag, dim_proj=projection_dim) r_op_projection_a = tf.matmul(tf.matmul(target, tf.transpose(target)), true_assignment_a) r_op_projection_b = tf.matmul(tf.matmul(target, tf.transpose(target)), true_assignment_b) r_op_loss = tf.minimum(tf.reduce_sum(tf.square(tf.sub(r_op_projection_a, true_assignment_a))), tf.reduce_sum(tf.square(tf.sub(r_op_projection_b, true_assignment_b)))) init = tf.initialize_all_variables() sess.run(init) a,b,c,d= sess.run([r, r_op, r_op_loss, tf.transpose(r_op_projection_a)]) a_lst = [] b_lst = [] c_lst = [] d_lst = [] a_lst.append(a) b_lst.append(b) c_lst.append(c) d_lst.append(d) print "initial r: {}. r_op = sqrt(average degree) : {} . Loss associated with r_op: {}. r_op assignments {}.".format(a, b, c, d) for i in range(iterations): try: sess.run(train) except: pass if i%print_ratio==0: #print i try: a,b,c,d = sess.run([r, loss, tf.gradients(loss, r), tf.transpose(projected_a)]) a_lst.append(a) b_lst.append(b) c_lst.append(c) d_lst.append(d) except: a,b,c,d = 0, 0, 0, 0 a_lst.append(a) b_lst.append(b) c_lst.append(c) d_lst.append(d) #print "current r: {}, current loss: {}, gradient of loss/r is {} and current assignments (up to sign) {}.".format(a,b,c,d) d = {"r_value": a_lst, "loss": b_lst, "gradient_loss_r": c_lst, "projection": d_lst} d = pd.DataFrame(d) d.to_csv("/Users/xiangli/Desktop/clusternet/plot_data/r{}rate{}p{}q{}iterations{}step{}.csv".format(mean, l_rate, p, q, iterations, print_ratio)) return d
A = tf.truncated_normal([2, 3]) print(sess.run(A)) # 2x3 constant matrix: B = tf.fill([2, 3], 5.0) print(sess.run(B)) # 3x2 random uniform matrix: C = tf.random_uniform([3, 2]) print(sess.run(C)) # Create matrix from np array: D = tf.convert_to_tensor(np.array([[1., 2., 3.], [-3., -7., -1.], [0., 5., -2.]])) print(sess.run(D)) # Matrix Operations Matrix addition/subtraction: print(sess.run(A + B)) print(sess.run(B - B)) # Matrix Multiplication: print(sess.run(tf.matmul(B, identity_matrix))) # Matrix Transpose: print(sess.run(tf.transpose(C))) # Matrix Determinant: print(sess.run(tf.matrix_determinant(D))) # Matrix Inverse: print(sess.run(tf.matrix_inverse(D))) # Cholesky Decomposition: print(sess.run(tf.cholesky(identity_matrix))) # Eigenvalues and Eigenvectors: We use tf.self_adjoint_eig() function, which returns two objects, # first one is an array of eigenvalues, the second is a matrix of the eigenvectors. eigenvalues, eigenvectors = sess.run(tf.self_adjoint_eig(D)) print(eigenvalues) print(eigenvectors)
def learn_average_deg_variable(communities = 2, group_size = 10, seed_v=None, projection_dim=2, l_rate=0.00000001, mean=0.3, sd=0.1): """testing to see if the loss will decrease backproping through very simple function""" #now p and q will be generated from a range of X = tf.placeholder(dtype=tf.float64, shape=[communities*group_size, communities*group_size]) B = tf.cast(X, dtype = tf.float64) Diag = tf.diag(tf.reduce_sum(B,0)) Diag = tf.cast(Diag, tf.float64) #by symmetry I should make this a bit more constrained. so v = tf.Variable(tf.random_normal(shape=[communities*group_size,1], mean=mean, stddev=sd, dtype=tf.float64, seed=seed_v, name=None)) degree = tf.cast(communities*group_size, dtype=tf.float64) r_param = tf.div(tf.cast(1.0, dtype=tf.float64), degree)*tf.matmul(tf.transpose(v), tf.matmul(Diag, v)) BH = (tf.square(r_param)-1)*tf.diag(tf.ones(shape=[communities*group_size], dtype=tf.float64))-tf.mul(r_param, B)+Diag with tf.Session() as sess: g = tf.get_default_graph() with g.gradient_override_map({'SelfAdjointEigV2': 'grassman_with_2d'}): eigenval, eigenvec = tf.self_adjoint_eig(BH) #we try to do svm in this subspace #or we can project it down to 1 dimensions, do the clustering there via some threshold and check if it makes sense #by computing the loss, if it is too big, we change the angle we project down to... eigenvec_proj = tf.slice(eigenvec, [0,0], [communities*group_size, projection_dim]) true_assignment_a = tf.concat(0, [-1*tf.ones([group_size], dtype=tf.float64), tf.ones([group_size], dtype=tf.float64)]) true_assignment_b = -1*true_assignment_a true_assignment_a = tf.expand_dims(true_assignment_a, 1) true_assignment_b = tf.expand_dims(true_assignment_b, 1) projected_a = tf.matmul(tf.matmul(eigenvec_proj, tf.transpose(eigenvec_proj)), true_assignment_a)#tf.transpose(true_assignment_a)) projected_b = tf.matmul(tf.matmul(eigenvec_proj, tf.transpose(eigenvec_proj)), true_assignment_b)#tf.transpose(true_assignment_b)) loss = tf.minimum(tf.reduce_sum(tf.square(tf.sub(projected_a, true_assignment_a))), tf.reduce_sum(tf.square(tf.sub(projected_b, true_assignment_b)))) optimizer = tf.train.AdamOptimizer(l_rate) train = optimizer.minimize(loss, var_list=[v]) eigenvec_grad = tf.gradients(eigenvec, v) loss_grad = tf.gradients(loss, v) r_op, target = target_subspace(adj=B, groupsize=group_size, communities=communities, diag=Diag, dim_proj=projection_dim) r_diff = (r_op-r_param) #difference between r_op and r_param is how close we are to the average degree init = tf.initialize_all_variables() sess.run(init) a,r, b, avg_deg= sess.run([v, r_param, r_diff, r_op], feed_dict={X:data[0]}) a_lst = [] r_lst = [] r_diff_list = [] b_lst = [] c_lst = [] d_lst = [] avg_deg_lst = [] data_lst = [] r_param_grad_lst = [] a_lst.append(a) r_lst.append(r) r_diff_list.append(b) b_lst.append(None) c_lst.append(None) d_lst.append(None) avg_deg_lst.append(avg_deg) data_lst.append(None) r_param_grad_lst.append(None) print "initial v: {}. r_param: {}, difference between r_param and sqrt average deg {}.".format(a, r, b) for i in range(len(data)): try: sess.run(train, feed_dict={X:data[i]}) #if i%print_ratio==0: #print i #try: a,r, k, b,c,d, r_param_grad, avg_deg = sess.run([v, r_param, r_diff, loss, tf.gradients(loss, v), tf.transpose(projected_a), tf.gradients(loss, r_param), r_op], feed_dict={X:data[i]}) a_lst.append(a) r_lst.append(r) r_diff_list.append(k) b_lst.append(b) c_lst.append(c) d_lst.append(d) r_param_grad_lst.append(r_param_grad) avg_deg_lst.append(avg_deg) data_lst.append(data[i]) print "step: {}: loss: {}, avg_deg: {} r_param:{}, r_diff: {}, r_param gradient: {}".format(i, b, avg_deg, r, k, r_param_grad) except: a,r, k, b,c,d, r_param_grad, avg_deg = None, None, None, None, None, None, None, None, None a_lst.append(a) r_lst.append(r) r_diff_list.append(k) b_lst.append(b) c_lst.append(c) d_lst.append(d) r_param_grad_lst.append(r_param_grad) avg_deg_lst.append(avg_deg) data_lst.append(data[i]) print "step:{} not sucessful".format(i) pass d = {"v": a_lst, "r_param": r_lst, "r_diff": r_diff_list, "loss": b_lst, "gradient_loss_v": c_lst, "projection": d_lst, "r_param_grad": r_param_grad_lst, "avg_deg": avg_deg_lst, "data":data_lst } d = pd.DataFrame(d) easy_size = len(data) d.to_csv("/Users/xiangli/Desktop/clusternet/Learning_r_matrix_data/mean{}l_rate{}data_size{}p_min{}p_max{}hard_ratio{}.csv".format(mean, l_rate, easy_size+hard_size, p_min, p_max, hard_ratio))
def computeStatsEigen(self): """ compute the eigen decomp using copied var stats to avoid concurrent read/write from other queue """ # TO-DO: figure out why this op has delays (possibly moving # eigenvectors around?) with tf.device('/cpu:0'): def removeNone(tensor_list): local_list = [] for item in tensor_list: if item is not None: local_list.append(item) return local_list def copyStats(var_list): print("copying stats to buffer tensors before eigen decomp") redundant_stats = {} copied_list = [] for item in var_list: if item is not None: if item not in redundant_stats: if self._use_float64: redundant_stats[item] = tf.cast( tf.identity(item), tf.float64) else: redundant_stats[item] = tf.identity(item) copied_list.append(redundant_stats[item]) else: copied_list.append(None) return copied_list #stats = [copyStats(self.fStats), copyStats(self.bStats)] #stats = [self.fStats, self.bStats] stats_eigen = self.stats_eigen computedEigen = {} eigen_reverse_lookup = {} updateOps = [] # sync copied stats # with tf.control_dependencies(removeNone(stats[0]) + # removeNone(stats[1])): with tf.control_dependencies([]): for stats_var in stats_eigen: if stats_var not in computedEigen: eigens = tf.self_adjoint_eig(stats_var) e = eigens[0] Q = eigens[1] if self._use_float64: e = tf.cast(e, tf.float32) Q = tf.cast(Q, tf.float32) updateOps.append(e) updateOps.append(Q) computedEigen[stats_var] = {'e': e, 'Q': Q} eigen_reverse_lookup[e] = stats_eigen[stats_var]['e'] eigen_reverse_lookup[Q] = stats_eigen[stats_var]['Q'] self.eigen_reverse_lookup = eigen_reverse_lookup self.eigen_update_list = updateOps if KFAC_DEBUG: self.eigen_update_list = [item for item in updateOps] with tf.control_dependencies(updateOps): updateOps.append(tf.Print(tf.constant( 0.), [tf.convert_to_tensor('computed factor eigen')])) return updateOps