def testNotInvertible(self): # The input should be invertible. with self.test_session(): with self.assertRaisesOpError("Input is not invertible."): # All rows of the matrix below add to zero. tensor3 = tf.constant([[1., 0., -1.], [-1., 1., 0.], [0., -1., 1.]]) tf.matrix_inverse(tensor3).eval()
def solve(a, b): if b.ndim == 1: return tf.reshape(tf.matmul(tf.matrix_inverse(a), tf.expand_dims(b, -1)), [-1]) elif b.ndim == 2: return tf.matmul(tf.matrix_inverse(a), b) else: import ipdb; ipdb.set_trace()
def hnet_loss(gt_pts, transformation_coeffcient, name): """ :param gt_pts: 原始的标签点对 [x, y, 1] :param transformation_coeffcient: 映射矩阵参数(6参数矩阵) [[a, b, c], [0, d, e], [0, f, 1]] :param name: :return: """ with tf.variable_scope(name): # 首先映射原始标签点对 transformation_coeffcient = tf.concat([transformation_coeffcient, [1.0]], axis=-1) H_indices = tf.constant([[0], [1], [2], [4], [5], [7], [8]]) H_shape = tf.constant([9]) H = tf.scatter_nd(H_indices, transformation_coeffcient, H_shape) H = tf.reshape(H, shape=[3, 3]) gt_pts = tf.transpose(gt_pts) pts_projects = tf.matmul(H, gt_pts) # 求解最小二乘二阶多项式拟合参数矩阵 Y = tf.transpose(pts_projects[1, :]) X = tf.transpose(pts_projects[0, :]) Y_One = tf.add(tf.subtract(Y, Y), tf.constant(1.0, tf.float32)) Y_stack = tf.stack([tf.pow(Y, 3), tf.pow(Y, 2), Y, Y_One], axis=1) w = tf.matmul(tf.matmul(tf.matrix_inverse(tf.matmul(tf.transpose(Y_stack), Y_stack)), tf.transpose(Y_stack)), tf.expand_dims(X, -1)) # 利用二阶多项式参数求解拟合位置并反算到原始投影空间计算损失 x_preds = tf.matmul(Y_stack, w) preds = tf.transpose(tf.stack([tf.squeeze(x_preds, -1), Y, Y_One], axis=1)) x_transformation_back = tf.matmul(tf.matrix_inverse(H), preds) loss = tf.reduce_mean(tf.pow(gt_pts[0, :] - x_transformation_back[0, :], 2)) return loss
def hnet_transformation(gt_pts, transformation_coeffcient, name): """ :param gt_pts: :param transformation_coeffcient: :param name: :return: """ with tf.variable_scope(name): # 首先映射原始标签点对 transformation_coeffcient = tf.concat([transformation_coeffcient, [1.0]], axis=-1) H_indices = tf.constant([[0], [1], [2], [4], [5], [7], [8]]) H_shape = tf.constant([9]) H = tf.scatter_nd(H_indices, transformation_coeffcient, H_shape) H = tf.reshape(H, shape=[3, 3]) gt_pts = tf.transpose(gt_pts) pts_projects = tf.matmul(H, gt_pts) # 求解最小二乘二阶多项式拟合参数矩阵 Y = tf.transpose(pts_projects[1, :]) X = tf.transpose(pts_projects[0, :]) Y_One = tf.add(tf.subtract(Y, Y), tf.constant(1.0, tf.float32)) Y_stack = tf.stack([tf.pow(Y, 3), tf.pow(Y, 2), Y, Y_One], axis=1) w = tf.matmul(tf.matmul(tf.matrix_inverse(tf.matmul(tf.transpose(Y_stack), Y_stack)), tf.transpose(Y_stack)), tf.expand_dims(X, -1)) # 利用二阶多项式参数求解拟合位置 x_preds = tf.matmul(Y_stack, w) preds = tf.transpose(tf.stack([tf.squeeze(x_preds, -1), Y, Y_One], axis=1)) preds_fit = tf.stack([tf.squeeze(x_preds, -1), Y], axis=1) x_transformation_back = tf.matmul(tf.matrix_inverse(H), preds) return x_transformation_back
def tf_prod_gauss(mu1, mu2, sig1, sig2): #assumes precision prec1 = tf.matrix_inverse(sig1) prec2 = tf.matrix_inverse(sig2) prec = tf.add(prec1, prec2) sig = tf.matrix_inverse(prec) mu = tf.add( tf.matmul(prec1, mu1), tf.matmul(prec2, mu1) ) return mu, sig
def F2_bound(y,Kmm,Knm,Knn,mu,Sigma): Eye=tf.constant(np.eye(N,N), shape=[N,N],dtype=tf.float32) sigEye=tf.mul(1.0,Eye) print(s.run(tf.matrix_inverse(sigEye))) #sigEye=tf.mul(tf.square(sigma),Eye) Kmn=tf.transpose(Knm) prec=Matrix_Inversion_Lemma(sigEye,Knm,Kmm,Kmn) zeros=tf.constant(np.zeros(N),shape=[N,1],dtype=tf.float32) log_den=log_density(y,zeros,prec) Kmm_inv=tf.matrix_inverse(Kmm) trace_term=tf.trace(Knn-Mul(Knm,Kmm_inv,Kmn))*(0.5) return log_den-trace_term
def inverse(pBatch,opt,name=None): with tf.name_scope("inverse"): pMtrxBatch = vec2mtrxBatch(pBatch,opt) pInvMtrxBatch = tf.matrix_inverse(pMtrxBatch) pInvBatch = mtrx2vecBatch(pInvMtrxBatch,opt) pInvBatch = tf.identity(pInvBatch,name=name) return pInvBatch
def get_log_likelihood(self): I = tf.constant(self.sigx*np.identity(self.dimx)) mu = tf.constant(np.zeros(self.dimx).reshape((self.dimx,1))) ll = tf.constant(0, tf.float64) n_samples = 1 for i in xrange(self.n): x = self.observed[i] x = tf.constant(x, shape = [3,1], dtype = tf.float64) mc = tf.Variable(0, dtype = tf.float64) for j in xrange(n_samples): w = self.sample_theta() var = tf.matmul(w, tf.transpose(w)) var = tf.add(var, I) prec = tf.matrix_inverse(var) pw = util.tf_evaluate_gauss(mu, prec, x) mc+= pw mc = tf.div(mc,float(n_samples) ) mc = tf.log(mc) ll = tf.add(ll, mc) self.llh = ll return ll
def compute_rigid_flow(depth, pose, intrinsics, reverse_pose=False): """Compute the rigid flow from target image plane to source image Args: depth: depth map of the target image [batch, height_t, width_t] pose: target to source (or source to target if reverse_pose=True) camera transformation matrix [batch, 6], in the order of tx, ty, tz, rx, ry, rz; intrinsics: camera intrinsics [batch, 3, 3] Returns: Rigid flow from target image to source image [batch, height_t, width_t, 2] """ batch, height, width = depth.get_shape().as_list() # Convert pose vector to matrix pose = pose_vec2mat(pose) if reverse_pose: pose = tf.matrix_inverse(pose) # Construct pixel grid coordinates pixel_coords = meshgrid(batch, height, width) tgt_pixel_coords = tf.transpose(pixel_coords[:,:2,:,:], [0, 2, 3, 1]) # Convert pixel coordinates to the camera frame cam_coords = pixel2cam(depth, pixel_coords, intrinsics) # Construct a 4x4 intrinsic matrix filler = tf.constant([0.0, 0.0, 0.0, 1.0], shape=[1, 1, 4]) filler = tf.tile(filler, [batch, 1, 1]) intrinsics = tf.concat([intrinsics, tf.zeros([batch, 3, 1])], axis=2) intrinsics = tf.concat([intrinsics, filler], axis=1) # Get a 4x4 transformation matrix from 'target' camera frame to 'source' # pixel frame. proj_tgt_cam_to_src_pixel = tf.matmul(intrinsics, pose) src_pixel_coords = cam2pixel(cam_coords, proj_tgt_cam_to_src_pixel) rigid_flow = src_pixel_coords - tgt_pixel_coords return rigid_flow
def run_training(train_X, train_Y): X = tf.placeholder(tf.float32, [m, n+1]) Y = tf.placeholder(tf.float32, [m, 1]) # add the column of 1s to X ones = np.ones([m, 1], dtype=np.float32) train_X = np.concatenate((ones, train_X), axis=1) # normal equations theta = tf.matmul(tf.matmul(tf.matrix_inverse(tf.matmul(tf.transpose(X), X)), tf.transpose(X)), Y) with tf.Session() as sess: theta = sess.run(theta, feed_dict={X: train_X, Y: train_Y}) print "Predict.... (Predict a house with 1650 square feet and 3 bedrooms.)" # Do not forget to add the column of 1s predict_X = np.array([1, 1650, 3], dtype=np.float32).reshape((1, 3)) predict_Y = tf.matmul(predict_X, theta) print "House price(Y) =", sess.run(predict_Y) print "theta" print theta sess.close()
def logdet_grad(op, grad): a = op.inputs[0] a_adj_inv = tf.check_numerics( tf.matrix_inverse(a, adjoint=True), 'zero determinant') out_shape = tf.concat([tf.shape(a)[:-2], [1, 1]], axis=0) return tf.reshape(grad, out_shape) * a_adj_inv
def initialize(self, *args, **kwargs): # Store latent variables in a temporary attribute; MAP will # optimize `PointMass` random variables, which subsequently # optimizes mean parameters of the normal approximations. latent_vars_normal = self.latent_vars.copy() self.latent_vars = {z: PointMass(params=qz.loc) for z, qz in six.iteritems(latent_vars_normal)} super(Laplace, self).initialize(*args, **kwargs) hessians = tf.hessians(self.loss, list(six.itervalues(self.latent_vars))) self.finalize_ops = [] for z, hessian in zip(six.iterkeys(self.latent_vars), hessians): qz = latent_vars_normal[z] if isinstance(qz, (MultivariateNormalDiag, Normal)): scale_var = get_variables(qz.variance())[0] scale = 1.0 / tf.diag_part(hessian) else: # qz is MultivariateNormalTriL scale_var = get_variables(qz.covariance())[0] scale = tf.matrix_inverse(tf.cholesky(hessian)) self.finalize_ops.append(scale_var.assign(scale)) self.latent_vars = latent_vars_normal.copy() del latent_vars_normal
def _define_distance_to_clusters(self, data): """Defines the Mahalanobis distance to the assigned Gaussian.""" # TODO(xavigonzalvo): reuse (input - mean) * cov^-1 * (input - # mean) from log probability function. self._all_scores = [] for shard in data: all_scores = [] shard = tf.expand_dims(shard, 0) for c in xrange(self._num_classes): if self._covariance_type == FULL_COVARIANCE: cov = self._covs[c, :, :] elif self._covariance_type == DIAG_COVARIANCE: cov = tf.diag(self._covs[c, :]) inverse = tf.matrix_inverse(cov + self._min_var) inv_cov = tf.tile( tf.expand_dims(inverse, 0), tf.pack([self._num_examples, 1, 1])) diff = tf.transpose(shard - self._means[c, :, :], perm=[1, 0, 2]) m_left = tf.batch_matmul(diff, inv_cov) all_scores.append(tf.sqrt(tf.batch_matmul( m_left, tf.transpose(diff, perm=[0, 2, 1]) ))) self._all_scores.append(tf.reshape( tf.concat(1, all_scores), tf.pack([self._num_examples, self._num_classes]))) # Distance to the associated class. self._all_scores = tf.concat(0, self._all_scores) assignments = tf.concat(0, self.assignments()) rows = tf.to_int64(tf.range(0, self._num_examples)) indices = tf.concat(1, [tf.expand_dims(rows, 1), tf.expand_dims(assignments, 1)]) self._scores = tf.gather_nd(self._all_scores, indices)
def update_centers(self, img_dataset): ''' Optimize: self.C = (U * hu^T + V * hv^T) (hu * hu^T + hv * hv^T)^{-1} self.C^T = (hu * hu^T + hv * hv^T)^{-1} (hu * U^T + hv * V^T) but all the C need to be replace with C^T : self.C = (hu * hu^T + hv * hv^T)^{-1} (hu^T * U + hv^T * V) ''' old_C_value = self.sess.run(self.C) h = self.img_b_all U = self.img_output_all smallResidual = tf.constant( np.eye(self.subcenter_num * self.subspace_num, dtype=np.float32) * 0.001) Uh = tf.matmul(tf.transpose(h), U) hh = tf.add(tf.matmul(tf.transpose(h), h), smallResidual) compute_centers = tf.matmul(tf.matrix_inverse(hh), Uh) update_C = self.C.assign(compute_centers) C_value = self.sess.run(update_C, feed_dict={ self.img_output_all: img_dataset.output, self.img_b_all: img_dataset.codes, }) C_sums = np.sum(np.square(C_value), axis=1) C_zeros_ids = np.where(C_sums < 1e-8) C_value[C_zeros_ids, :] = old_C_value[C_zeros_ids, :] self.sess.run(self.C.assign(C_value))
def invertible_1x1_conv(z, logdet, reverse=False, name=None, use_bias=False): with tf.variable_scope(name, "invconv"): shape = z.get_shape().as_list() w_shape = [shape[3], shape[3]] # Sample a random orthogonal matrix: w_init = np.linalg.qr(np.random.randn(*w_shape))[0].astype('float32') w = tf.get_variable("W", dtype=tf.float32, initializer=w_init) det_w = tf.matrix_determinant(tf.cast(w, 'float64')) dlogdet = tf.cast(tf.log(abs(det_w)), 'float32') * shape[1] * shape[2] if use_bias: b = tf.get_variable("bias", [1, 1, 1, shape[3]]) if not reverse: _w = w[tf.newaxis, tf.newaxis, ...] z = tf.nn.conv2d(z, _w, [1, 1, 1, 1], 'SAME', data_format='NHWC') logdet += dlogdet if use_bias: z += b else: if use_bias: z -= b w_inv = tf.matrix_inverse(w) _w = w_inv[tf.newaxis, tf.newaxis, ...] z = tf.nn.conv2d(z, _w, [1, 1, 1, 1], 'SAME', data_format='NHWC') logdet -= dlogdet return z, logdet
def logpdf(self, x, mean=None, cov=1): """Log of the probability density function. Parameters ---------- x : tf.Tensor A 1-D or 2-D tensor. mean : tf.Tensor, optional A 1-D tensor. Defaults to zero mean. cov : tf.Tensor, optional A 1-D or 2-D tensor. Defaults to identity matrix. Returns ------- tf.Tensor A tensor of one dimension less than the input. """ x = tf.cast(x, dtype=tf.float32) x_shape = get_dims(x) if len(x_shape) == 1: d = x_shape[0] else: d = x_shape[1] if mean is None: r = x else: mean = tf.cast(mean, dtype=tf.float32) r = x - mean if cov is 1: L_inv = tf.diag(tf.ones([d])) det_cov = tf.constant(1.0) else: cov = tf.cast(cov, dtype=tf.float32) if len(cov.get_shape()) == 1: # vector L_inv = tf.diag(1.0 / tf.sqrt(cov)) det_cov = tf.reduce_prod(cov) else: # matrix L = tf.cholesky(cov) L_inv = tf.matrix_inverse(L) det_cov = tf.pow(tf.reduce_prod(tf.diag_part(L)), 2) lps = -0.5*d*tf.log(2*np.pi) - 0.5*tf.log(det_cov) if len(x_shape) == 1: # vector r = tf.reshape(r, shape=(d, 1)) inner = tf.matmul(L_inv, r) lps -= 0.5 * tf.matmul(inner, inner, transpose_a=True) return tf.squeeze(lps) else: # matrix # TODO vectorize further out = [] for r_vec in tf.unpack(r): r_vec = tf.reshape(r_vec, shape=(d, 1)) inner = tf.matmul(L_inv, r_vec) out += [tf.squeeze(lps - 0.5 * tf.matmul(inner, inner, transpose_a=True))] return tf.pack(out)
def predict2(): # predicitions cov=h.Mul(K_mm_2,tf.matrix_inverse(K_mm_2+K_mnnm_2/tf.square(sigma_2)),K_mm_2) cov_chol=tf.cholesky(cov) mu=h.Mul(K_mm_2,tf.cholesky_solve(cov_chol,K_mn_2),Ytr)/tf.square(sigma_2) mean=h.Mul(K_nm_2,tf.matrix_solve(K_mm_1,mu)) variance=K_nn_2-h.Mul(K_nm_2,h.safe_chol(K_mm_2,tf.transpose(K_nm_2))) var_terms=2*tf.sqrt(tf.reshape(tf.diag_part(variance)+tf.square(sigma_2),[N,1])) return mean, var_terms
def predict(K_mn,sigma,K_mm,K_nn): # predicitions K_nm=tf.transpose(K_mn) K_mm_I=tf.matrix_inverse(K_mm) Sig_Inv=1e-1*np.eye(M)+K_mm+K_mnnm_2/tf.square(sigma) mu_post=h.Mul(tf.matrix_solve(Sig_Inv,K_mn),Ytr)/tf.square(sigma) mean=h.Mul(K_nm,mu_post) variance=K_nn-h.Mul(K_nm,h.safe_chol(K_mm,K_mn))+h.Mul(K_nm,tf.matrix_solve(Sig_Inv,K_mn)) var_terms=2*tf.sqrt(tf.reshape(tf.diag_part(variance)+tf.square(sigma),[N,1])) return mean, var_terms
def tf_exponent_gauss(mu1, prec, n): ''' takes in precision, and returns cov ''' prec = tf.mul(n, prec) sig = tf.matrix_inverse(prec) mu = tf.matmul(prec, mu1) mu = tf.mul(n, mu) mu = tf.matmul(sig, mu) return mu, sig
def get_q(self): n = tf.constant(self.n, dtype = tf.float64) prec = tf.mul(self.V, n) prec = tf.add(prec, self.SEP_prior_prec) qsig = tf.matrix_inverse(prec) mu = tf.matmul(self.V, self.u) mu = tf.mul(n, mu) qmu = tf.matmul(qsig, mu) return qmu, qsig
def add_predict_op(self): """ get predict todo: get variance of predictions """ tmp_K_star=self.get_covariance_x(self.test_x, self.train_x) self.K_star=tf.reshape( tf.tile(tf.reshape(self.K_c, [self.config.task_num, 1, self.config.task_num, 1]), [1, self.config.test_length, 1, self.config.train_length]) \ * tf.tile(tf.reshape(tmp_K_star, [1, self.config.test_length, 1, self.config.train_length]), [self.config.task_num, 1, self.config.task_num, 1]), \ [self.config.test_length*self.config.task_num, self.config.train_length*self.config.task_num]) self.predict_op=tf.matmul(tf.matmul(self.K_star, tf.matrix_inverse(self.sigma)), self.train_y)
def read_data(self): """Provides images and camera intrinsics.""" with tf.name_scope('data_loading'): with tf.name_scope('enqueue_paths'): seed = random.randint(0, 2**31 - 1) self.file_lists = self.compile_file_list(self.data_dir, 'train') image_paths_queue = tf.train.string_input_producer( self.file_lists['image_file_list'], seed=seed, shuffle=True) cam_paths_queue = tf.train.string_input_producer( self.file_lists['cam_file_list'], seed=seed, shuffle=True) img_reader = tf.WholeFileReader() _, image_contents = img_reader.read(image_paths_queue) image_seq = tf.image.decode_jpeg(image_contents) with tf.name_scope('load_intrinsics'): cam_reader = tf.TextLineReader() _, raw_cam_contents = cam_reader.read(cam_paths_queue) rec_def = [] for _ in range(9): rec_def.append([1.0]) raw_cam_vec = tf.decode_csv(raw_cam_contents, record_defaults=rec_def) raw_cam_vec = tf.stack(raw_cam_vec) intrinsics = tf.reshape(raw_cam_vec, [3, 3]) with tf.name_scope('convert_image'): image_seq = self.preprocess_image(image_seq) # Converts to float. with tf.name_scope('image_augmentation'): image_seq = self.augment_image_colorspace(image_seq) image_stack = self.unpack_images(image_seq) with tf.name_scope('image_augmentation_scale_crop'): image_stack, intrinsics = self.augment_images_scale_crop( image_stack, intrinsics, self.img_height, self.img_width) with tf.name_scope('multi_scale_intrinsics'): intrinsic_mat = self.get_multi_scale_intrinsics(intrinsics, self.num_scales) intrinsic_mat.set_shape([self.num_scales, 3, 3]) intrinsic_mat_inv = tf.matrix_inverse(intrinsic_mat) intrinsic_mat_inv.set_shape([self.num_scales, 3, 3]) with tf.name_scope('batching'): image_stack, intrinsic_mat, intrinsic_mat_inv = ( tf.train.shuffle_batch( [image_stack, intrinsic_mat, intrinsic_mat_inv], batch_size=self.batch_size, capacity=QUEUE_SIZE + QUEUE_BUFFER * self.batch_size, min_after_dequeue=QUEUE_SIZE)) logging.info('image_stack: %s', util.info(image_stack)) return image_stack, intrinsic_mat, intrinsic_mat_inv
def logpdf(self, x, mean=None, cov=1): """ Parameters ---------- x : np.array or tf.Tensor vector or matrix mean : np.array or tf.Tensor, optional vector. Defaults to zero mean. cov : np.array or tf.Tensor, optional vector or matrix. Defaults to identity. """ x = tf.cast(tf.convert_to_tensor(x), dtype=tf.float32) x_shape = get_dims(x) if len(x_shape) == 1: d = x_shape[0] else: d = x_shape[1] if mean is None: r = x else: mean = tf.cast(tf.convert_to_tensor(mean), dtype=tf.float32) r = x - mean if cov is 1: cov_inv = tf.diag(tf.ones([d])) det_cov = tf.constant(1.0) else: cov = tf.cast(tf.convert_to_tensor(cov), dtype=tf.float32) if len(cov.get_shape()) == 1: # vector cov_inv = tf.diag(1.0 / cov) det_cov = tf.reduce_prod(cov) else: # matrix cov_inv = tf.matrix_inverse(cov) det_cov = tf.matrix_determinant(cov) lps = -0.5*d*tf.log(2*np.pi) - 0.5*tf.log(det_cov) if len(x_shape) == 1: r = tf.reshape(r, shape=(d, 1)) lps -= 0.5 * tf.matmul(tf.matmul(r, cov_inv, transpose_a=True), r) return tf.squeeze(lps) else: # TODO vectorize further out = [] for r_vec in tf.unpack(r): r_vec = tf.reshape(r_vec, shape=(d, 1)) out += [tf.squeeze(lps - 0.5 * tf.matmul( tf.matmul(r_vec, cov_inv, transpose_a=True), r_vec))] return tf.pack(out) """
def __init__(self): self.NUM_CLASSES = 1 super().__init__(self.NUM_CLASSES) trainDataWithBias = tf.cast(np.concatenate((np.ones((self.data.trainData.shape[0], 1)), self.data.trainData), axis=1), tf.float32) self.normal_w = tf.matmul(tf.matrix_inverse(tf.matmul(tf.transpose(trainDataWithBias), trainDataWithBias)), tf.matmul(tf.transpose(trainDataWithBias), tf.cast(self.data.trainTarget, tf.float32))) self.normal_y = tf.matmul(self.x_plus_bias, self.normal_w) self.normal_mse = tf.reduce_mean(tf.square(self.y_ - self.normal_y))/2 self.loss_fcn = tf.reduce_mean(tf.square(self.y_ - self.y))/2 # this value is overriden in the case of normal equations self.correct_prediction = tf.equal(self.y_, tf.round(self.y)) self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32))
def fit(self, X, y): n, p = X.shape q = 1 if len(y.shape) == 1 else y.shape[1] W = tf.Variable(tf.random_uniform([p,q], -1., 1.)) X = tf.constant(X) y = tf.constant(y, shape = (n,q)) W = tf.matmul(tf.matrix_inverse(tf.matmul(tf.transpose(X), X)), tf.matmul(tf.transpose(X), y)) init = tf.initialize_all_variables() sess = tf.Session() sess.run(init) clf.W = sess.run(W) return self
def make_train_tf(self): x = tf.reshape(self.states, [-1, self.n_hidden_2]) y = tf.reshape(self.targets, [-1, self.n_outputs]) indices = tf.random_shuffle(tf.range(tf.shape(x)[0])) x_ = tf.gather(x, indices) y_ = tf.gather(y, indices) r = tf.matmul(tf.transpose(x_), x_) p = tf.matmul(tf.transpose(x_), y_) i = tf.diag(tf.ones([self.n_hidden_2])) w = tf.matmul(tf.matrix_inverse(r + self.ridge_beta * i), p) return tf.assign(self.w_hy, w)
def logpdf(self, x, mean=None, cov=1): """ Arguments ---------- x: tf.Tensor vector mean: tf.Tensor, optional vector. Defaults to zero mean. cov: tf.Tensor, optional vector or matrix. Defaults to identity. Returns ------- tf.Tensor scalar """ x = tf.cast(tf.squeeze(x), dtype=tf.float32) d = get_dims(x)[0] if mean is None: r = tf.ones([d]) * x else: mean = tf.cast(tf.squeeze(mean), dtype=tf.float32) r = x - mean if cov == 1: cov_inv = tf.diag(tf.ones([d])) det_cov = tf.constant(1.0) else: cov = tf.cast(tf.squeeze(cov), dtype=tf.float32) if len(cov.get_shape()) == 1: cov_inv = tf.diag(1.0 / cov) det_cov = tf.reduce_prod(cov) else: cov_inv = tf.matrix_inverse(cov) det_cov = tf.matrix_determinant(cov) r = tf.reshape(r, shape=(d, 1)) lps = -0.5*d*tf.log(2*np.pi) - 0.5*tf.log(det_cov) - \ 0.5 * tf.matmul(tf.matmul(r, cov_inv, transpose_a=True), r) """ # TensorFlow can't reverse-mode autodiff Cholesky L = tf.cholesky(cov) L_inv = tf.matrix_inverse(L) det_cov = tf.pow(tf.matrix_determinant(L), 2) inner = dot(L_inv, r) out = -0.5*d*tf.log(2*np.pi) - \ 0.5*tf.log(det_cov) - \ 0.5*tf.matmul(tf.transpose(inner), inner) """ return tf.squeeze(lps)
def get_scg_energy_fn(): """Get energy function for 2d strongly correlated Gaussian.""" # Avoid recreating tf constants on each invocation of gradients mu = tf.constant([0., 0.]) sigma = tf.constant([[50.05, -49.95], [-49.95, 50.05]]) sigma_inv = tf.matrix_inverse(sigma) def energy(x): """Unnormalized log density/energy of 2d strongly correlated Gaussian.""" xmmu = x - mu return .5 * tf.diag_part( tf.matmul(tf.matmul(xmmu, sigma_inv), tf.transpose(xmmu))) return energy
def _verifyInverse(self, x): for np_type in [np.float32, np.float64]: for adjoint in False, True: y = x.astype(np_type) with self.test_session(): # Verify that x^{-1} * x == Identity matrix. inv = tf.matrix_inverse(y, adjoint=adjoint) tf_ans = tf.batch_matmul(inv, y, adj_y=adjoint) np_ans = np.identity(y.shape[-1]) if x.ndim > 2: tiling = list(y.shape) tiling[-2:] = [1, 1] np_ans = np.tile(np_ans, tiling) out = tf_ans.eval() self.assertAllClose(np_ans, out) self.assertShapeEqual(y, tf_ans)
def solve_ridge(x, y, ridge_factor): with tf.name_scope("solve_ridge"): # Added a column of ones to the end of the feature matrix for bias A = tf.concat([x, tf.ones((x.shape.as_list()[0], 1))], axis=1) # Analytic solution for the ridge regression loss inv_target = tf.matmul(A, A, transpose_a=True) np_diag_penalty = ridge_factor * np.ones( A.shape.as_list()[1], dtype="float32") # Remove penalty on bias component of weights np_diag_penalty[-1] = 0. diag_penalty = tf.constant(np_diag_penalty) inv_target += tf.diag(diag_penalty) inv = tf.matrix_inverse(inv_target) w = tf.matmul(inv, tf.matmul(A, y, transpose_a=True)) return w
def q2part3(): sess = tf.Session() init = tf.global_variables_initializer() sess.run(init) trainData, trainTarget, validData, validTarget, testData, testTarget = data_gen() biasTrainData = tf.convert_to_tensor(trainData) biasTrainData = tf.reshape(biasTrainData, [3500, -1]) biasVec = tf.convert_to_tensor([1 for i in range(3500)]) biasVec = tf.cast(biasVec, tf.float64) biasVec = tf.reshape(biasVec, [-1, 3500]) biasTrainData = tf.transpose(biasTrainData) biasTrainData = tf.concat([biasVec, biasTrainData], 0) biasTrainData = tf.transpose(biasTrainData) biasTrainTarget = tf.convert_to_tensor(trainTarget) biasTrainTarget = tf.cast(biasTrainTarget, tf.float64) biasValidData = tf.convert_to_tensor(validData) biasValidData = tf.reshape(biasValidData, [100, -1]) biasVec = tf.convert_to_tensor([1 for i in range(100)]) biasVec = tf.cast(biasVec, tf.float64) biasVec = tf.reshape(biasVec, [-1, 100]) biasValidData = tf.transpose(biasValidData) biasValidData = tf.concat([biasVec, biasValidData], 0) biasValidData = tf.transpose(biasValidData) biasValidTarget = tf.convert_to_tensor(validTarget) biasValidTarget = tf.cast(biasValidTarget, tf.float64) biasTestData = tf.convert_to_tensor(testData) biasTestData = tf.reshape(biasTestData, [145, -1]) biasVec = tf.convert_to_tensor([1 for i in range(145)]) biasVec = tf.cast(biasVec, tf.float64) biasVec = tf.reshape(biasVec, [-1, 145]) biasTestData = tf.transpose(biasTestData) biasTestData = tf.concat([biasVec, biasTestData], 0) biasTestData = tf.transpose(biasTestData) biasTestTarget = tf.convert_to_tensor(testTarget) biasTestTarget = tf.cast(biasTestTarget, tf.float64) W_norm = tf.matmul(tf.transpose(biasTrainData), biasTrainData) W_norm = tf.matrix_inverse(W_norm) W_norm = tf.matmul(W_norm, tf.transpose(biasTrainData)) W_norm = tf.matmul(W_norm, biasTrainTarget) epoch_size = len(trainData) batch_size = 500 numBatches = epoch_size / batch_size # flatten training data trainData = np.reshape(trainData, [epoch_size, -1]) # flatten validation data validData = np.reshape(validData, [100, -1]) # flatten test data testData = np.reshape(testData, [145, -1]) # reshape training target trainTarget = np.reshape(trainTarget, [epoch_size, -1]) # reshape validation target validTarget = np.reshape(validTarget, [100, -1]) # reshape test target testTarget = np.reshape(testTarget, [145, -1]) # variable creation W = tf.Variable(tf.random_normal(shape=[trainData.shape[1], 1], stddev=0.35, seed=521), name="weights") # W = tf.Variable(tf.random_normal(shape=[trainData.shape[1], 1], stddev=0.1), name="weights") # W = tf.Variable(tf.zeros([trainData.shape[1], 1]), dtype=tf.float32) b = tf.Variable(0.0, name="biases") X = tf.placeholder(tf.float32, name="input_x") y_target = tf.placeholder(tf.float32, name="target_y") # graph definition y_pred = tf.matmul(X, W) + b # need to define W_lambda, l_rate l_rate = tf.placeholder(tf.float32, [], name="learning_rate") W_lambda = tf.placeholder(tf.float32, [], name="weight_decay") # error definition logits_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=y_pred, labels=y_target) / 2.0, name="logits_loss") W_decay = tf.reduce_sum(tf.multiply(W, W)) * W_lambda / 2.0 loss = logits_loss + W_decay # accuracy definition y_pred_sigmoid = tf.sigmoid(y_pred) accuracy = tf.reduce_mean( tf.cast(tf.equal(tf.cast(tf.greater(y_pred_sigmoid, 0.5), tf.float32), y_target), tf.float32)) # training mechanism optimizer = tf.train.AdamOptimizer(learning_rate=l_rate) train = optimizer.minimize(loss=loss) # specify learning rate each_l_rate = 0.005 # specify weight decay coefficient each_W_lambda = 0.0 train_error_list = [] # initialize session sess = tf.Session() init = tf.global_variables_initializer() sess.run(init) sess.run(W) sess.run(b) for step in range(0, 5000): batch_idx = step % numBatches trainDataBatch = trainData[int(batch_idx * batch_size):int((batch_idx + 1) * batch_size)] trainTargetBatch = trainTarget[int(batch_idx * batch_size):int((batch_idx + 1) * batch_size)] _, currentW, currentb, yhat = sess.run([train, W, b, y_pred], feed_dict={X: trainDataBatch, y_target: trainTargetBatch, l_rate: each_l_rate, W_lambda: each_W_lambda}) if batch_idx == numBatches - 1: train_error = sess.run(loss, feed_dict={X: trainDataBatch, y_target: trainTargetBatch, W_lambda: each_W_lambda}) train_accuracy = sess.run(accuracy, feed_dict={X: trainDataBatch, y_target: trainTargetBatch}) training_error = sess.run(loss, feed_dict={X: trainDataBatch, y_target: trainTargetBatch, W_lambda: each_W_lambda}) train_error_list.append(training_error) # trainData accuracy y = tf.convert_to_tensor(trainTarget) y = tf.reshape(y, [3500,-1]) y = tf.cast(y, tf.float64) yhat_norm = tf.matmul(biasTrainData, W_norm) yhat_bound_norm = tf.cast(tf.greater(yhat_norm, 0.5), tf.float64) equal_norm = tf.cast(tf.equal(y, yhat_bound_norm), tf.float64) accuracy_norm = tf.reduce_sum(tf.reduce_sum(equal_norm, 0), 0)/3500 print("TrainData Accuracy (linear regression normal equation):", sess.run(accuracy_norm)) accuracy_SGD = sess.run(accuracy, feed_dict={X: trainData, y_target: trainTarget}) print("TrainData Accuracy (logistic regression SGD):", accuracy_SGD) # validData accuracy y = tf.convert_to_tensor(validTarget) y = tf.reshape(y, [100,-1]) y = tf.cast(y, tf.float64) yhat_norm = tf.matmul(biasValidData, W_norm) yhat_bound_norm = tf.cast(tf.greater(yhat_norm, 0.5), tf.float64) equal_norm = tf.cast(tf.equal(y, yhat_bound_norm), tf.float64) accuracy_norm = tf.reduce_sum(tf.reduce_sum(equal_norm, 0), 0)/100 print("ValidData Accuracy (linear regression normal equation):", sess.run(accuracy_norm)) accuracy_SGD = sess.run(accuracy, feed_dict={X: validData, y_target: validTarget}) print("ValidData Accuracy (logistic regression SGD):", accuracy_SGD) # testData accuracy y = tf.convert_to_tensor(testTarget) y = tf.reshape(y, [145,-1]) y = tf.cast(y, tf.float64) yhat_norm = tf.matmul(biasTestData, W_norm) yhat_bound_norm = tf.cast(tf.greater(yhat_norm, 0.5), tf.float64) equal_norm = tf.cast(tf.equal(y, yhat_bound_norm), tf.float64) accuracy_norm = tf.reduce_sum(tf.reduce_sum(equal_norm, 0), 0)/145 print("TestData Accuracy (linear regression normal equation):", sess.run(accuracy_norm)) accuracy_SGD = sess.run(accuracy, feed_dict={X: testData, y_target: testTarget}) print("TestData Accuracy (logistic regression SGD):", accuracy_SGD) # specify learning rate each_l_rate = 0.001 # specify weight decay coefficient each_W_lambda = 0.0 train_error_list = [] train_accuracy_list = [] linear_train_error_list = [] linear_train_accuracy_list = [] # variable creation linear_W = tf.Variable(tf.random_normal(shape=[int(np.shape(trainData)[1]), 1], stddev=0.5, seed=521), name="weights") # linear_W = tf.Variable(tf.truncated_normal(shape=[int(np.shape(trainData)[1]), 1], stddev=0.1), name="weights") # linear_W = tf.Variable(tf.zeros([trainData.shape[1], 1]), dtype=tf.float32) linear_b = tf.Variable(0.0, name="biases") linear_X = tf.placeholder(tf.float32, [batch_size, int(np.shape(trainData)[1])], name="input_x") linear_y_target = tf.placeholder(tf.float32, [batch_size, 1], name="target_y") # graph definition linear_y_pred = tf.matmul(linear_X,linear_W) + linear_b linear_l_rate = tf.placeholder(tf.float32, [], name="learning_rate") linear_W_lambda = tf.placeholder(tf.float32, [], name="weight_decay") # error definition linear_mse = tf.reduce_sum(tf.square(linear_y_pred-linear_y_target), name="mse")/(2.0*batch_size) linear_W_decay = tf.reduce_sum(tf.multiply(linear_W,linear_W))*linear_W_lambda/2 linear_loss = linear_mse + linear_W_decay # training mechanism linear_optimizer = tf.train.AdamOptimizer(learning_rate=linear_l_rate) linear_train = linear_optimizer.minimize(loss=linear_loss) linear_accuracy = tf.reduce_mean( tf.cast(tf.equal(tf.cast(tf.greater(linear_y_pred, 0.5), tf.float32), linear_y_target), tf.float32)) # initialize session sess = tf.Session() init = tf.global_variables_initializer() sess.run(init) sess.run(W) sess.run(b) sess.run(linear_W) sess.run(linear_b) for step in range(0, 5000): batch_idx = step % numBatches trainDataBatch = trainData[int(batch_idx * batch_size):int((batch_idx + 1) * batch_size)] trainTargetBatch = trainTarget[int(batch_idx * batch_size):int((batch_idx + 1) * batch_size)] _, currentW, currentb, yhat = sess.run([train, W, b, y_pred], feed_dict={X: trainDataBatch, y_target: trainTargetBatch, l_rate: each_l_rate, W_lambda: each_W_lambda}) _, currentW, currentb, yhat = sess.run([linear_train, linear_W, linear_b, linear_y_pred], feed_dict={linear_X: trainDataBatch, linear_y_target: trainTargetBatch, linear_l_rate: each_l_rate, linear_W_lambda: each_W_lambda}) if batch_idx == numBatches - 1: train_error = sess.run(loss, feed_dict={X: trainDataBatch, y_target: trainTargetBatch, W_lambda: each_W_lambda}) linear_train_error = sess.run(linear_loss, feed_dict={linear_X: trainDataBatch, linear_y_target: trainTargetBatch, linear_W_lambda: each_W_lambda}) train_accuracy = sess.run(accuracy, feed_dict={X: trainDataBatch, y_target: trainTargetBatch}) linear_train_accuracy = sess.run(linear_accuracy, feed_dict={linear_X: trainDataBatch, linear_y_target: trainTargetBatch}) train_error_list.append(train_error) train_accuracy_list.append(train_accuracy) linear_train_error_list.append(linear_train_error) linear_train_accuracy_list.append(linear_train_accuracy) print("Logistic Regression Cross-Entropy Loss:", train_error_list[-1]) print("Linear Regression MSE Loss:", linear_train_error_list[-1]) print("Logistic Regression Accuracy:", train_accuracy_list[-1]) print("Linear Regression Accuracy:", linear_train_accuracy_list[-1]) plt.clf() plt.plot(train_error_list) plt.title('Training Cross-Entropy Loss - Logistic Regression') plt.xlabel('Number of Epoch') plt.ylabel('Cross-Entropy Loss') x1, x2, x3, x4 = plt.axis() plt.axis((x1, x2, 0, 0.5)) plt.savefig('logistic_regression_error.png') plt.clf() plt.plot(linear_train_error_list) plt.title('Training MSE Loss - Linear Regression') plt.xlabel('Number of Epoch') plt.ylabel('MSE Loss') x1, x2, x3, x4 = plt.axis() plt.axis((x1, x2, 0, 2)) plt.savefig('linear_regression_loss.png') plt.clf() plt.plot(train_accuracy_list) plt.title('Training Accuracy - Logistic Regression') plt.xlabel('Number of Epoch') plt.ylabel('Accuracy') plt.savefig('logistic_regression_accuracy.png') plt.clf() plt.plot(linear_train_accuracy_list) plt.title('Training Accuracy - Linear Regression') plt.xlabel('Number of Epoch') plt.ylabel('Accuracy') plt.savefig('linear_regression_accuracy.png') # Comparison between cross-entropy and MSE loss over y = [0,1] y = 0 yhat = np.linspace(0,1,500) cross_entropy_loss = -1 * y * np.log(yhat) - (1 - y) * np.log(1 - yhat) mse_loss = np.power(y - yhat, 2) plt.clf() plt.plot(yhat, cross_entropy_loss, label='cross-entropy loss') plt.plot(yhat, mse_loss, label='MSE loss') plt.title('Cross-Entropy Loss vs MSE loss') plt.xlabel('prediction yhat (given y = 0)') plt.ylabel('Loss') plt.legend() plt.savefig('cross_entropy_vs_mse_loss.png')
validData = validData.reshape([validData.shape[0], 28 * 28]) testData = testData.reshape([testData.shape[0], 28 * 28]) #NORMAL EQUATIONS FOR LEAST SQUARE SOLUTION # set parameters and adjust for bias term x_bias_train = tf.ones([trainData.shape[0], 1]) x_bias_valid = tf.ones([validData.shape[0], 1]) x_bias_test = tf.ones([testData.shape[0], 1]) trainDataMatrix = tf.concat([x_bias_train, trainData], 1) validDataMatrix = tf.concat([x_bias_valid, validData], 1) testDataMatrix = tf.concat([x_bias_test, testData], 1) # implement normal equations w_ls = tf.matmul( tf.matmul( tf.matrix_inverse( tf.matmul(tf.transpose(trainDataMatrix), trainDataMatrix)), tf.transpose(trainDataMatrix)), tf.cast(trainTarget, tf.float32)) # compute MSE for training dataset Y_pred_train = tf.matmul(trainDataMatrix, w_ls) Y_pred_valid = tf.matmul(validDataMatrix, w_ls) Y_pred_test = tf.matmul(testDataMatrix, w_ls) # compute accuracy for training dataset Y_pred_train_set = tf.sign(tf.sign(Y_pred_train - 0.5) + 1) Y_pred_valid_set = tf.sign(tf.sign(Y_pred_valid - 0.5) + 1) Y_pred_test_set = tf.sign(tf.sign(Y_pred_test - 0.5) + 1) accuracy_train = tf.contrib.metrics.accuracy(tf.to_int32(Y_pred_train_set), tf.to_int32(trainTarget)) accuracy_valid = tf.contrib.metrics.accuracy(tf.to_int32(Y_pred_valid_set), tf.to_int32(validTarget))
all_variables[i].assign(all_variables[i] - alpha * tf.squeeze(update[i]))) return tf.pack(optmize_variables) x = tf.Variable(np.random.random_sample(), dtype=tf.float32) y = tf.Variable(np.random.random_sample(), dtype=tf.float32) alpha = cons(0.1) # f = tf.pow(x, cons(2)) + cons(2) * x * y + cons(3) * tf.pow(y, cons(2)) + cons(4) * x + cons(5) * y + cons(6) f = cons(0.5) * tf.pow(x, 2) + cons(2.5) * tf.pow(y, 2) all_variables = [x, y] hessian = compute_hessian(f, all_variables) hessian_inv = tf.matrix_inverse(hessian) g = compute_grads(f, all_variables) update = tf.unpack(tf.matmul(hessian_inv, g)) optimize_op = optimize(all_variables, update) sess = tf.Session() sess.run(tf.initialize_all_variables()) func = np.inf for i in range(10): prev = func v1, v2, func = sess.run([x, y, f]) print(v1, v2, func) sess.run(optimize_op)
def main(_): if FLAGS.input_type == 'PP': assert ('hrestgt' not in FLAGS.supervision) tf.logging.set_verbosity(tf.logging.INFO) tf.set_random_seed(FLAGS.random_seed) FLAGS.checkpoint_dir += '/%s/' % FLAGS.experiment_name if not tf.gfile.IsDirectory(FLAGS.checkpoint_dir): tf.gfile.MakeDirs(FLAGS.checkpoint_dir) tf.logging.info("Image dir: %s" % FLAGS.image_dir) if 'hrestgt' in FLAGS.supervision: tf.logging.info("High-resolution image dir: %s" % FLAGS.image_dir) if FLAGS.input_type == 'REALESTATE_PP': data_loader = RealEstateSequenceDataLoader(FLAGS.cameras_glob, FLAGS.image_dir, True, 1, FLAGS.shuffle_seq_length, FLAGS.random_seed) else: data_loader = ReplicaSequenceDataLoader(FLAGS.cameras_glob, FLAGS.image_dir, FLAGS.hres_image_dir, True, FLAGS.shuffle_seq_length, FLAGS.random_seed) train_batch = data_loader.sample_batch() # additional input tensors if 'PP' in FLAGS.input_type: train_batch['interp_pose'] = interpolate_pose(train_batch["ref_pose"], train_batch["src_pose"]) train_batch['interp_pose_inv'] = tf.matrix_inverse( train_batch["interp_pose"], name='interp_pose_inv') train_batch['intrinsics_inv'] = tf.matrix_inverse( train_batch['intrinsics'], name='intrinsics_inv') train_batch['ref_pose_inv'] = tf.matrix_inverse(train_batch['ref_pose'], name='ref_pose_inv') train_batch['jitter_pose'] = tf.identity(tf_random_rotation( FLAGS.rot_factor, FLAGS.tr_factor), name='jitter_pose') train_batch['jitter_pose_inv'] = tf.matrix_inverse( train_batch['jitter_pose'], name='jitter_pose_inv') if FLAGS.gcn: coord, support, pix2vertex_lookup = load_mesh_input() train_batch['p2v'] = tf.dtypes.cast(tf.identity(pix2vertex_lookup, name='p2v'), dtype=tf.float32) train_batch['coord'] = tf.identity(coord, name='coord') train_batch['support'] = [ tf.SparseTensor(indices=np.asarray(support[i][0], dtype=np.float32), values=np.asarray(support[i][1], dtype=np.float32), dense_shape=np.asarray(support[i][2], dtype=np.float32)) for i in range(len(support)) ] model = MSI() train_op = model.build_train_graph(train_batch, FLAGS.min_depth, FLAGS.max_depth, FLAGS.num_psv_planes, FLAGS.num_msi_planes, FLAGS.which_color_pred, FLAGS.which_loss, FLAGS.learning_rate, FLAGS.beta1) model.train(train_op, FLAGS.checkpoint_dir, FLAGS.continue_train, FLAGS.summary_freq, FLAGS.save_latest_freq, FLAGS.max_steps)
def __affine_shape(self,shapes,r,t,isinv=False): if isinv: r = tf.matrix_inverse(r) t = tf.matmul(-t,r) shapes = tf.matmul(shapes,r) + t return shapes
def __call__(self, L, X, HtY, HtH, profile_x, regularization=None): train_samples = tf.shape(HtY)[0] X_ZF = tf.matmul(tf.expand_dims(HtY, 1), tf.matrix_inverse(HtH)) X_ZF = tf.squeeze(X_ZF, 1) loss_ZF = tf.reduce_mean(tf.square(X - X_ZF)) ber_ZF = tf.reduce_mean( tf.cast(tf.not_equal(X, tf.sign(X_ZF)), tf.float32)) v_size = 2 * self.Nt ## size of the second sub-layer (see table ) hl_size = 8 * self.Nt ## size of the first sub-layer S = [] S.append(tf.zeros([train_samples, self.Nt])) a = [] a.append(tf.zeros([train_samples, v_size])) LOSS = [] LOSS.append(tf.zeros([])) BER = [] BER.append(tf.zeros([])) ## Regularizing coefficients lam_da = tf.Variable(0.001, trainable=True) l1_regularizer = tf.contrib.layers.l1_regularizer(scale=0.005, scope=None) weights = tf.trainable_variables() # all vars of your graph L1_nrom_reg = tf.contrib.layers.apply_regularization( l1_regularizer, weights) ## Sublayer weights W1r = [] W2r = [] W3r = [] for i in range(1, L + 1): u_1 = tf.matmul(tf.expand_dims(S[-1], 1), HtH) u = tf.squeeze(u_1, 1) x = tf.concat([HtY, S[-1], u, a[-1]], 1) Z, W1, b1 = tf.multiply( relu_layer(x, 3 * self.Nt + v_size, hl_size, 'relu' + str(i)), profile_x) # 1st sub-layer (18) W1r.append(W1) a, W2, b2 = linear_layer(tf.multiply(Z, profile_x), hl_size, v_size, 'aff' + str(i)) # 2nd sub-layer (19) a.append(a) W2r.append(W2) a[i] = (1 - alpha) * a[i] + alpha * a[i - 1] S, W3, b3 = sign_layer(tf.multiply(Z, profile_x), hl_size, self.Nt, 'sign' + str(i)) # 3rd sub-layer (10) S.append(S) W3r.append(W3) S[i] = (1 - alpha) * S[i] + alpha * S[i - 1] error_linear = tf.reduce_mean( tf.square(X - X_ZF), 1) ## Linear receiver error symbol estimated error_wesnet = tf.reduce_mean( tf.square(X - S[-1]), 1) ## WeSNet symbol error symbol estimate ''' Sparcity inforcing regularisation for for i = l to L; where l is the layer from which the sparcity is initially enforced. See the paper for the details. ''' if regularization: if 10 <= i <= L + 1: L1_nrom_reg = tf.norm(W1r[i], ord=1) + tf.norm( W2r[i], ord=1) + tf.norm( W3r[i], ord=1) ## L1 norm regularizer (23) regularization_penalty = lam_da * np.log( 1 + (i - 1) * L1_nrom_reg) ## L1 norm regularizer (22) loss = np.log(i) * tf.reduce_mean( error_wesnet / error_linear ) + regularization_penalty ## Regularised loss function (21) else: loss = np.log(i) * tf.reduce_mean( error_wesnet / error_linear) ## Non-regularised loss function (20) LOSS.append(loss) BER.append( tf.reduce_mean( tf.cast(tf.not_equal(X, tf.sign(S[-1])), tf.float32))) TOTAL_LOSS = tf.add_n(LOSS) return TOTAL_LOSS, LOSS, BER
def eigendecomp(A): d, P = tf.self_adjoint_eig(A) return P, tf.diag(d), tf.matrix_inverse(P)
import tensorflow as tf a = tf.constant([[1., 2.], [3., 4.]]) b = tf.constant([[5., 6.], [7., 8.]]) x1 = tf.matmul(a, b) x2 = tf.matrix_determinant(a) x3 = tf.matrix_inverse(a) with tf.Session() as sess: y1, y2, y3 = sess.run([x1, x2, x3]) print(y1) print(y2) print(y3)
def train_cholesky(): tf.reset_default_graph() dtype = tf.float64 XY0 = np.genfromtxt('linear-cholesky-XY.csv', delimiter=",") X0 = XY0[:-1, :] # 2 x d Y0 = XY0[-1:, :] # 1 x d dsize = X0.shape[1] fsize = X0.shape[0] X_ = tf.placeholder(dtype, shape=X0.shape) Y_ = tf.placeholder(dtype, shape=Y0.shape) X = tf.Variable(X_, trainable=False) Y = tf.Variable(Y_, trainable=False) W = tf.Variable([[2, 1]], dtype=dtype) error = tf.matmul(W, X) - Y # 1 x d loss = tf.reduce_sum(tf.square(error)) / dsize sess = tf.Session() cov_ = tf.matmul(X_, tf.transpose(X_)) / dsize cov = tf.Variable(cov_, trainable=False) covI = tf.Variable(tf.matrix_inverse(cov_), trainable=False) init_dict = {X_: X0, Y_: Y0} lr = tf.constant(1, dtype=dtype) opt = tf.train.GradientDescentOptimizer(learning_rate=lr) (grad, var) = opt.compute_gradients(loss, tf.trainable_variables())[0] ones_list = tf.constant([1.0] * fsize, dtype=dtype) # decomposition of covariance matrix Ch = tf.cholesky(cov) D1 = tf.diag(tf.diag_part(Ch)) L = Ch @ tf.matrix_inverse(D1) cov2 = L @ D1 @ D1 @ tf.transpose(L) # decomposition of precision matrix T = tf.matrix_inverse(L) covI2 = tf.transpose(T) @ tf.matrix_inverse(D1 @ D1) @ T # 1/2 of precision matrix recovers inverse Hessian pre = covI2 / 2 grad = tf.matmul(grad, pre) train_op = opt.apply_gradients([(grad, var)]) sess.run(tf.global_variables_initializer(), feed_dict=init_dict) assert sess.run(tf.norm(cov2 - cov)) == 0. assert sess.run(tf.norm(covI2 - covI)) == 0. observed_losses = [] for i in range(100): loss0, _ = sess.run([loss, train_op]) observed_losses.append(loss0) # from accompanying notebook # [1.03475, 4.90422*10^-28, 0., 0., 0., 0., 0., 0., 0., 0.... expected_losses = np.loadtxt("linear-cholesky-losses1.csv") np.testing.assert_allclose(expected_losses[:10], observed_losses[:10], rtol=1e-12, atol=1e-12)
def body(Sin, i): # We distinguish the case of diagonal and full covariance matrices # because computations can be made more efficiently for diagonal ones. n = self.enr_unique_counts[i[-1]] # Number of sessions start = self.enr_counts_offset[i[-1]] end = self.enr_counts_offset[i[-1] + 1] M1 = self.M1[start:end, :] I_1xn1 = tf.transpose( tf.ones_like(M1[:, 0:1]) ) # "0:1" is just so that shape becomes (?,1) instead of (?,) if (self.kaldi_norm): norm_vector = 1.0 / (B + 1.0 / tf.cast(n, self.floatX)) norm = tf.reduce_sum((M1**2) * norm_vector, axis=1, keep_dims=True) norm = tf.sqrt(self.dim / norm) # Get dim!!!!!!!!! M1 = M1 * norm if self.diag_cov: P0 = 1.0 / (W + B / W ) # Take this and the below out from "body" ?? C0 = 0.5 * tf.reduce_sum( tf.log(P0) ) # We don't include 2pi^(0.5d) since it will be cancelled out. G = B / (tf.cast(n, self.floatX) * B + W ) # ( d x d ) Need this at two places M1n = tf.cast( n, self.floatX ) * M1 * G # ( n1 x d ) The "normalized" enrollemnt means P = 1.0 / (W + G) # Precision C = 0.5 * tf.reduce_sum(tf.log(P)) # For each normalized enrolment ivector, m1n, and each test i-vector, m2, we would like # to calculate (m1n - ,2)P(m1n - ,2)'. We do that with a couple of matrix products rather # than a loop. It could be worth thinking about whether it can be done with one operation # involving 3D tensors. S1 = tf.matmul(M1n, tf.transpose(M2 * P)) S2_1 = tf.matmul( tf.reduce_sum(((M1n * P) * M1n), axis=1)[:, None], I_1xn2) S2_2 = tf.matmul( tf.reduce_sum(((M2 * P) * M2), axis=1)[:, None], I_1xn1) S2_2_0 = tf.matmul( tf.reduce_sum(((M2 * P0) * M2), axis=1)[:, None], I_1xn1) else: # THIS IS WRONG P0 = tf.matrix_inverse(W + tf.matmul(B, tf.matrix_inverse(W))) C0 = 0.5 * tf.reduce_sum( tf.log(tf.diag_part(P0)) ) # We don't include 2pi^(0.5d) since it will be cancelled out. #n = 1 # Number of sessions G = tf.matmul( B, tf.matrix_inverse(n * B + W)) # ( d x d ) Need this at two places M1n = n * tf.matmul( M1, G) # ( n1 x d ) The "normalized" enrollemnt means P = tf.matrix_inverse(W + G) # Precision #S, C = 0.5*tf.slogdet(P) # We don't include 2pi^(0.5d) since it will be cancelled out. C = 0.5 * tf.reduce_sum(tf.log(tf.diag_part(P))) # For each normalized enrolment ivector, m1n, and each test i-vector, m2, we would like # to calculate (m1n - ,2)P(m1n - ,2)'. We do that with a couple of matrix products rather # than a loop. It could be worth thinking about whether it can be done with one operation # involving 3D tensors. S1 = tf.matmul(M1n, tf.matmul(P, M2, transpose_b=True)) S2_1 = tf.matmul( tf.reduce_sum(((tf.matmul(M1n, P)) * M1n), axis=1)[:, None], I_1xn2) S2_2 = tf.matmul( tf.reduce_sum(((tf.matmul(M2, P)) * M2), axis=1)[:, None], I_1xn1) S2_2_0 = tf.matmul( tf.reduce_sum(((tf.matmul(M2, P0)) * M2), axis=1)[:, None], I_1xn1) S = 0.5 * (2 * S1 - S2_1 - tf.transpose(S2_2) + tf.transpose(S2_2_0)) + C - C0 S = tf.concat([Sin, S], axis=0) return [S, tf.add(i, 1)]
def invertible_1x1_conv(name, z, logdet, reverse=False): if True: # Set to "False" to use the LU-decomposed version with tf.variable_scope(name): shape = Z.int_shape(z) w_shape = [shape[3], shape[3]] # Sample a random orthogonal matrix: w_init = np.linalg.qr( np.random.randn(*w_shape))[0].astype('float32') w = tf.get_variable("W", dtype=tf.float32, initializer=w_init) # dlogdet = tf.linalg.LinearOperator(w).log_abs_determinant() * shape[1]*shape[2] dlogdet = tf.cast( tf.log(abs(tf.matrix_determinant(tf.cast(w, 'float64')))), 'float32') * shape[1] * shape[2] if not reverse: _w = tf.reshape(w, [1, 1] + w_shape) z = tf.nn.conv2d(z, _w, [1, 1, 1, 1], 'SAME', data_format='NHWC') logdet += dlogdet return z, logdet else: _w = tf.matrix_inverse(w) _w = tf.reshape(_w, [1, 1] + w_shape) z = tf.nn.conv2d(z, _w, [1, 1, 1, 1], 'SAME', data_format='NHWC') logdet -= dlogdet return z, logdet else: # LU-decomposed version shape = Z.int_shape(z) with tf.variable_scope(name): dtype = 'float64' # Random orthogonal matrix: import scipy np_w = scipy.linalg.qr(np.random.randn( shape[3], shape[3]))[0].astype('float32') np_p, np_l, np_u = scipy.linalg.lu(np_w) np_s = np.diag(np_u) np_sign_s = np.sign(np_s) np_log_s = np.log(abs(np_s)) np_u = np.triu(np_u, k=1) p = tf.get_variable("P", initializer=np_p, trainable=False) l = tf.get_variable("L", initializer=np_l) sign_s = tf.get_variable("sign_S", initializer=np_sign_s, trainable=False) log_s = tf.get_variable("log_S", initializer=np_log_s) # S = tf.get_variable("S", initializer=np_s) u = tf.get_variable("U", initializer=np_u) p = tf.cast(p, dtype) l = tf.cast(l, dtype) sign_s = tf.cast(sign_s, dtype) log_s = tf.cast(log_s, dtype) u = tf.cast(u, dtype) w_shape = [shape[3], shape[3]] l_mask = np.tril(np.ones(w_shape, dtype=dtype), -1) l = l * l_mask + tf.eye(*w_shape, dtype=dtype) u = u * np.transpose(l_mask) + tf.diag(sign_s * tf.exp(log_s)) w = tf.matmul(p, tf.matmul(l, u)) if True: u_inv = tf.matrix_inverse(u) l_inv = tf.matrix_inverse(l) p_inv = tf.matrix_inverse(p) w_inv = tf.matmul(u_inv, tf.matmul(l_inv, p_inv)) else: w_inv = tf.matrix_inverse(w) w = tf.cast(w, tf.float32) w_inv = tf.cast(w_inv, tf.float32) log_s = tf.cast(log_s, tf.float32) if not reverse: w = tf.reshape(w, [1, 1] + w_shape) z = tf.nn.conv2d(z, w, [1, 1, 1, 1], 'SAME', data_format='NHWC') logdet += tf.reduce_sum(log_s) * (shape[1] * shape[2]) return z, logdet else: w_inv = tf.reshape(w_inv, [1, 1] + w_shape) z = tf.nn.conv2d(z, w_inv, [1, 1, 1, 1], 'SAME', data_format='NHWC') logdet -= tf.reduce_sum(log_s) * (shape[1] * shape[2]) return z, logdet
def batch_ll(Kt, t, Y, sigma2_n, sigma2_g, A, W, N, T, S): """ Approximates log-expected likelihood term using Monte-Carlo samples. Args: Kt: Tensor of shape N x N containing kernel values at observation times `t'. t: Tensor of shape N x 1 containing observation times. Y: Tensor of shape T x N containing T observations from N nodes. sigma2_n: Tensor of shape () containing observation noise variance. sigma2_g: Tensor of shape () containing connection noise variance A: Tensor of shape S x N x N containing samples from A for each connection ij. W: Tensor of shape S x N x N containing samples from W for each connection ij. N: Number of nodes. T: Number of observation per nodes. S: Number of samples. Returns: Tesnro of shape () which contains approximated expected log-likelihood. """ matmul = tf.matmul I = tf.constant(np.identity(N), dtype=Latnet.FLOAT) B = tf.multiply(A, W) IB = tf.subtract(I, B) L = tf.matrix_inverse(IB) Eg = matmul(matmul(L, matmul(B, tf.transpose(B, [0, 2, 1]))), tf.transpose(L, [0, 2, 1])) Sn = tf.add(tf.multiply(sigma2_g, Eg), tf.multiply(sigma2_n, I)) Kf = tf.matrix_inverse(matmul(tf.transpose(IB, [0, 2, 1]), IB)) lt, Qt = tf.self_adjoint_eig(Kt) lt = tf.cast(lt, Latnet.FLOAT) Qt = tf.cast(Qt, Latnet.FLOAT) ln, Qn = tf.self_adjoint_eig(Sn) Ln_inv = tf.matrix_diag( tf.sqrt(tf.math.divide(tf.constant(1., dtype=Latnet.FLOAT), ln))) Ln_inv_Qn = matmul(Ln_inv, tf.transpose(Qn, [0, 2, 1])) Ktilda_f = matmul(matmul(Ln_inv_Qn, Kf), tf.transpose(Ln_inv_Qn, [0, 2, 1])) ltilda_f, Qtilda_f = tf.self_adjoint_eig(Ktilda_f) Lt_Lf = tf.add( matmul( tf.transpose( tf.expand_dims(tf.tile(tf.expand_dims(lt, -1), [1, S]), -3)), tf.expand_dims(ltilda_f, -2)), tf.constant(1.0, dtype=Latnet.FLOAT)) logdet = tf.multiply(tf.cast(T, Latnet.FLOAT), tf.reduce_sum( tf.log(ln))) + tf.reduce_sum(tf.log(Lt_Lf)) Ytilda = matmul(matmul(tf.tile(tf.expand_dims(Y, -3), [S, 1, 1]), Qn), Ln_inv) Qt_expanded = tf.tile(tf.expand_dims(Qt, -3), [S, 1, 1]) Ytf = tf.multiply( tf.math.divide(tf.constant(1.0, dtype=Latnet.FLOAT), Lt_Lf), matmul(tf.transpose(Qt_expanded, [0, 2, 1]), matmul(Ytilda, Qtilda_f))) ySy = tf.reduce_sum( tf.matrix_diag_part( matmul( matmul( matmul(tf.transpose(Ytilda, [0, 2, 1]), Qt_expanded), Ytf), tf.transpose(Qtilda_f, [0, 2, 1])))) _half = tf.constant(0.5, dtype=Latnet.FLOAT) return tf.multiply(-_half, tf.math.divide(logdet, S)) + tf.multiply( -_half, tf.math.divide( ySy, S)) - (N * T) / tf.constant(2.0, Latnet.FLOAT) * tf.log( tf.constant(2.0, dtype=Latnet.FLOAT) * np.pi)
def get(self): """ Provides input data to the graph. """ # calculate size of each record (this lists what is contained in the db and how many bytes are occupied) record_bytes = 2 encoding_bytes = 4 kp_xyz_entries = 3 * self.num_kp record_bytes += encoding_bytes * kp_xyz_entries encoding_bytes = 4 kp_uv_entries = 2 * self.num_kp record_bytes += encoding_bytes * kp_uv_entries cam_matrix_entries = 9 record_bytes += encoding_bytes * cam_matrix_entries image_bytes = self.image_size[0] * self.image_size[1] * 3 record_bytes += image_bytes hand_parts_bytes = self.image_size[0] * self.image_size[1] record_bytes += hand_parts_bytes kp_vis_bytes = self.num_kp record_bytes += kp_vis_bytes """ READ DATA ITEMS""" # Start reader reader = tf.FixedLengthRecordReader(header_bytes=0, record_bytes=record_bytes) _, value = reader.read( tf.train.string_input_producer([self.path_to_db])) # decode to floats bytes_read = 0 data_dict = dict() record_bytes_float32 = tf.decode_raw(value, tf.float32) # 1. Read keypoint xyz keypoint_xyz = tf.reshape( tf.slice(record_bytes_float32, [bytes_read // 4], [kp_xyz_entries]), [self.num_kp, 3]) bytes_read += encoding_bytes * kp_xyz_entries # calculate palm coord if not self.use_wrist_coord: palm_coord_l = tf.expand_dims( 0.5 * (keypoint_xyz[0, :] + keypoint_xyz[12, :]), 0) palm_coord_r = tf.expand_dims( 0.5 * (keypoint_xyz[21, :] + keypoint_xyz[33, :]), 0) keypoint_xyz = tf.concat([ palm_coord_l, keypoint_xyz[1:21, :], palm_coord_r, keypoint_xyz[-20:, :] ], 0) # 2. Read keypoint uv keypoint_uv = tf.cast( tf.reshape( tf.slice(record_bytes_float32, [bytes_read // 4], [kp_uv_entries]), [self.num_kp, 2]), tf.int32) bytes_read += encoding_bytes * kp_uv_entries keypoint_uv = tf.cast(keypoint_uv, tf.float32) # calculate palm coord if not self.use_wrist_coord: palm_coord_uv_l = tf.expand_dims( 0.5 * (keypoint_uv[0, :] + keypoint_uv[12, :]), 0) palm_coord_uv_r = tf.expand_dims( 0.5 * (keypoint_uv[21, :] + keypoint_uv[33, :]), 0) keypoint_uv = tf.concat([ palm_coord_uv_l, keypoint_uv[1:21, :], palm_coord_uv_r, keypoint_uv[-20:, :] ], 0) if self.coord_uv_noise: noise = tf.truncated_normal([42, 2], mean=0.0, stddev=self.coord_uv_noise_sigma) keypoint_uv += noise # 3. Read camera intrinsics cam_mat = tf.reshape( tf.slice(record_bytes_float32, [bytes_read // 4], [cam_matrix_entries]), [3, 3]) bytes_read += encoding_bytes * cam_matrix_entries data_dict['cam_mat'] = cam_mat # decode to uint8 bytes_read += 2 record_bytes_uint8 = tf.decode_raw(value, tf.uint8) # 4. Read image image = tf.reshape( tf.slice(record_bytes_uint8, [bytes_read], [image_bytes]), [self.image_size[0], self.image_size[1], 3]) image = tf.cast(image, tf.float32) bytes_read += image_bytes # subtract mean image = image / 255.0 - 0.5 #image = image / 255.0 if self.hue_aug: image = tf.image.random_hue(image, self.hue_aug_max) data_dict['image'] = image # 5. Read mask hand_parts_mask = tf.reshape( tf.slice(record_bytes_uint8, [bytes_read], [hand_parts_bytes]), [self.image_size[0], self.image_size[1]]) hand_parts_mask = tf.cast(hand_parts_mask, tf.int32) bytes_read += hand_parts_bytes ''' data_dict['hand_parts'] = hand_parts_mask hand_mask = tf.greater(hand_parts_mask, 1)#@@yafei: 1->0 bg_mask = tf.logical_not(hand_mask) data_dict['hand_mask'] = tf.cast(tf.stack([bg_mask, hand_mask], 2), tf.int32) ''' # 6. Read visibilty keypoint_vis_val = tf.reshape( tf.slice(record_bytes_uint8, [bytes_read], [kp_vis_bytes]), [self.num_kp]) keypoint_vis = tf.cast(keypoint_vis_val, tf.bool) bytes_read += kp_vis_bytes data_dict['vis_val'] = keypoint_vis_val # calculate palm visibility if not self.use_wrist_coord: palm_vis_l = tf.expand_dims( tf.logical_or(keypoint_vis[0], keypoint_vis[12]), 0) palm_vis_r = tf.expand_dims( tf.logical_or(keypoint_vis[21], keypoint_vis[33]), 0) keypoint_vis = tf.concat([ palm_vis_l, keypoint_vis[1:21], palm_vis_r, keypoint_vis[-20:] ], 0) #data_dict['keypoint_vis'] = keypoint_vis assert bytes_read == record_bytes, "Doesnt add up." """ DEPENDENT DATA ITEMS: SUBSET of 21 keypoints""" ''' # figure out dominant hand by analysis of the segmentation mask one_map, zero_map = tf.ones_like(hand_parts_mask), tf.zeros_like(hand_parts_mask) cond_l = tf.logical_and(tf.greater(hand_parts_mask, one_map), tf.less(hand_parts_mask, one_map*18))#@@yafei_0607:onemap*18 -> onemap cond_r = tf.greater(hand_parts_mask, one_map*17)#@@yafei_0607:onemap*17 -> zeromap hand_map_l = tf.where(cond_l, one_map, zero_map) hand_map_r = tf.where(cond_r, one_map, zero_map) num_px_left_hand = tf.reduce_sum(hand_map_l) num_px_right_hand = tf.reduce_sum(hand_map_r) # PRODUCE the 21 subset using the segmentation masks # We only deal with the more prominent hand for each frame and discard the second set of keypoints kp_coord_xyz_left = keypoint_xyz[:21, :] kp_coord_xyz_right = keypoint_xyz[-21:, :] cond_left = tf.logical_and(tf.cast(tf.ones_like(kp_coord_xyz_left), tf.bool), tf.greater(num_px_left_hand, num_px_right_hand)) kp_coord_xyz21 = tf.where(cond_left, kp_coord_xyz_left, kp_coord_xyz_right) hand_side = tf.where(tf.greater(num_px_left_hand, num_px_right_hand), tf.constant(0, dtype=tf.int32), tf.constant(1, dtype=tf.int32)) # left hand = 0; right hand = 1 data_dict['hand_side'] = tf.one_hot(hand_side, depth=2, on_value=1.0, off_value=0.0, dtype=tf.float32) data_dict['keypoint_xyz21'] = kp_coord_xyz21 ''' # GANerated Dataset has only left hand data_dict['hand_side'] = tf.constant([1.0, 0.0]) data_dict['keypoint_xyz21'] = keypoint_xyz[:21, :] kp_coord_xyz21 = keypoint_xyz[:21, :] # make coords relative to root joint kp_coord_xyz_root = kp_coord_xyz21[0, :] # this is the palm coord kp_coord_xyz21_rel = kp_coord_xyz21 - kp_coord_xyz_root # relative coords in metric coords ''' index_root_bone_length = tf.sqrt(tf.reduce_sum(tf.square(kp_coord_xyz21_rel[12, :] - kp_coord_xyz21_rel[11, :]))) data_dict['keypoint_scale'] = index_root_bone_length data_dict['keypoint_xyz21_normed'] = kp_coord_xyz21_rel / index_root_bone_length # normalized by length of 12->11 # calculate local coordinates kp_coord_xyz21_local = bone_rel_trafo(data_dict['keypoint_xyz21_normed']) kp_coord_xyz21_local = tf.squeeze(kp_coord_xyz21_local) data_dict['keypoint_xyz21_local'] = kp_coord_xyz21_local ''' # calculate viewpoint and coords in canonical coordinates data_dict[ 'keypoint_xyz21_normed'] = kp_coord_xyz21_rel #keypoint_xyz[:21, :] kp_coord_xyz21_rel_can, rot_mat = canonical_trafo( data_dict['keypoint_xyz21_normed']) kp_coord_xyz21_rel_can, rot_mat = tf.squeeze( kp_coord_xyz21_rel_can), tf.squeeze(rot_mat) #kp_coord_xyz21_rel_can = flip_right_hand(kp_coord_xyz21_rel_can, tf.logical_not(cond_left)) data_dict['keypoint_xyz21_can'] = kp_coord_xyz21_rel_can data_dict['rot_mat'] = tf.matrix_inverse(rot_mat) ''' # Set of 21 for visibility keypoint_vis_left = keypoint_vis[:21] keypoint_vis_right = keypoint_vis[-21:] keypoint_vis21 = tf.where(cond_left[:, 0], keypoint_vis_left, keypoint_vis_right) data_dict['keypint_vis21'] = keypoint_vis21 ''' keypoint_vis21 = keypoint_vis[:21] data_dict['keypoint_vis21'] = keypoint_vis21 #data_dict['keypoint_vis21'] = tf.cast(tf.ones_like(keypoint_vis21),tf.bool) ''' # Set of 21 for UV coordinates keypoint_uv_left = keypoint_uv[:21, :] keypoint_uv_right = keypoint_uv[-21:, :] keypoint_uv21 = tf.where(cond_left[:, :2], keypoint_uv_left, keypoint_uv_right) data_dict['keypoint_uv21'] = keypoint_uv21 ''' keypoint_uv21 = keypoint_uv[:21, :] data_dict['keypoint_uv21'] = keypoint_uv21 """ DEPENDENT DATA ITEMS: HAND CROP """ if self.hand_crop: crop_center = keypoint_uv21[12, ::-1] # catch problem, when no valid kp available (happens almost never) crop_center = tf.cond(tf.reduce_all(tf.is_finite(crop_center)), lambda: crop_center, lambda: tf.constant([0.0, 0.0])) crop_center.set_shape([ 2, ]) if self.crop_center_noise: noise = tf.truncated_normal( [2], mean=0.0, stddev=self.crop_center_noise_sigma) crop_center += noise crop_scale_noise = tf.constant(1.0) if self.crop_scale_noise: crop_scale_noise = tf.squeeze( tf.random_uniform([1], minval=1.0, maxval=1.2)) # select visible coords only kp_coord_h = tf.boolean_mask(keypoint_uv21[:, 1], keypoint_vis21) kp_coord_w = tf.boolean_mask(keypoint_uv21[:, 0], keypoint_vis21) kp_coord_hw = tf.stack([kp_coord_h, kp_coord_w], 1) # determine size of crop (measure spatial extend of hw coords first) min_coord = tf.maximum(tf.reduce_min(kp_coord_hw, 0), 0.0) max_coord = tf.minimum(tf.reduce_max(kp_coord_hw, 0), self.image_size) # find out larger distance wrt the center of crop crop_size_best = 2 * tf.maximum(max_coord - crop_center, crop_center - min_coord) crop_size_best = tf.reduce_max(crop_size_best) crop_size_best = tf.minimum(tf.maximum(crop_size_best, 50.0), 500.0) # catch problem, when no valid kp available crop_size_best = tf.cond( tf.reduce_all(tf.is_finite(crop_size_best)), lambda: crop_size_best, lambda: tf.constant(200.0)) crop_size_best.set_shape([]) # calculate necessary scaling scale = tf.cast(self.crop_size, tf.float32) / crop_size_best scale = tf.minimum(tf.maximum(scale, 1.0), 10.0) scale *= crop_scale_noise data_dict['crop_scale'] = scale if self.crop_offset_noise: noise = tf.truncated_normal( [2], mean=0.0, stddev=self.crop_offset_noise_sigma) crop_center += noise # Crop image img_crop = crop_image_from_xy(tf.expand_dims(image, 0), crop_center, self.crop_size, scale) #data_dict['image_crop'] = tf.squeeze(img_crop) # Modify uv21 coordinates crop_center_float = tf.cast(crop_center, tf.float32) keypoint_uv21_u = (keypoint_uv21[:, 0] - crop_center_float[1] ) * scale + self.crop_size // 2 keypoint_uv21_v = (keypoint_uv21[:, 1] - crop_center_float[0] ) * scale + self.crop_size // 2 keypoint_uv21 = tf.stack([keypoint_uv21_u, keypoint_uv21_v], 1) data_dict['keypoint_uv21'] = keypoint_uv21 # Modify camera intrinsics scale = tf.reshape(scale, [ 1, ]) scale_matrix = tf.dynamic_stitch([ [0], [1], [2], [3], [4], [5], [6], [7], [8] ], [scale, [0.0], [0.0], [0.0], scale, [0.0], [0.0], [0.0], [1.0]]) scale_matrix = tf.reshape(scale_matrix, [3, 3]) crop_center_float = tf.cast(crop_center, tf.float32) trans1 = crop_center_float[0] * scale - self.crop_size // 2 trans2 = crop_center_float[1] * scale - self.crop_size // 2 trans1 = tf.reshape(trans1, [ 1, ]) trans2 = tf.reshape(trans2, [ 1, ]) trans_matrix = tf.dynamic_stitch( [[0], [1], [2], [3], [4], [5], [6], [7], [8]], [[1.0], [0.0], -trans2, [0.0], [1.0], -trans1, [0.0], [0.0], [1.0]]) trans_matrix = tf.reshape(trans_matrix, [3, 3]) data_dict['cam_mat'] = tf.matmul(trans_matrix, tf.matmul(scale_matrix, cam_mat)) """ DEPENDENT DATA ITEMS: Scoremap from the SUBSET of 21 keypoints""" # create scoremaps from the subset of 2D annoataion keypoint_hw21 = tf.stack([keypoint_uv21[:, 1], keypoint_uv21[:, 0]], -1) scoremap_size = self.image_size if self.hand_crop: scoremap_size = (self.crop_size, self.crop_size) scoremap = self.create_multiple_gaussian_map(keypoint_hw21, scoremap_size, self.sigma, valid_vec=keypoint_vis21) if self.scoremap_dropout: scoremap = tf.nn.dropout(scoremap, self.scoremap_dropout_prob, noise_shape=[1, 1, 21]) scoremap *= self.scoremap_dropout_prob data_dict['scoremap'] = scoremap if self.scale_to_size: image, keypoint_uv21, keypoint_vis21 = data_dict[ 'image'], data_dict['keypoint_uv21'], data_dict[ 'keypoint_vis21'] s = image.get_shape().as_list() image = tf.image.resize_images(image, self.scale_target_size) scale = (self.scale_target_size[0] / float(s[0]), self.scale_target_size[1] / float(s[1])) keypoint_uv21 = tf.stack([ keypoint_uv21[:, 0] * scale[1], keypoint_uv21[:, 1] * scale[0] ], 1) data_dict = dict( ) # delete everything else because the scaling makes the data invalid anyway data_dict['image'] = image data_dict['keypoint_uv21'] = keypoint_uv21 data_dict['keypoint_vis21'] = keypoint_vis21 elif self.random_crop_to_size: tensor_stack = tf.concat([ data_dict['image'], tf.expand_dims(tf.cast(data_dict['hand_parts'], tf.float32), -1), tf.cast(data_dict['hand_mask'], tf.float32) ], 2) s = tensor_stack.get_shape().as_list() tensor_stack_cropped = tf.random_crop( tensor_stack, [self.random_crop_size, self.random_crop_size, s[2]]) data_dict = dict( ) # delete everything else because the random cropping makes the data invalid anyway data_dict['image'], data_dict['scoremap'], data_dict['keypoint_vis21'] = tensor_stack_cropped[:, :, :3],\ tf.cast(scoremap, tf.float32),\ tf.cast(keypoint_vis21, tf.int32) names, tensors = zip(*data_dict.items()) print(names) if self.shuffle: tensors = tf.train.shuffle_batch_join([tensors], batch_size=self.batch_size, capacity=100, min_after_dequeue=50, enqueue_many=False) else: tensors = tf.train.batch_join([tensors], batch_size=self.batch_size, capacity=100, enqueue_many=False) return dict(zip(names, tensors))
combination2 = tf.matmul(tf.matmul(tf.transpose(H), H), xk) concatenation = tf.concat([combination1, xk, combination2, vk], 0, name="Concatenate") Cal1 = tf.matmul(w1k, concatenation) + b1k zk = tf.nn.relu(Cal1, name="Zk") cal2 = tf.matmul(w2k, zk) + b2k t = 0.5 xk_1 = -1 + tf.nn.relu(cal2 + t) / abs(t) - tf.nn.relu(cal2 - t) / abs(t) vk_1 = tf.matmul(w3k, zk) + b3k #Define loss function and backpropagation algorithm xwave_part1 = tf.matrix_inverse(tf.matmul(tf.transpose(H), H)) xwave_part2 = tf.matmul(xwave_part1, tf.transpose(H)) xwave = tf.matmul(xwave_part2, y) lossfunction = tf.log( tf.reduce_sum(tf.squared_difference(x, xk_1)) / tf.reduce_sum(tf.squared_difference(x, xwave))) train_step = tf.train.AdamOptimizer(0.001).minimize(lossfunction) #Create a session to run Tensorflow sess = tf.Session() init_op = tf.global_variables_initializer() sess.run(init_op) print(sess.run(w1k)) print(sess.run(w2k)) print(sess.run(w3k))
avg_dim_variance, shape=[COMPONENTS, DIMENSIONS, DIMENSIONS]) initial_weights = tf.placeholder_with_default(tf.cast( tf.constant(1.0 / COMPONENTS, shape=[COMPONENTS]), tf.float64), shape=[COMPONENTS]) # trainable variables: component means, covariances, and weights means = tf.Variable(initial_means, dtype=tf.float64) covariances = tf.Variable(initial_covariances, dtype=tf.float64) weights = tf.Variable(initial_weights, dtype=tf.float64) # E-step: recomputing responsibilities with respect to the current parameter values differences = tf.subtract(tf.expand_dims(input, 0), tf.expand_dims(means, 1)) diff_times_inv_cov = tf.matmul(differences, tf.matrix_inverse(covariances)) sum_sq_dist_times_inv_cov = tf.reduce_sum(diff_times_inv_cov * differences, 2) log_coefficients = tf.expand_dims( ln2piD + tf.log(tf.matrix_determinant(covariances)), 1) log_components = -0.5 * (log_coefficients + sum_sq_dist_times_inv_cov) log_weighted = log_components + tf.expand_dims(tf.log(weights), 1) # Numerator # print("Log weighted", log_weighted.shape) log_shift = tf.expand_dims(tf.reduce_max(log_weighted, 0), 0) # print('Log shift shape', log_shift.shape) exp_log_shifted = tf.exp(log_weighted - log_shift) # print('Exp log shift shape', exp_log_shifted.shape) exp_log_shifted_sum = tf.reduce_sum(exp_log_shifted, 0) # print('Exp log shifted sum', exp_log_shifted_sum.shape)
def get_update_UVW_ops_for_2sgd_sals(self, rho): ''' See 2SGD/SALS algorithms in Expected Tensor Decomp paper ''' def gamma(A, B): ATA = tf.matmul(A, A, transpose_a=True) # A^T * A BTB = tf.matmul(B, B, transpose_a=True) # B^T * B return tf.multiply(ATA, BTB) # hadamard product of A^T*A and B^T*B X = self.X_t t = self.global_step + 1 alpha = .25 # smaller => decays slower (more quickly get updates from the gradients) #batch_size = 1. / 500. * tf.sqrt(tf.cast(X.shape[0], tf.float32)) batch_size = 1. eta_t = batch_size / (1. + t**alpha) X_VW = tf.Variable(tf.constant(0.0, shape=[self.shape[0], self.rank])) XU_W = tf.Variable(tf.constant(0.0, shape=[self.shape[1], self.rank])) XUV_ = tf.Variable(tf.constant(0.0, shape=[self.shape[2], self.rank])) def assign_zeroes(): a1 = tf.assign(X_VW, tf.constant(0.0, shape=[self.shape[0], self.rank])) a2 = tf.assign(XU_W, tf.constant(0.0, shape=[self.shape[1], self.rank])) a3 = tf.assign(XUV_, tf.constant(0.0, shape=[self.shape[2], self.rank])) return tf.group(a1, a2, a3) def body( ix, X_VW2, XU_W2, XUV_2, ): ijk = tf.gather(X.indices, ix) val = tf.gather(X.values, ix) Ui = tf.gather(self.U, ijk[0]) # R-dimensional Vj = tf.gather(self.V, ijk[1]) Wk = tf.gather(self.W, ijk[2]) Xijk_Vj_Wk = val * tf.multiply(Vj, Wk) Xijk_Ui_Wk = val * tf.multiply(Ui, Wk) Xijk_Ui_Vj = val * tf.multiply(Ui, Vj) r1 = tf.scatter_add( X_VW, ijk[0], Xijk_Vj_Wk ) # add Xijk(Vj*Wk) to X_VW(i,:) (as vectors in \mathbb{R}^R) r2 = tf.scatter_add(XU_W, ijk[1], Xijk_Ui_Wk) r3 = tf.scatter_add(XUV_, ijk[2], Xijk_Ui_Vj) new_ix = tf.add(ix, tf.constant(1)) return new_ix, r1, r2, r3 N = tf.shape(X.values)[0] def cond(ix, *args): ass = tf.cond(tf.equal(ix, tf.constant(0)), lambda: assign_zeroes(), lambda: tf.no_op()) with tf.control_dependencies([ass]): return tf.less(ix, N) printer = tf.Print(XUV_, [X_VW, XU_W, XUV_]) ix = tf.Variable(tf.constant( 0, dtype=tf.int32)) # which index/value we're looking at _, X_VW, XU_W, XUV_ = tf.while_loop(cond, body, [ix, X_VW, XU_W, XUV_]) U = self.U V = self.V W = self.W self.D1 = X_VW self.D2 = XU_W self.D3 = XUV_ gamma_rho = gamma(V, W) + rho * tf.eye(self.rank) self.gr1 = gamma_rho inv_gamma_rho = tf.matrix_inverse(gamma_rho) self.gr1inv = gamma_rho self.grad_value_U = tf.matmul(X_VW, inv_gamma_rho) gamma_rho = gamma(U, W) + rho * tf.eye(self.rank) self.gr2 = gamma_rho inv_gamma_rho = tf.matrix_inverse(gamma_rho) self.grad_value_V = tf.matmul(XU_W, inv_gamma_rho) gamma_rho = gamma(U, V) + rho * tf.eye(self.rank) self.gr3 = gamma_rho inv_gamma_rho = tf.matrix_inverse(gamma_rho) self.grad_value_W = tf.matmul(XUV_, inv_gamma_rho) update_U_op = tf.assign(U, (1 - eta_t) * U + eta_t * self.grad_value_U) update_V_op = tf.assign(V, (1 - eta_t) * V + eta_t * self.grad_value_V) update_W_op = tf.assign(W, (1 - eta_t) * W + eta_t * self.grad_value_W) return [update_U_op, update_V_op, update_W_op]
def __init__(self, seq_length, emb_dim, hidden_dim, embeddings, emb_train): ## Define hyperparameters ## note: embedding_dim and hidden_dim are both 300, used interchangeably self.embedding_dim = emb_dim self.dim = hidden_dim self.sequence_length = seq_length ## Define the placeholders self.premise_x = tf.placeholder(tf.int32, [None, self.sequence_length]) self.hypothesis_x = tf.placeholder(tf.int32, [None, self.sequence_length]) self.y = tf.placeholder(tf.int32, [None]) self.keep_rate_ph = tf.placeholder(tf.float32, []) ## Define parameters self.E = tf.Variable(embeddings, trainable=emb_train) self.W_mlp = tf.Variable( tf.random_normal([self.dim * 8, self.dim], stddev=0.1)) self.b_mlp = tf.Variable(tf.random_normal([self.dim], stddev=0.1)) self.W_cl = tf.Variable(tf.random_normal([self.dim, 3], stddev=0.1)) self.b_cl = tf.Variable(tf.random_normal([3], stddev=0.1)) ## Function for embedding lookup and dropout at embedding layer def emb_drop(x): emb = tf.nn.embedding_lookup(self.E, x) emb_drop = tf.nn.dropout(emb, self.keep_rate_ph) return emb_drop # Get lengths of unpadded sentences prem_seq_lengths, mask_prem = blocks.length(self.premise_x) hyp_seq_lengths, mask_hyp = blocks.length(self.hypothesis_x) ### First biLSTM layer ### premise_in = emb_drop(self.premise_x) hypothesis_in = emb_drop(self.hypothesis_x) premise_outs, c1 = blocks.biLSTM(premise_in, dim=self.dim, seq_len=prem_seq_lengths, name='premise') hypothesis_outs, c2 = blocks.biLSTM(hypothesis_in, dim=self.dim, seq_len=hyp_seq_lengths, name='hypothesis') premise_bi = tf.concat(premise_outs, axis=2) hypothesis_bi = tf.concat(hypothesis_outs, axis=2) premise_list = tf.unstack(premise_bi, axis=1) hypothesis_list = tf.unstack(hypothesis_bi, axis=1) ### Attention ### scores_all = [] premise_attn = [] alphas = [] for i in range(self.sequence_length): scores_i_list = [] for j in range(self.sequence_length): score_ij = tf.reduce_sum(tf.multiply(premise_list[i], hypothesis_list[j]), 1, keep_dims=True) scores_i_list.append(score_ij) scores_i = tf.stack(scores_i_list, axis=1) alpha_i = blocks.masked_softmax(scores_i, mask_hyp) a_tilde_i = tf.reduce_sum(tf.multiply(alpha_i, hypothesis_bi), 1) premise_attn.append(a_tilde_i) scores_all.append(scores_i) alphas.append(alpha_i) scores_stack = tf.stack(scores_all, axis=2) scores_list = tf.unstack(scores_stack, axis=1) hypothesis_attn = [] betas = [] for j in range(self.sequence_length): scores_j = scores_list[j] beta_j = blocks.masked_softmax(scores_j, mask_prem) b_tilde_j = tf.reduce_sum(tf.multiply(beta_j, premise_bi), 1) hypothesis_attn.append(b_tilde_j) betas.append(beta_j) # Make attention-weighted sentence representations into one tensor, premise_attns = tf.stack(premise_attn, axis=1) hypothesis_attns = tf.stack(hypothesis_attn, axis=1) # For making attention plots, self.alpha_s = tf.stack(alphas, axis=2) self.beta_s = tf.stack(betas, axis=2) ### Subcomponent Inference ### prem_diff = tf.subtract(premise_bi, premise_attns) prem_mul = tf.multiply(premise_bi, premise_attns) hyp_diff = tf.subtract(hypothesis_bi, hypothesis_attns) hyp_mul = tf.multiply(hypothesis_bi, hypothesis_attns) m_a = tf.concat([premise_bi, premise_attns, prem_diff, prem_mul], 2) m_b = tf.concat([hypothesis_bi, hypothesis_attns, hyp_diff, hyp_mul], 2) ### Inference Composition ### v1_outs, c3 = blocks.biLSTM(m_a, dim=self.dim, seq_len=prem_seq_lengths, name='v1') v2_outs, c4 = blocks.biLSTM(m_b, dim=self.dim, seq_len=hyp_seq_lengths, name='v2') v1_bi = tf.concat(v1_outs, axis=2) v2_bi = tf.concat(v2_outs, axis=2) ### Pooling Layer ### v_1_sum = tf.reduce_sum(v1_bi, 1) v_1_ave = tf.div( v_1_sum, tf.expand_dims(tf.cast(prem_seq_lengths, tf.float32), -1)) v_2_sum = tf.reduce_sum(v2_bi, 1) v_2_ave = tf.div( v_2_sum, tf.expand_dims(tf.cast(hyp_seq_lengths, tf.float32), -1)) v_1_max = tf.reduce_max(v1_bi, 1) v_2_max = tf.reduce_max(v2_bi, 1) v = tf.concat([v_1_ave, v_2_ave, v_1_max, v_2_max], 1) # MLP layer h_mlp = tf.nn.tanh(tf.matmul(v, self.W_mlp) + self.b_mlp) ############### MY CODE STARTS ##### # Define layer size self.bow_layer_size = 300 # LSTM layer (final layer of the original esim model) h_fc1 = h_mlp # Bag-of-word input (concat word embeddings) self.sentence_dim = self.dim * self.sequence_length self.input_dim = self.sentence_dim * 2 bow_pre = premise_in bow_hyp = hypothesis_in bag_of_word_in = tf.concat([bow_pre, bow_hyp], 1) bag_of_word_in = tf.reshape(bag_of_word_in, [-1, self.input_dim]) # Bag-of-word input layer params W_fc2 = tf.Variable( tf.random_normal([self.input_dim, self.bow_layer_size], stddev=0.1)) b_fc2 = tf.Variable(tf.zeros([self.bow_layer_size])) h_fc2 = tf.nn.sigmoid(tf.matmul(bag_of_word_in, W_fc2) + b_fc2) # Bag-of-word output layer params self.W_cl = tf.Variable( tf.random_normal([self.dim + self.bow_layer_size, 3], stddev=0.1)) self.b_cl = tf.Variable(tf.random_normal([3], stddev=0.1)) # Compute prediction using [h_fc1, 0(pad)] pad = tf.zeros_like(h_fc2, tf.float32) yconv_contact_pred = tf.nn.dropout(tf.concat([h_fc1, pad], 1), self.keep_rate_ph) y_conv_pred = tf.matmul(yconv_contact_pred, self.W_cl) + self.b_cl self.logits = y_conv_pred # Prediction # Compute loss using [h_fc1, h_fc2] and [0(pad2), h_fc2] pad2 = tf.zeros_like(h_fc1, tf.float32) yconv_contact_H = tf.nn.dropout(tf.concat([pad2, h_fc2], 1), self.keep_rate_ph) y_conv_H = tf.matmul(yconv_contact_H, self.W_cl) + self.b_cl # get Fg yconv_contact_loss = tf.nn.dropout(tf.concat([h_fc1, h_fc2], 1), self.keep_rate_ph) y_conv_loss = tf.matmul(yconv_contact_loss, self.W_cl) + self.b_cl # get Fb ########################################################## # Warning: Here it may enconter invertible matrix issue # ########################################################## y_conv_loss = y_conv_loss - tf.matmul( tf.matmul(tf.matmul( y_conv_H, tf.matrix_inverse( tf.matmul(y_conv_H, y_conv_H, transpose_a=True))), y_conv_H, transpose_b=True), y_conv_loss) # get loss cost_logits = y_conv_loss self.total_cost = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( labels=self.y, logits=y_conv_loss)) # Cost
import tensorflow as tf import numpy as np A = tf.placeholder(dtype=tf.float64, shape=[2, 2]) b = tf.placeholder(dtype=tf.float64, shape=[2]) A_pow = tf.sin(A) A_relu = tf.nn.relu(A) A_inverse = tf.matrix_inverse(A) A_T = tf.transpose(A) b_diag = tf.diag(b) I = tf.eye(6) init = tf.global_variables_initializer() sess = tf.Session() sess.run(init) print(sess.run(A_pow, feed_dict={A: [[1, 2], [-1, 1]]})) print(sess.run(A_relu, feed_dict={A: [[1, 2], [-1, 1]], b: [1, 1]})) print(sess.run(A_inverse, feed_dict={A: [[1, 2], [-1, 1]], b: [1, 1]})) print(sess.run([b_diag, I], feed_dict={A: [[1, 2], [-1, 1]], b: [1, 1]}))
# Matrix rank print("rank(A) = ") print(np.linalg.matrix_rank(A.eval())) print("rank(C) = ") print(np.linalg.matrix_rank(C.eval())) # Transpose print("tran(A) = ") print(tf.matrix_transpose(A).eval()) print("tran(B) = ") print(tf.matrix_transpose(B).eval()) # Inverse print("inv(A) = ") print(tf.matrix_inverse(A).eval()) # Inverse of diagonal matrix has diag elements of the reciprocal of diag elements B print("inv(B) = ") print(tf.matrix_inverse(B).eval()) print("inv(C) = ") # since C has rank 1, this will cause error try: print(tf.matrix_inverse(C).eval()) except: print("C is not invertible") # Product of a matrix and its inverse is an identity (non-singular) print("A*inv(A) = Eye(2)") print(tf.matmul(A, tf.matrix_inverse(A)).eval()) # Element-wise multiplication print("elem(A)*elem(B) = ")
def QuantizedActiv(x, nbit=2): """ Quantize activation. Args: x (tf.Tensor): a 4D tensor. nbit (int): number of bits of quantized activation. Defaults to 2. Returns: tf.Tensor with attribute `variables`. Variable Names: * ``basis``: basis of quantized activation. Note: About multi-GPU training: moving averages across GPUs are not aggregated. Batch statistics are computed by main training tower. This is consistent with most frameworks. """ init_basis = [(NORM_PPF_0_75 * 2 / (2**nbit - 1)) * (2.**i) for i in range(nbit)] init_basis = tf.constant_initializer(init_basis) bit_dims = [nbit, 1] num_levels = 2**nbit # initialize level multiplier init_level_multiplier = [] for i in range(0, num_levels): level_multiplier_i = [0. for j in range(nbit)] level_number = i for j in range(nbit): level_multiplier_i[j] = float(level_number % 2) level_number = level_number // 2 init_level_multiplier.append(level_multiplier_i) # initialize threshold multiplier init_thrs_multiplier = [] for i in range(1, num_levels): thrs_multiplier_i = [0. for j in range(num_levels)] thrs_multiplier_i[i - 1] = 0.5 thrs_multiplier_i[i] = 0.5 init_thrs_multiplier.append(thrs_multiplier_i) with tf.variable_scope('ActivationQuantization'): basis = tf.get_variable('basis', bit_dims, tf.float32, initializer=init_basis, trainable=False) ctx = get_current_tower_context() # current tower context # calculate levels and sort level_codes = tf.constant(init_level_multiplier) levels = tf.matmul(level_codes, basis) levels, sort_id = tf.nn.top_k(tf.transpose(levels, [1, 0]), num_levels) levels = tf.reverse(levels, [-1]) sort_id = tf.reverse(sort_id, [-1]) levels = tf.transpose(levels, [1, 0]) sort_id = tf.transpose(sort_id, [1, 0]) # calculate threshold thrs_multiplier = tf.constant(init_thrs_multiplier) thrs = tf.matmul(thrs_multiplier, levels) # calculate output y and its binary code y = tf.zeros_like(x) # output reshape_x = tf.reshape(x, [-1]) zero_dims = tf.stack([tf.shape(reshape_x)[0], nbit]) bits_y = tf.fill(zero_dims, 0.) zero_y = tf.zeros_like(x) zero_bits_y = tf.fill(zero_dims, 0.) for i in range(num_levels - 1): g = tf.greater(x, thrs[i]) y = tf.where(g, zero_y + levels[i + 1], y) bits_y = tf.where(tf.reshape(g, [-1]), zero_bits_y + level_codes[sort_id[i + 1][0]], bits_y) # training if ctx.is_main_training_tower: BT = tf.matrix_transpose(bits_y) # calculate BTxB BTxB = [] for i in range(nbit): for j in range(nbit): BTxBij = tf.multiply(BT[i], BT[j]) BTxBij = tf.reduce_sum(BTxBij) BTxB.append(BTxBij) BTxB = tf.reshape(tf.stack(values=BTxB), [nbit, nbit]) BTxB_inv = tf.matrix_inverse(BTxB) # calculate BTxX BTxX = [] for i in range(nbit): BTxXi0 = tf.multiply(BT[i], reshape_x) BTxXi0 = tf.reduce_sum(BTxXi0) BTxX.append(BTxXi0) BTxX = tf.reshape(tf.stack(values=BTxX), [nbit, 1]) new_basis = tf.matmul(BTxB_inv, BTxX) # calculate new basis # create moving averages op updata_moving_basis = moving_averages.assign_moving_average( basis, new_basis, MOVING_AVERAGES_FACTOR) add_model_variable(basis) tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, updata_moving_basis) for i in range(nbit): tf.summary.scalar('basis%d' % i, new_basis[i][0]) x_clip = tf.minimum(x, levels[num_levels - 1]) # gradient clip y = x_clip + tf.stop_gradient(-x_clip) + tf.stop_gradient( y) # gradient: y=clip(x) y.variables = VariableHolder(basis=basis) return y
import numpy as np import tensorflow as tf from sklearn.datasets import fetch_california_housing housing = fetch_california_housing() m , n = housing.data.shape housing_data_plus_bias = np.c_[np.ones((m , 1)), housing.data] X = tf.constant(housing_data_plus_bias, dtype=tf.float32, name='X') y = tf.constant(housing.target.reshape(-1,1), dtype=tf.float32, name='y') XT = tf.transpose(X) theta = tf.matmul(tf.matmul(tf.matrix_inverse(tf.matmul(XT, X)), XT), y) with tf.Session() as sess: theta_value = theta.eval()
def build_model(self): # Define prediction rnn rnn_outputs_hypo, final_state_hypo = lstm_layer( self.e_hypo, self.lstm_size, self.batch_size, self.seq_len_hypo, "hypo") rnn_outputs_prem, final_state_prem = lstm_layer( self.e_prem, self.lstm_size, self.batch_size, self.seq_len_prem, "prem") last_output_hypo, alphas_hypo = attention_layer(self.attention_size, rnn_outputs_hypo, "encoder_hypo", sparse=self.sparse) last_output_prem, alphas_prem = attention_layer(self.attention_size, rnn_outputs_prem, "encoder_prem", sparse=self.sparse) self.alphas_hypo = alphas_hypo self.alphas_prem = alphas_prem # last_output = tf.nn.dropout(last_output, self.keep_probs) # Define key-word model rnn kwm_rnn_outputs_hypo, kwm_final_state_hypo = lstm_layer( self.e_hypo, self.lstm_size, self.batch_size, self.seq_len_hypo, scope="kwm_hypo") kwm_rnn_outputs_prem, kwm_final_state_prem = lstm_layer( self.e_prem, self.lstm_size, self.batch_size, self.seq_len_prem, scope="kwm_prem") kwm_last_output_hypo, kwm_alphas_hypo = attention_layer( self.attention_size, kwm_rnn_outputs_hypo, "kwm_encoder_hypo", sparse=self.sparse) kwm_last_output_prem, kwm_alphas_prem = attention_layer( self.attention_size, kwm_rnn_outputs_prem, "kwm_encoder_prem", sparse=self.sparse) last_output = tf.concat([last_output_hypo, last_output_prem], axis=1) kwm_last_output = tf.concat( [kwm_last_output_hypo, kwm_last_output_prem], axis=1) ############################ # Hex ######################### h_fc1 = last_output h_fc2 = kwm_last_output # Hex layer definition """ self.W_cl_1 = tf.Variable(tf.random_normal([self.dim, 3], stddev=0.1)) self.W_cl_2 = tf.Variable(tf.random_normal([1200, 3]), trainable=True) self.b_cl = tf.Variable(tf.random_normal((3,)), trainable=True) self.W_cl = tf.concat([self.W_cl_1, self.W_cl_2], 0) """ # Compute prediction using [h_fc1, 0(pad)] pad = tf.zeros_like(h_fc2, tf.float32) # print(pad.shape) -> (?, 600) yconv_contact_pred = tf.nn.dropout(tf.concat([h_fc1, pad], 1), self.keep_probs) # y_conv_pred = tf.matmul(yconv_contact_pred, self.W_cl) + self.b_cl y_conv_pred = dense_layer(yconv_contact_pred, 3, name="conv_pred") self.logits = y_conv_pred # Prediction # Compute loss using [h_fc1, h_fc2] and [0(pad2), h_fc2] pad2 = tf.zeros_like(h_fc1, tf.float32) yconv_contact_H = tf.concat([pad2, h_fc2], 1) # Get Fg # y_conv_H = tf.matmul(yconv_contact_H, self.W_cl) + self.b_cl # get Fg y_conv_H = dense_layer(yconv_contact_H, 3, name="conv_H") yconv_contact_loss = tf.nn.dropout(tf.concat([h_fc1, h_fc2], 1), self.keep_probs) # Get Fb # y_conv_loss = tf.matmul(yconv_contact_loss, self.W_cl) + self.b_cl # get Fb y_conv_loss = dense_layer(yconv_contact_loss, 3, name="conv_loss") temp = tf.matmul(y_conv_H, y_conv_H, transpose_a=True) self.temp = temp y_conv_loss = y_conv_loss - tf.matmul( tf.matmul(tf.matmul(y_conv_H, tf.matrix_inverse(temp)), y_conv_H, transpose_b=True), y_conv_loss) # get loss self.logits = y_conv_loss self.y = tf.nn.softmax(self.logits) self.cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=tf.one_hot( self.y_holder, depth=3), logits=self.logits)) # Regularize kwm attention reg1 = get_reg(kwm_alphas_hypo, lam=self.lam, type=self.reg) reg2 = get_reg(kwm_alphas_prem, lam=self.lam, type=self.reg) self.cost += reg1 + reg2 self.optimizer = tf.train.GradientDescentOptimizer(self.learning_rate) self.train_op = self.optimizer.minimize(self.cost) self.accuracy = tf.reduce_mean( tf.cast(tf.equal(self.y_holder, tf.argmax(self.y, 1)), tf.float32))
def forward(self, inputs, grid, is_training=True, reuse=False): def preprocessing(inputs): dims = inputs.get_shape() if len(dims) == 3: inputs = tf.expand_dims(inputs, dim=0) mean_BGR = tf.reshape(self.mean_BGR, [1, 1, 1, 3]) inputs = inputs[:, :, :, ::-1] + mean_BGR return inputs ## -----------------------depth and normal FCN-------------------------- inputs = preprocessing(inputs) with slim.arg_scope( [slim.conv2d, slim.conv2d_transpose], activation_fn=tf.nn.relu, stride=1, padding='SAME', weights_initializer=weight_from_caffe(self.pretrain_weight), biases_initializer=bias_from_caffe(self.pretrain_weight)): with tf.variable_scope('fcn', reuse=reuse): ##---------------------vgg depth------------------------------------ conv1 = slim.repeat(inputs, 2, slim.conv2d, 64, [3, 3], scope='conv1') pool1 = slim.max_pool2d(conv1, [3, 3], stride=2, padding='SAME', scope='pool1') conv2 = slim.repeat(pool1, 2, slim.conv2d, 128, [3, 3], scope='conv2') pool2 = slim.max_pool2d(conv2, [3, 3], stride=2, padding='SAME', scope='pool2') conv3 = slim.repeat(pool2, 3, slim.conv2d, 256, [3, 3], scope='conv3') pool3 = slim.max_pool2d(conv3, [3, 3], stride=2, padding='SAME', scope='pool3') conv4 = slim.repeat(pool3, 3, slim.conv2d, 512, [3, 3], scope='conv4') pool4 = slim.max_pool2d(conv4, [3, 3], stride=1, padding='SAME', scope='pool4') conv5 = slim.repeat(pool4, 3, slim.conv2d, 512, [3, 3], rate=2, scope='conv5') pool5 = slim.max_pool2d(conv5, [3, 3], stride=1, padding='SAME', scope='pool5') pool5a = slim.avg_pool2d(pool5, [3, 3], stride=1, padding='SAME', scope='pool5a') fc6 = slim.conv2d(pool5a, 1024, [3, 3], stride=1, rate=12, scope='fc6') fc6 = slim.dropout(fc6, 0.5, is_training=is_training, scope='drop6') fc7 = slim.conv2d(fc6, 1024, [1, 1], scope='fc7') fc7 = slim.dropout(fc7, 0.5, is_training=is_training, scope='drop7') pool6_1x1 = slim.avg_pool2d(fc7, [61, 81], stride=[61, 81], padding='SAME', scope='pool6_1x1') pool6_1x1_norm = slim.unit_norm(pool6_1x1, dim=3, scope='pool6_1x1_norm_new') pool6_1x1_norm_scale = pool6_1x1_norm * 10 pool6_1x1_norm_upsample = tf.tile( pool6_1x1_norm_scale, [1, 61, 81, 1], name='pool6_1x1_norm_upsample') out = tf.concat([fc7, pool6_1x1_norm_upsample], axis=-1, name='out') out_reduce = slim.conv2d(out, 256, [1, 1], activation_fn=tf.nn.relu, stride=1, scope='out_reduce', padding='SAME', weights_initializer=weight_from_caffe( self.pretrain_weight), biases_initializer=bias_from_caffe( self.pretrain_weight)) out_conv = slim.conv2d(out_reduce, 256, [3, 3], activation_fn=tf.nn.relu, stride=1, scope='out_conv', padding='SAME', weights_initializer=weight_from_caffe( self.pretrain_weight), biases_initializer=bias_from_caffe( self.pretrain_weight)) out_conv_increase = slim.conv2d( out_conv, 1024, [1, 1], activation_fn=tf.nn.relu, stride=1, scope='out_conv_increase', padding='SAME', weights_initializer=weight_from_caffe( self.pretrain_weight), biases_initializer=bias_from_caffe(self.pretrain_weight)) fc8_nyu_depth = slim.conv2d(out_conv_increase, 1, [1, 1], activation_fn=None, scope='fc8_nyu_depth') fc8_upsample = tf.image.resize_images( fc8_nyu_depth, [self.crop_size_h, self.crop_size_w], method=0, align_corners=True) # ---------------------------------------vgg depth end --------------------------------------- ## ----------------- vgg norm--------------------------------------------------------------- conv1_norm = slim.repeat(inputs, 2, slim.conv2d, 64, [3, 3], scope='conv1_norm') pool1_norm = slim.max_pool2d(conv1_norm, [3, 3], stride=2, padding='SAME', scope='pool1_norm') conv2_norm = slim.repeat(pool1_norm, 2, slim.conv2d, 128, [3, 3], scope='conv2_norm') pool2_norm = slim.max_pool2d(conv2_norm, [3, 3], stride=2, padding='SAME', scope='pool2_norm') conv3_norm = slim.repeat(pool2_norm, 3, slim.conv2d, 256, [3, 3], scope='conv3_norm') pool3_norm = slim.max_pool2d(conv3_norm, [3, 3], stride=2, padding='SAME', scope='pool3_norm') conv4_norm = slim.repeat(pool3_norm, 3, slim.conv2d, 512, [3, 3], scope='conv4_norm') pool4_norm = slim.max_pool2d(conv4_norm, [3, 3], stride=1, padding='SAME', scope='pool4_norm') conv5_norm = slim.repeat(pool4_norm, 3, slim.conv2d, 512, [3, 3], rate=2, scope='conv5_norm') pool5_norm = slim.max_pool2d(conv5_norm, [3, 3], stride=1, padding='SAME', scope='pool5_norm') pool5a_norm = slim.avg_pool2d(pool5_norm, [3, 3], stride=1, padding='SAME', scope='pool5a_norm') fc6_norm = slim.conv2d(pool5a_norm, 1024, [3, 3], stride=1, rate=12, scope='fc6_norm') fc6_norm = slim.dropout(fc6_norm, 0.5, is_training=is_training, scope='drop6_norm') fc7_norm = slim.conv2d(fc6_norm, 1024, [1, 1], scope='fc7_norm') fc7_norm = slim.dropout(fc7_norm, 0.5, is_training=is_training, scope='drop7_norm') pool6_1x1_norm_new = slim.avg_pool2d( fc7_norm, [61, 81], stride=[61, 81], padding='SAME', scope='pool6_1x1_norm_new') pool6_1x1_norm_norm = slim.unit_norm( pool6_1x1_norm_new, dim=3, scope='pool6_1x1_norm_new') pool6_1x1_norm_scale_norm = pool6_1x1_norm_norm * 10 pool6_1x1_norm_upsample_norm = tf.tile( pool6_1x1_norm_scale_norm, [1, 61, 81, 1], name='pool6_1x1_norm_upsample') out_norm = tf.concat([fc7_norm, pool6_1x1_norm_upsample_norm], axis=-1, name='out_norm') fc8_nyu_norm_norm = slim.conv2d(out_norm, 3, [1, 1], activation_fn=None, scope='fc8_nyu_norm_norm') fc8_upsample_norm = tf.image.resize_images( fc8_nyu_norm_norm, [self.crop_size_h, self.crop_size_w], method=0, align_corners=True) fc8_upsample_norm = slim.unit_norm(fc8_upsample_norm, dim=3) # -------------------------------------vgg norm end--------------------------------------------- # ------------- depth to normal + norm refinement--------------------------------------------------- with tf.variable_scope('noise', reuse=reuse): fc8_upsample_norm = tf.squeeze(fc8_upsample_norm) fc8_upsample_norm = tf.reshape( fc8_upsample_norm, [self.batch_size, self.crop_size_h, self.crop_size_w, 3]) norm_matrix = tf.extract_image_patches( images=fc8_upsample_norm, ksizes=[1, self.k, self.k, 1], strides=[1, 1, 1, 1], rates=[1, self.rate, self.rate, 1], padding='SAME') matrix_c = tf.reshape(norm_matrix, [ self.batch_size, self.crop_size_h, self.crop_size_w, self.k * self.k, 3 ]) fc8_upsample_norm = tf.expand_dims(fc8_upsample_norm, axis=4) angle = tf.matmul(matrix_c, fc8_upsample_norm) valid_condition = tf.greater(angle, self.thresh) valid_condition_all = tf.tile(valid_condition, [1, 1, 1, 1, 3]) exp_depth = tf.exp(fc8_upsample * 0.69314718056) depth_repeat = tf.tile(exp_depth, [1, 1, 1, 3]) points = tf.multiply(grid, depth_repeat) point_matrix = tf.extract_image_patches( images=points, ksizes=[1, self.k, self.k, 1], strides=[1, 1, 1, 1], rates=[1, self.rate, self.rate, 1], padding='SAME') matrix_a = tf.reshape(point_matrix, [ self.batch_size, self.crop_size_h, self.crop_size_w, self.k * self.k, 3 ]) matrix_a_zero = tf.zeros_like(matrix_a, dtype=tf.float32) matrix_a_valid = tf.where(valid_condition_all, matrix_a, matrix_a_zero) matrix_a_trans = tf.matrix_transpose(matrix_a_valid, name='matrix_transpose') matrix_b = tf.ones(shape=[ self.batch_size, self.crop_size_h, self.crop_size_w, self.k * self.k, 1 ]) point_multi = tf.matmul(matrix_a_trans, matrix_a_valid, name='matrix_multiplication') with tf.device('cpu:0'): matrix_deter = tf.matrix_determinant(point_multi) inverse_condition = tf.greater(matrix_deter, 1e-5) inverse_condition = tf.expand_dims(inverse_condition, axis=3) inverse_condition = tf.expand_dims(inverse_condition, axis=4) inverse_condition_all = tf.tile(inverse_condition, [1, 1, 1, 3, 3]) diag_constant = tf.ones([3], dtype=tf.float32) diag_element = tf.diag(diag_constant) diag_element = tf.expand_dims(diag_element, axis=0) diag_element = tf.expand_dims(diag_element, axis=0) diag_element = tf.expand_dims(diag_element, axis=0) diag_matrix = tf.tile(diag_element, [ self.batch_size, self.crop_size_h, self.crop_size_w, 1, 1 ]) inversible_matrix = tf.where(inverse_condition_all, point_multi, diag_matrix) with tf.device('cpu:0'): inv_matrix = tf.matrix_inverse(inversible_matrix) generated_norm = tf.matmul( tf.matmul(inv_matrix, matrix_a_trans), matrix_b) norm_normalize = slim.unit_norm((generated_norm), dim=3) norm_normalize = tf.reshape( norm_normalize, [self.batch_size, self.crop_size_h, self.crop_size_w, 3]) norm_scale = norm_normalize * 10.0 conv1_noise = slim.repeat(norm_scale, 2, slim.conv2d, 64, [3, 3], scope='conv1_noise') pool1_noise = slim.max_pool2d(conv1_noise, [3, 3], stride=2, padding='SAME', scope='pool1_noise') # conv2_noise = slim.repeat(pool1_noise, 2, slim.conv2d, 128, [3, 3], scope='conv2_noise') conv3_noise = slim.repeat(conv2_noise, 3, slim.conv2d, 256, [3, 3], scope='conv3_noise') fc1_noise = slim.conv2d(conv3_noise, 512, [1, 1], activation_fn=tf.nn.relu, stride=1, scope='fc1_noise', padding='SAME') encode_norm_noise = slim.conv2d(fc1_noise, 3, [3, 3], activation_fn=None, stride=1, scope='encode_norm_noise', padding='SAME') encode_norm_upsample_noise = tf.image.resize_images( encode_norm_noise, [self.crop_size_h, self.crop_size_w], method=0, align_corners=True) sum_norm_noise = tf.add(norm_normalize, encode_norm_upsample_noise) norm_pred_noise = slim.unit_norm(sum_norm_noise, dim=3) norm_pred_all = tf.concat([ tf.expand_dims(tf.squeeze(fc8_upsample_norm), axis=0), norm_pred_noise, inputs * 0.00392156862 ], axis=3) norm_pred_all = slim.repeat( norm_pred_all, 3, slim.conv2d, 128, [3, 3], rate=2, weights_initializer=tf.contrib.layers.xavier_initializer( uniform=False), biases_initializer=tf.constant_initializer(0.0), scope='conv1_norm_noise_new') norm_pred_all = slim.repeat( norm_pred_all, 3, slim.conv2d, 128, [3, 3], weights_initializer=tf.contrib.layers.xavier_initializer( uniform=False), biases_initializer=tf.constant_initializer(0.0), scope='conv2_norm_noise_new') norm_pred_final = slim.conv2d( norm_pred_all, 3, [3, 3], activation_fn=None, weights_initializer=tf.contrib.layers.xavier_initializer( uniform=False), biases_initializer=tf.constant_initializer(0.0), scope='norm_conv3_noise_new') norm_pred_final = slim.unit_norm((norm_pred_final), dim=3) # ------------- normal to depth + depth refinement--------------------------------------------------- with tf.variable_scope('norm_depth', reuse=reuse): grid_patch = tf.extract_image_patches( images=grid, ksizes=[1, self.k, self.k, 1], strides=[1, 1, 1, 1], rates=[1, self.rate, self.rate, 1], padding='SAME') grid_patch = tf.reshape(grid_patch, [ self.batch_size, self.crop_size_h, self.crop_size_w, self.k * self.k, 3 ]) _, _, depth_data = tf.split(value=matrix_a, num_or_size_splits=3, axis=4) tmp_matrix_zero = tf.zeros_like(angle, dtype=tf.float32) valid_angle = tf.where(valid_condition, angle, tmp_matrix_zero) lower_matrix = tf.matmul(matrix_c, tf.expand_dims(grid, axis=4)) condition = tf.greater(lower_matrix, 1e-5) tmp_matrix = tf.ones_like(lower_matrix) lower_matrix = tf.where(condition, lower_matrix, tmp_matrix) lower = tf.reciprocal(lower_matrix) valid_angle = tf.where(condition, valid_angle, tmp_matrix_zero) upper = tf.reduce_sum(tf.multiply(matrix_c, grid_patch), [4]) ratio = tf.multiply(lower, tf.expand_dims(upper, axis=4)) estimate_depth = tf.multiply(ratio, depth_data) valid_angle = tf.multiply( valid_angle, tf.reciprocal( tf.tile( tf.reduce_sum(valid_angle, [3, 4], keep_dims=True) + 1e-5, [1, 1, 1, 81, 1]))) depth_stage1 = tf.reduce_sum( tf.multiply(estimate_depth, valid_angle), [3, 4]) depth_stage1 = tf.expand_dims(tf.squeeze(depth_stage1), axis=2) depth_stage1 = tf.clip_by_value(depth_stage1, 0, 10.0) exp_depth = tf.expand_dims(tf.squeeze(exp_depth), axis=2) depth_all = tf.expand_dims(tf.concat([ depth_stage1, exp_depth, tf.squeeze(inputs) * 0.00392156862 ], axis=2), axis=0) depth_pred_all = slim.repeat( depth_all, 3, slim.conv2d, 128, [3, 3], rate=2, weights_initializer=tf.contrib.layers.xavier_initializer( uniform=False), biases_initializer=tf.constant_initializer(0.0), scope='conv1_depth_noise_new') depth_pred_all = slim.repeat( depth_pred_all, 3, slim.conv2d, 128, [3, 3], weights_initializer=tf.contrib.layers.xavier_initializer( uniform=False), biases_initializer=tf.constant_initializer(0.0), scope='conv2_depth_noise_new') final_depth = slim.conv2d( depth_pred_all, 1, [3, 3], activation_fn=None, weights_initializer=tf.contrib.layers.xavier_initializer( uniform=False), biases_initializer=tf.constant_initializer(0.0), scope='depth_conv3_noise_new') with tf.variable_scope('edge_refinemet', reuse=reuse): print(inputs.shape) edges = tf.py_func(myfunc_canny, [inputs], tf.float32) edges = tf.reshape(edges, [1, self.crop_size_h, self.crop_size_w, 1]) edge_input_depth = final_depth edge_input_norm = norm_pred_final # edge prediction for depth edge_inputs = tf.concat([edges, inputs * 0.00784], axis=3) edges_encoder = slim.repeat( edge_inputs, 3, slim.conv2d, 32, [3, 3], rate=2, weights_initializer=tf.contrib.layers.xavier_initializer( uniform=False), biases_initializer=tf.constant_initializer(0.0), scope='conv1_edge_refinement') edges_encoder = slim.repeat( edges_encoder, 3, slim.conv2d, 32, [3, 3], weights_initializer=tf.contrib.layers.xavier_initializer( uniform=False), biases_initializer=tf.constant_initializer(0.0), scope='conv2_edge_refinement') edges_predictor = slim.conv2d( edges_encoder, 8, [3, 3], activation_fn=None, weights_initializer=tf.contrib.layers.xavier_initializer( uniform=False), biases_initializer=tf.constant_initializer(0.0), scope='edge_weight') edges_all = edges_predictor + tf.tile(edges, [1, 1, 1, 8]) edges_all = tf.clip_by_value(edges_all, 0.0, 1.0) dlr, drl, dud, ddu, nlr, nrl, nud, ndu = tf.split( edges_all, num_or_size_splits=8, axis=3) # 4 iteration depth final_depth = propagate(edge_input_depth, dlr, drl, dud, ddu, 1) final_depth = propagate(final_depth, dlr, drl, dud, ddu, 1) final_depth = propagate(final_depth, dlr, drl, dud, ddu, 1) final_depth = propagate(final_depth, dlr, drl, dud, ddu, 1) # 4 iteration norm norm_pred_final = propagate(edge_input_norm, nlr, nrl, nud, ndu, 3) norm_pred_final = slim.unit_norm((norm_pred_final), dim=3) norm_pred_final = propagate(norm_pred_final, nlr, nrl, nud, ndu, 3) norm_pred_final = slim.unit_norm((norm_pred_final), dim=3) norm_pred_final = propagate(norm_pred_final, nlr, nrl, nud, ndu, 3) norm_pred_final = slim.unit_norm((norm_pred_final), dim=3) norm_pred_final = propagate(norm_pred_final, nlr, nrl, nud, ndu, 3) norm_pred_final = slim.unit_norm((norm_pred_final), dim=3) return final_depth, fc8_upsample_norm, norm_pred_final, fc8_upsample
def invertible_1x1_conv(name, x, reverse=False): """1X1 convolution on x. The 1X1 convolution is parametrized as P*L*(U + sign(s)*exp(log(s))) where 1. P is a permutation matrix. 2. L is a lower triangular matrix with diagonal entries unity. 3. U is a upper triangular matrix where the diagonal entries zero. 4. s is a vector. sign(s) and P are fixed and the remaining are optimized. P, L, U and s are initialized by the PLU decomposition of a random rotation matrix. Args: name: scope x: Input Tensor. reverse: whether the pass is from z -> x or x -> z. Returns: x_conv: x after a 1X1 convolution is applied on x. objective: sum(log(s)) """ _, height, width, channels = common_layers.shape_list(x) w_shape = [channels, channels] # Random rotation-matrix Q random_matrix = np.random.rand(channels, channels) np_w = scipy.linalg.qr(random_matrix)[0].astype("float32") # Initialize P,L,U and s from the LU decomposition of a random rotation matrix np_p, np_l, np_u = scipy.linalg.lu(np_w) np_s = np.diag(np_u) np_sign_s = np.sign(np_s) np_log_s = np.log(np.abs(np_s)) np_u = np.triu(np_u, k=1) with tf.variable_scope(name, reuse=tf.AUTO_REUSE): p = tf.get_variable("P", initializer=np_p, trainable=False) l = tf.get_variable("L", initializer=np_l) sign_s = tf.get_variable( "sign_S", initializer=np_sign_s, trainable=False) log_s = tf.get_variable("log_S", initializer=np_log_s) u = tf.get_variable("U", initializer=np_u) # W = P * L * (U + sign_s * exp(log_s)) l_mask = np.tril(np.ones([channels, channels], dtype=np.float32), -1) l = l * l_mask + tf.eye(channels, channels) u = u * np.transpose(l_mask) + tf.diag(sign_s * tf.exp(log_s)) w = tf.matmul(p, tf.matmul(l, u)) # If height or width cannot be statically determined then they end up as # tf.int32 tensors, which cannot be directly multiplied with a floating # point tensor without a cast. objective = tf.reduce_sum(log_s) * tf.cast(height * width, log_s.dtype) if not reverse: w = tf.reshape(w, [1, 1] + w_shape) x = tf.nn.conv2d(x, w, [1, 1, 1, 1], "SAME", data_format="NHWC") else: u_inv = tf.matrix_inverse(u) l_inv = tf.matrix_inverse(l) p_inv = tf.matrix_inverse(p) w_inv = tf.matmul(u_inv, tf.matmul(l_inv, p_inv)) w_inv = tf.reshape(w_inv, [1, 1]+w_shape) x = tf.nn.conv2d( x, w_inv, [1, 1, 1, 1], "SAME", data_format="NHWC") objective *= -1 return x, objective
def handle_empty_marg(params, all_vars): print_op = tf.Print(tf.constant(1), [], "Inside handling empty marg") # Set Q_s as [[0]] with tf.control_dependencies([print_op]): all_vars = all_vars._replace(Q_s=tf.constant([[0.]], dtype=tf.float64)) # Calculate min b for transition g_c_ = all_vars.g_c x_c_ = all_vars.x_c y_c_ = all_vars.y_c trans_c = -1 * g_c_ # calc trans b for err and rem # For err_vec, err_vec_y should have same sign as y_c_ elems = (all_vars.err_vec_y, all_vars.g_err) trans_err = tf.map_fn( lambda x: tf.cond(tf.greater(x[0] * y_c_, 0), lambda: -1 * x[1], lambda: tf.constant(float("Inf"), dtype=tf.float64)), elems, dtype=tf.float64) # For rem_vec, rem_vec_y should have opposite sign as y_c_ elems = (all_vars.rem_vec_y, all_vars.g_rem) trans_rem = tf.map_fn( lambda x: tf.cond(tf.less(x[0] * y_c_, 0), lambda: x[1], lambda: tf. constant(float("Inf"), dtype=tf.float64)), elems, dtype=tf.float64) # Find the min transition_val for b all_trans = tf.concat([trans_err, trans_rem, tf.reshape(trans_c, [ 1, ])], 0) abs_min_del_b = tf.reduce_min(all_trans) # change the sign of min_del_b to the sign of y_c_ as # this is the dirn of update min_del_b = tf.cast(y_c_ * abs_min_del_b, dtype=tf.float64) # update values b_ = all_vars.b + min_del_b g_c_ = g_c_ + (min_del_b * y_c_) # update g_err g_err_ = all_vars.g_err + (all_vars.err_vec_y * min_del_b) # update g_rem g_rem_ = all_vars.g_rem + (all_vars.rem_vec_y * min_del_b) # print_op = tf.Print(tf.constant(1.), [all_vars.g_c, all_vars.Q_s, min_del_b], # "Line 351") # with tf.control_dependencies([print_op]): all_vars = all_vars._replace(b=b_, g_c=g_c_, g_err=g_err_, g_rem=g_rem_) # Transition vectors if g is zero and dirn of update is same as candidate # Check if min is in err sup vectors min_err_indx = tf.cond( tf.equal(tf.shape(trans_err)[0], 0), lambda: tf.constant(1), lambda: check_min(trans_err, abs_min_del_b), ) all_vars = tf.cond( tf.less(min_err_indx, tf.shape(trans_err)[0]), lambda: handle_empty_err(params, min_err_indx, all_vars), lambda: all_vars) # Check if min is in rem vectors min_rem_indx = tf.cond( tf.equal(tf.shape(trans_rem)[0], 0), lambda: tf.constant(1), lambda: check_min(trans_rem, abs_min_del_b), ) all_vars = tf.cond( tf.less(min_rem_indx, tf.shape(trans_rem)[0]), lambda: handle_empty_rem(params, min_rem_indx, all_vars), lambda: all_vars) # Check if min is in trans_c all_vars = tf.cond( tf.equal(trans_c, abs_min_del_b), lambda: free_add_to_marg(params["kernel"], all_vars.alpha_c, all_vars. x_c, all_vars.y_c, all_vars), lambda: all_vars) # update R as inverse of Q R_ = tf.matrix_inverse(all_vars.Q_s) return all_vars._replace(R=R_)
Tr_Knn, K_nm_2, K_mnnm_2 = KS.build_psi_stats_rbf(X_m_2,tf.square(noise_2), len_sc_2, h_mu, h_S) K_mm_2=h.tf_SE_K(X_m_2,X_m_2,len_sc_2,noise_2) K_nn_2=h.tf_SE_K(h_mu,h_mu,len_sc_2,noise_2) K_mn_2=h.tf.transpose(K_nm_2) ''' opti_mu1= opti_Sig1= opti_mu2= opti_sig2= ''' a_mn=tf.matrix_solve(K_mm_1,K_mn_1) a_nm=tf.transpose(a_mn) sig_inv=tf.matrix_inverse(K_mm_1)+h.Mul(a_mn,a_nm)/tf.square(sigma_1) mu1=h.Mul(tf.matrix_inverse(sig_inv),a_mn,Ytr)/tf.square(sigma_1) F_v_1=GPLVM.Bound1(h_mu,h_S,K_mm_1,K_nm_1,Tr_Knn_1,sigma_1) s.run(tf.initialize_all_variables()) F_v_2=GPLVM.Bound2(Tr_Knn,K_nm_2,K_mnnm_2,sigma_2,K_mm_2,Ytr) mean,var_terms=predict(K_mn_2,sigma_2,K_mm_2,K_nn_2) mean=tf.reshape(mean,[-1]) var_terms=tf.reshape(var_terms,[-1]) global_step = tf.Variable(0, trainable=False) starter_learning_rate = 0.1 lrate=tf.train.exponential_decay(starter_learning_rate, global_step,
def evaluate_task(inputs): features_train, train_outputs, features_test, test_outputs, representation, mean_features = inputs q_representation = multihead_attention(features_test, mean_features, mean_features) with tf.variable_scope('inference_network'): r_mu = inference_block(representation, args.d_theta, args.d_theta, 'infer_mean') r_log_var = inference_block(representation, args.d_theta, args.d_theta, 'infer_var') p_mu = inference_block(q_representation, args.d_theta, args.d_theta, 'p_mean') p_log_var = inference_block(q_representation, args.d_theta, args.d_theta, 'p_var') # Infer classification layer from q with tf.variable_scope('classifier'): # compute bases rs = tf.squeeze( sample_normal(r_mu, r_log_var, args.d_rn_f, eps_=eps)) # compute the support kernel x_supp_phi_t = rand_features(rs, tf.transpose(features_train, [1, 0]), bias) # (d_rn_f , w*s) support_kernel = dotp_kernel(tf.transpose(x_supp_phi_t), x_supp_phi_t) # (w*s , w*s) # closed-form solution with trainable lmd alpha = tf.matmul( tf.matrix_inverse(support_kernel + (lmd_abs + 0.01) * tf.eye(tf.shape(support_kernel)[0])), train_outputs) x_que_phi_t = rand_features( rs, tf.transpose(features_test, [1, 0]), bias ) # tf.cos(tf.matmul(rs, tf.transpose(features_test, [1, 0]))) # compute the cross kernel cross_kernel = dotp_kernel(tf.transpose(x_supp_phi_t), x_que_phi_t) # prediction with calibration params logits = gamma * tf.matmul(cross_kernel, alpha, transpose_a=True) + beta # align loss target_kernel = dotp_kernel(train_outputs, tf.transpose(train_outputs)) target_kernel = 0.99 * (target_kernel + 0.01) kernel_align_loss = cosine_dist(target_kernel, support_kernel) # kl loss kl_loss = KL_divergence(r_mu, r_log_var, p_mu, p_log_var) # cross entry loss cross_entry_loss = mse(logits, test_outputs) task_loss = cross_entry_loss + args.zeta * kernel_align_loss + args.tau * kl_loss return [ task_loss, cross_entry_loss, kernel_align_loss, kl_loss, logits ]
def __init__(self, sess, batch_size, input_len, hidden_num, output_len, task='main'): ''' Args: sess : TensorFlow session. batch_size : The batch size (N) input_len : The length of input. (L) hidden_num : The number of hidden node. (K) output_len : The length of output. (O) ''' self._sess = sess self._batch_size = batch_size self._input_len = input_len self._hidden_num = hidden_num self._output_len = output_len # for train self._x0 = tf.placeholder(tf.float32, [self._batch_size, self._input_len]) self._t0 = tf.placeholder(tf.float32, [self._batch_size, self._output_len]) # for test self._x1 = tf.placeholder(tf.float32, [None, self._input_len]) self._t1 = tf.placeholder(tf.float32, [None, self._output_len]) self._W = tf.Variable(tf.random_normal( [self._input_len, self._hidden_num]), trainable=False, dtype=tf.float32, name=task + '_w') self._b = tf.Variable(tf.random_normal([self._hidden_num]), trainable=False, dtype=tf.float32, name=task + '_b') self._beta = tf.Variable(tf.zeros([self._hidden_num, self._output_len]), trainable=False, dtype=tf.float32, name=task + '_beta') self._var_list = [self._W, self._b, self._beta] self.H0 = tf.matmul(self._x0, self._W) + self._b # N x L self.H0_T = tf.transpose(self.H0) self.H1 = tf.matmul(self._x1, self._W) + self._b # N x L self.H1_T = tf.transpose(self.H1) # beta analytic solution : self._beta_s (K x O) if self._input_len < self._hidden_num: # L < K identity = tf.constant(np.identity(self._hidden_num), dtype=tf.float32) self._beta_s = tf.matmul( tf.matmul( tf.matrix_inverse( tf.matmul(self.H0_T, self.H0) + identity / omega), self.H0_T), self._t0) # _beta_s = (H_T*H + I/om)^(-1)*H_T*T else: identity = tf.constant(np.identity(self._batch_size), dtype=tf.float32) self._beta_s = tf.matmul( tf.matmul( self.H0_T, tf.matrix_inverse( tf.matmul(self.H0, self.H0_T) + identity / omega)), self._t0) # _beta_s = H_T*(H*H_T + I/om)^(-1)*T self._assign_beta = self._beta.assign(self._beta_s) self._fx0 = tf.matmul(self.H0, self._beta) self._fx1 = tf.matmul(self.H1, self._beta) self._cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(self._fx0, self._t0)) self._init = False self._feed = False # for the mnist test self._correct_prediction = tf.equal(tf.argmax(self._fx1, 1), tf.argmax(self._t1, 1)) self._accuracy = tf.reduce_mean( tf.cast(self._correct_prediction, tf.float32))
def __init__(self, params, is_training=True): self.is_training = is_training batch_size = params["batch_size"] num_layers = params['nlayer'] rnn_size = params['n_hidden'] grad_clip = params["grad_clip"] self.output_keep_prob = tf.placeholder(tf.float32) NOUT = params['n_output'] # Transition LSTM cell_lst = [] for i in range(num_layers): cell = rnncell.ModifiedLSTMCell( rnn_size, forget_bias=1, initializer=tf.contrib.layers.xavier_initializer(), num_proj=None, is_training=self.is_training) if i > 10 and is_training == True: cell_drop = rnncell.DropoutWrapper( cell, output_keep_prob=self.output_keep_prob) cell = cell_drop cell_lst.append(cell) self.cell = rnncell.MultiRNNCell(cell_lst) # LSTM for Q noise cell_lst = [] for i in range(params['Qnlayer']): cell_Q_noise = rnncell.ModifiedLSTMCell( params['Qn_hidden'], forget_bias=1, initializer=tf.contrib.layers.xavier_initializer(), num_proj=None, is_training=self.is_training) if i > 10 and is_training == True: cell_drop = rnncell.DropoutWrapper( cell_Q_noise, output_keep_prob=self.output_keep_prob) cell_Q_noise = cell_drop cell_lst.append(cell_Q_noise) self.cell_Q_noise = rnncell.MultiRNNCell(cell_lst) # LSTM for R noise cell_lst = [] for i in range(params['Rnlayer']): cell_R_noise = rnncell.ModifiedLSTMCell( params['Rn_hidden'], forget_bias=1, initializer=tf.contrib.layers.xavier_initializer(), num_proj=None, is_training=self.is_training) if i > 10 and is_training == True: cell_drop = rnncell.DropoutWrapper( cell_R_noise, output_keep_prob=self.output_keep_prob) cell_R_noise = cell_drop cell_lst.append(cell_R_noise) self.cell_R_noise = rnncell.MultiRNNCell(cell_lst) self.initial_state = self.cell.zero_state( batch_size=params['batch_size'], dtype=tf.float32) self.initial_state_Q_noise = self.cell_Q_noise.zero_state( batch_size=params['batch_size'], dtype=tf.float32) self.initial_state_R_noise = self.cell_Q_noise.zero_state( batch_size=params['batch_size'], dtype=tf.float32) self.initial_state_R_noise = self.cell_R_noise.zero_state( batch_size=params['batch_size'], dtype=tf.float32) self.repeat_data = tf.placeholder( dtype=tf.int32, shape=[params["batch_size"], params['seq_length']]) #Measurements self._z = tf.placeholder(dtype=tf.float32, shape=[None, params['seq_length'], NOUT ]) # batch size, seqlength, feature self.target_data = tf.placeholder( dtype=tf.float32, shape=[None, params['seq_length'], NOUT]) # batch size, seqlength, feature self._P_inp = tf.placeholder(dtype=tf.float32, shape=[None, NOUT, NOUT], name='P') self._F = 0.0 # state transition matrix self._alpha_sq = 1. # fading memory control self.M = 0.0 # process-measurement cross correlation self._I = tf.placeholder(dtype=tf.float32, shape=[None, NOUT, NOUT], name='I') self.u = 0.0 xres_lst = [] xpred_lst = [] pres_lst = [] tres_lst = [] kres_lst = [] with tf.variable_scope('rnnlm'): output_w1 = tf.get_variable( "output_w", [rnn_size, NOUT], initializer=tf.contrib.layers.xavier_initializer()) output_b1 = tf.get_variable("output_b", [NOUT]) output_w1_Q_noise = tf.get_variable( "output_w_Q_noise", [params['Qn_hidden'], NOUT], initializer=tf.contrib.layers.xavier_initializer()) output_b1_Q_noise = tf.get_variable("output_b_Q_noise", [NOUT]) output_w1_R_noise = tf.get_variable( "output_w_R_noise", [params['Rn_hidden'], NOUT], initializer=tf.contrib.layers.xavier_initializer()) output_b1_R_noise = tf.get_variable("output_b_R_noise", [NOUT]) state_F = self.initial_state state_Q = self.initial_state_Q_noise state_R = self.initial_state_R_noise with tf.variable_scope("rnnlm"): for time_step in range(params['seq_length']): if time_step > 0: tf.get_variable_scope().reuse_variables() z = self._z[:, time_step, :] #bs,features if time_step == 0: self._x = z self._P = self._P_inp with tf.variable_scope("transitionF"): (pred, state_F, ls_internals) = self.cell(self._x, state_F) pred = tf.matmul(pred, output_w1) + output_b1 with tf.variable_scope("noiseQ"): (pred_Q_noise, state_Q, ls_internals) = self.cell_Q_noise(self._x, state_Q) pred_Q_noise = tf.matmul( pred_Q_noise, output_w1_Q_noise) + output_b1_Q_noise with tf.variable_scope("noiseR"): (pred_R_noise, state_R, ls_internals) = self.cell_R_noise(z, state_R) pred_R_noise = tf.matmul( pred_R_noise, output_w1_R_noise) + output_b1_R_noise self._x = pred # lst=tf.unpack(pred, axis=1) Q = tf.matrix_diag(tf.exp(pred_Q_noise)) # R=tf.matrix_diag(tf.exp(pred_R_noise)) #predict P = self._P self._P = P + Q #update P = self._P x = self._x self._y = z - x # S = HPH' + R # project system uncertainty into measurement space # S = P + R S = P # K = PH'inv(S) # map system uncertainty into kalman gain K = tf.matmul(P, tf.matrix_inverse(S)) # x = x + Ky # predict new x with residual scaled by the kalman gain self._x = x + tf.squeeze( tf.matmul(K, tf.expand_dims(self._y, 2))) xpred_lst.append(x) xres_lst.append(self._x) tres_lst.append(x) kres_lst.append(K) # P = (I-KH)P(I-KH)' + KRK' I_KH = self._I - K # self._P = tf.matmul(I_KH, tf.matmul(P, tf.matrix_transpose(I_KH))) + tf.matmul(K, tf.matmul(R, tf.matrix_transpose(K))) self._P = tf.matmul( I_KH, tf.matmul(P, tf.matrix_transpose(I_KH))) + tf.matmul( K, tf.matrix_transpose(K)) self._S = S self._K = K final_output = tf.reshape(tf.transpose(tf.pack(xres_lst), [1, 0, 2]), [-1, params['n_output']]) final_pred_output = tf.reshape( tf.transpose(tf.pack(xpred_lst), [1, 0, 2]), [-1, params['n_output']]) flt = tf.squeeze(tf.reshape(self.repeat_data, [-1, 1]), [1]) where_flt = tf.not_equal(flt, 0) indices = tf.where(where_flt) y = tf.reshape(self.target_data, [-1, params["n_output"]]) self.final_output = tf.gather(final_output, tf.squeeze(indices, [1])) self.final_pred_output = tf.gather(final_pred_output, tf.squeeze(indices, [1])) self.y = tf.gather(y, tf.squeeze(indices, [1])) tmp = self.final_output - self.y loss = tf.nn.l2_loss(tmp) tmp_pred = self.final_pred_output - self.y loss_pred = tf.nn.l2_loss(tmp_pred) self.tvars = tf.trainable_variables() l2_reg = tf.reduce_sum([tf.nn.l2_loss(var) for var in self.tvars]) l2_reg = tf.mul(l2_reg, 1e-4) self.cost = tf.reduce_mean(loss) + l2_reg self.lr = tf.Variable(0.0, trainable=False) tvars = tf.trainable_variables() total_parameters = 0 for variable in self.tvars: # shape is an array of tf.Dimension shape = variable.get_shape() variable_parametes = 1 for dim in shape: variable_parametes *= dim.value total_parameters += variable_parametes self.total_parameters = total_parameters grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, tvars), grad_clip) optimizer = tf.train.AdamOptimizer(self.lr) self.train_op = optimizer.apply_gradients(zip(grads, tvars)) self.states = {} self.states["F_t"] = state_F self.states["Q_t"] = state_Q self.states["R_t"] = state_R self.states["PCov_t"] = self._P self.xres_lst = xres_lst self.pres_lst = pres_lst self.tres_lst = tres_lst self.kres_lst = kres_lst
def QuantizedWeight(name, x, n, nbit=2): """ Quantize weight. Args: x (tf.Tensor): a 4D tensor. Must have known number of channels, but can have other unknown dimensions. name (str): operator's name. n (int or double): variance of weight initialization. nbit (int): number of bits of quantized weight. Defaults to 2. Returns: tf.Tensor with attribute `variables`. Variable Names: * ``basis``: basis of quantized weight. Note: About multi-GPU training: moving averages across GPUs are not aggregated. Batch statistics are computed by main training tower. This is consistent with most frameworks. """ num_filters = x.get_shape().as_list()[-1] init_basis = [] base = NORM_PPF_0_75 * ((2. / n)**0.5) / (2**(nbit - 1)) for j in range(nbit): init_basis.append([(2**j) * base for i in range(num_filters)]) init_basis = tf.constant_initializer(init_basis) bit_dims = [nbit, num_filters] num_levels = 2**nbit delta = EPS # initialize level multiplier init_level_multiplier = [] for i in range(num_levels): level_multiplier_i = [0. for j in range(nbit)] level_number = i for j in range(nbit): binary_code = level_number % 2 if binary_code == 0: binary_code = -1 level_multiplier_i[j] = float(binary_code) level_number = level_number // 2 init_level_multiplier.append(level_multiplier_i) # initialize threshold multiplier init_thrs_multiplier = [] for i in range(1, num_levels): thrs_multiplier_i = [0. for j in range(num_levels)] thrs_multiplier_i[i - 1] = 0.5 thrs_multiplier_i[i] = 0.5 init_thrs_multiplier.append(thrs_multiplier_i) with tf.variable_scope(name): basis = tf.get_variable('basis', bit_dims, tf.float32, initializer=init_basis, trainable=False) level_codes = tf.constant(init_level_multiplier) thrs_multiplier = tf.constant(init_thrs_multiplier) sum_multiplier = tf.constant( 1., shape=[1, tf.reshape(x, [-1, num_filters]).get_shape()[0]]) sum_multiplier_basis = tf.constant(1., shape=[1, nbit]) ctx = get_current_tower_context() # current tower context # calculate levels and sort levels = tf.matmul(level_codes, basis) levels, sort_id = tf.nn.top_k(tf.transpose(levels, [1, 0]), num_levels) levels = tf.reverse(levels, [-1]) sort_id = tf.reverse(sort_id, [-1]) levels = tf.transpose(levels, [1, 0]) sort_id = tf.transpose(sort_id, [1, 0]) # calculate threshold thrs = tf.matmul(thrs_multiplier, levels) # calculate level codes per channel reshape_x = tf.reshape(x, [-1, num_filters]) level_codes_channelwise_dims = tf.stack( [num_levels * num_filters, nbit]) level_codes_channelwise = tf.fill(level_codes_channelwise_dims, 0.) for i in range(num_levels): eq = tf.equal(sort_id, i) level_codes_channelwise = tf.where( tf.reshape(eq, [-1]), level_codes_channelwise + level_codes[i], level_codes_channelwise) level_codes_channelwise = tf.reshape(level_codes_channelwise, [num_levels, num_filters, nbit]) # calculate output y and its binary code y = tf.zeros_like(x) + levels[0] # output zero_dims = tf.stack([tf.shape(reshape_x)[0] * num_filters, nbit]) bits_y = tf.fill(zero_dims, -1.) zero_y = tf.zeros_like(x) zero_bits_y = tf.fill(zero_dims, 0.) zero_bits_y = tf.reshape(zero_bits_y, [-1, num_filters, nbit]) for i in range(num_levels - 1): g = tf.greater(x, thrs[i]) y = tf.where(g, zero_y + levels[i + 1], y) bits_y = tf.where( tf.reshape(g, [-1]), tf.reshape(zero_bits_y + level_codes_channelwise[i + 1], [-1, nbit]), bits_y) bits_y = tf.reshape(bits_y, [-1, num_filters, nbit]) # training if ctx.is_main_training_tower: BT = tf.transpose(bits_y, [2, 0, 1]) # calculate BTxB BTxB = [] for i in range(nbit): for j in range(nbit): BTxBij = tf.multiply(BT[i], BT[j]) BTxBij = tf.matmul(sum_multiplier, BTxBij) if i == j: mat_one = tf.ones([1, num_filters]) BTxBij = BTxBij + (delta * mat_one) # + E BTxB.append(BTxBij) BTxB = tf.reshape(tf.stack(values=BTxB), [nbit, nbit, num_filters]) # calculate inverse of BTxB if nbit > 2: BTxB_transpose = tf.transpose(BTxB, [2, 0, 1]) BTxB_inv = tf.matrix_inverse(BTxB_transpose) BTxB_inv = tf.transpose(BTxB_inv, [1, 2, 0]) elif nbit == 2: det = tf.multiply(BTxB[0][0], BTxB[1][1]) - tf.multiply( BTxB[0][1], BTxB[1][0]) inv = [] inv.append(BTxB[1][1] / det) inv.append(-BTxB[0][1] / det) inv.append(-BTxB[1][0] / det) inv.append(BTxB[0][0] / det) BTxB_inv = tf.reshape(tf.stack(values=inv), [nbit, nbit, num_filters]) elif nbit == 1: BTxB_inv = tf.reciprocal(BTxB) # calculate BTxX BTxX = [] for i in range(nbit): BTxXi0 = tf.multiply(BT[i], reshape_x) BTxXi0 = tf.matmul(sum_multiplier, BTxXi0) BTxX.append(BTxXi0) BTxX = tf.reshape(tf.stack(values=BTxX), [nbit, num_filters]) BTxX = BTxX + (delta * basis) # + basis # calculate new basis new_basis = [] for i in range(nbit): new_basis_i = tf.multiply(BTxB_inv[i], BTxX) new_basis_i = tf.matmul(sum_multiplier_basis, new_basis_i) new_basis.append(new_basis_i) new_basis = tf.reshape(tf.stack(values=new_basis), [nbit, num_filters]) # create moving averages op updata_moving_basis = moving_averages.assign_moving_average( basis, new_basis, MOVING_AVERAGES_FACTOR) add_model_variable(basis) tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, updata_moving_basis) y = x + tf.stop_gradient(-x) + tf.stop_gradient(y) # gradient: y=x y.variables = VariableHolder(basis=basis) return y