def testNotInvertible(self):
# The input should be invertible.
with self.test_session():
with self.assertRaisesOpError("Input is not invertible."):
# All rows of the matrix below add to zero.
tensor3 = tf.constant([[1., 0., -1.], [-1., 1., 0.], [0., -1., 1.]])
tf.matrix_inverse(tensor3).eval()
def solve(a, b):
if b.ndim == 1:
return tf.reshape(tf.matmul(tf.matrix_inverse(a), tf.expand_dims(b, -1)), [-1])
elif b.ndim == 2:
return tf.matmul(tf.matrix_inverse(a), b)
else:
import ipdb; ipdb.set_trace()
def hnet_loss(gt_pts, transformation_coeffcient, name):
"""
:param gt_pts: 原始的标签点对 [x, y, 1]
:param transformation_coeffcient: 映射矩阵参数(6参数矩阵) [[a, b, c], [0, d, e], [0, f, 1]]
:param name:
:return:
"""
with tf.variable_scope(name):
# 首先映射原始标签点对
transformation_coeffcient = tf.concat([transformation_coeffcient, [1.0]], axis=-1)
H_indices = tf.constant([[0], [1], [2], [4], [5], [7], [8]])
H_shape = tf.constant([9])
H = tf.scatter_nd(H_indices, transformation_coeffcient, H_shape)
H = tf.reshape(H, shape=[3, 3])
gt_pts = tf.transpose(gt_pts)
pts_projects = tf.matmul(H, gt_pts)
# 求解最小二乘二阶多项式拟合参数矩阵
Y = tf.transpose(pts_projects[1, :])
X = tf.transpose(pts_projects[0, :])
Y_One = tf.add(tf.subtract(Y, Y), tf.constant(1.0, tf.float32))
Y_stack = tf.stack([tf.pow(Y, 3), tf.pow(Y, 2), Y, Y_One], axis=1)
w = tf.matmul(tf.matmul(tf.matrix_inverse(tf.matmul(tf.transpose(Y_stack), Y_stack)),
tf.transpose(Y_stack)),
tf.expand_dims(X, -1))
# 利用二阶多项式参数求解拟合位置并反算到原始投影空间计算损失
x_preds = tf.matmul(Y_stack, w)
preds = tf.transpose(tf.stack([tf.squeeze(x_preds, -1), Y, Y_One], axis=1))
x_transformation_back = tf.matmul(tf.matrix_inverse(H), preds)
loss = tf.reduce_mean(tf.pow(gt_pts[0, :] - x_transformation_back[0, :], 2))
return loss
def hnet_transformation(gt_pts, transformation_coeffcient, name):
"""
:param gt_pts:
:param transformation_coeffcient:
:param name:
:return:
"""
with tf.variable_scope(name):
# 首先映射原始标签点对
transformation_coeffcient = tf.concat([transformation_coeffcient, [1.0]], axis=-1)
H_indices = tf.constant([[0], [1], [2], [4], [5], [7], [8]])
H_shape = tf.constant([9])
H = tf.scatter_nd(H_indices, transformation_coeffcient, H_shape)
H = tf.reshape(H, shape=[3, 3])
gt_pts = tf.transpose(gt_pts)
pts_projects = tf.matmul(H, gt_pts)
# 求解最小二乘二阶多项式拟合参数矩阵
Y = tf.transpose(pts_projects[1, :])
X = tf.transpose(pts_projects[0, :])
Y_One = tf.add(tf.subtract(Y, Y), tf.constant(1.0, tf.float32))
Y_stack = tf.stack([tf.pow(Y, 3), tf.pow(Y, 2), Y, Y_One], axis=1)
w = tf.matmul(tf.matmul(tf.matrix_inverse(tf.matmul(tf.transpose(Y_stack), Y_stack)),
tf.transpose(Y_stack)),
tf.expand_dims(X, -1))
# 利用二阶多项式参数求解拟合位置
x_preds = tf.matmul(Y_stack, w)
preds = tf.transpose(tf.stack([tf.squeeze(x_preds, -1), Y, Y_One], axis=1))
preds_fit = tf.stack([tf.squeeze(x_preds, -1), Y], axis=1)
x_transformation_back = tf.matmul(tf.matrix_inverse(H), preds)
return x_transformation_back
def tf_prod_gauss(mu1, mu2, sig1, sig2): #assumes precision prec1 = tf.matrix_inverse(sig1) prec2 = tf.matrix_inverse(sig2) prec = tf.add(prec1, prec2) sig = tf.matrix_inverse(prec) mu = tf.add( tf.matmul(prec1, mu1), tf.matmul(prec2, mu1) ) return mu, sig
def F2_bound(y,Kmm,Knm,Knn,mu,Sigma):
Eye=tf.constant(np.eye(N,N), shape=[N,N],dtype=tf.float32)
sigEye=tf.mul(1.0,Eye)
print(s.run(tf.matrix_inverse(sigEye)))
#sigEye=tf.mul(tf.square(sigma),Eye)
Kmn=tf.transpose(Knm)
prec=Matrix_Inversion_Lemma(sigEye,Knm,Kmm,Kmn)
zeros=tf.constant(np.zeros(N),shape=[N,1],dtype=tf.float32)
log_den=log_density(y,zeros,prec)
Kmm_inv=tf.matrix_inverse(Kmm)
trace_term=tf.trace(Knn-Mul(Knm,Kmm_inv,Kmn))*(0.5)
return log_den-trace_term
def inverse(pBatch,opt,name=None):
with tf.name_scope("inverse"):
pMtrxBatch = vec2mtrxBatch(pBatch,opt)
pInvMtrxBatch = tf.matrix_inverse(pMtrxBatch)
pInvBatch = mtrx2vecBatch(pInvMtrxBatch,opt)
pInvBatch = tf.identity(pInvBatch,name=name)
return pInvBatch
def get_log_likelihood(self): I = tf.constant(self.sigx*np.identity(self.dimx)) mu = tf.constant(np.zeros(self.dimx).reshape((self.dimx,1))) ll = tf.constant(0, tf.float64) n_samples = 1 for i in xrange(self.n): x = self.observed[i] x = tf.constant(x, shape = [3,1], dtype = tf.float64) mc = tf.Variable(0, dtype = tf.float64) for j in xrange(n_samples): w = self.sample_theta() var = tf.matmul(w, tf.transpose(w)) var = tf.add(var, I) prec = tf.matrix_inverse(var) pw = util.tf_evaluate_gauss(mu, prec, x) mc+= pw mc = tf.div(mc,float(n_samples) ) mc = tf.log(mc) ll = tf.add(ll, mc) self.llh = ll return ll
def compute_rigid_flow(depth, pose, intrinsics, reverse_pose=False):
"""Compute the rigid flow from target image plane to source image
Args:
depth: depth map of the target image [batch, height_t, width_t]
pose: target to source (or source to target if reverse_pose=True)
camera transformation matrix [batch, 6], in the order of
tx, ty, tz, rx, ry, rz;
intrinsics: camera intrinsics [batch, 3, 3]
Returns:
Rigid flow from target image to source image [batch, height_t, width_t, 2]
"""
batch, height, width = depth.get_shape().as_list()
# Convert pose vector to matrix
pose = pose_vec2mat(pose)
if reverse_pose:
pose = tf.matrix_inverse(pose)
# Construct pixel grid coordinates
pixel_coords = meshgrid(batch, height, width)
tgt_pixel_coords = tf.transpose(pixel_coords[:,:2,:,:], [0, 2, 3, 1])
# Convert pixel coordinates to the camera frame
cam_coords = pixel2cam(depth, pixel_coords, intrinsics)
# Construct a 4x4 intrinsic matrix
filler = tf.constant([0.0, 0.0, 0.0, 1.0], shape=[1, 1, 4])
filler = tf.tile(filler, [batch, 1, 1])
intrinsics = tf.concat([intrinsics, tf.zeros([batch, 3, 1])], axis=2)
intrinsics = tf.concat([intrinsics, filler], axis=1)
# Get a 4x4 transformation matrix from 'target' camera frame to 'source'
# pixel frame.
proj_tgt_cam_to_src_pixel = tf.matmul(intrinsics, pose)
src_pixel_coords = cam2pixel(cam_coords, proj_tgt_cam_to_src_pixel)
rigid_flow = src_pixel_coords - tgt_pixel_coords
return rigid_flow
def run_training(train_X, train_Y):
X = tf.placeholder(tf.float32, [m, n+1])
Y = tf.placeholder(tf.float32, [m, 1])
# add the column of 1s to X
ones = np.ones([m, 1], dtype=np.float32)
train_X = np.concatenate((ones, train_X), axis=1)
# normal equations
theta = tf.matmul(tf.matmul(tf.matrix_inverse(tf.matmul(tf.transpose(X), X)), tf.transpose(X)), Y)
with tf.Session() as sess:
theta = sess.run(theta, feed_dict={X: train_X, Y: train_Y})
print "Predict.... (Predict a house with 1650 square feet and 3 bedrooms.)"
# Do not forget to add the column of 1s
predict_X = np.array([1, 1650, 3], dtype=np.float32).reshape((1, 3))
predict_Y = tf.matmul(predict_X, theta)
print "House price(Y) =", sess.run(predict_Y)
print "theta"
print theta
sess.close()
def logdet_grad(op, grad):
a = op.inputs[0]
a_adj_inv = tf.check_numerics(
tf.matrix_inverse(a, adjoint=True),
'zero determinant')
out_shape = tf.concat([tf.shape(a)[:-2], [1, 1]], axis=0)
return tf.reshape(grad, out_shape) * a_adj_inv
def initialize(self, *args, **kwargs):
# Store latent variables in a temporary attribute; MAP will
# optimize `PointMass` random variables, which subsequently
# optimizes mean parameters of the normal approximations.
latent_vars_normal = self.latent_vars.copy()
self.latent_vars = {z: PointMass(params=qz.loc)
for z, qz in six.iteritems(latent_vars_normal)}
super(Laplace, self).initialize(*args, **kwargs)
hessians = tf.hessians(self.loss, list(six.itervalues(self.latent_vars)))
self.finalize_ops = []
for z, hessian in zip(six.iterkeys(self.latent_vars), hessians):
qz = latent_vars_normal[z]
if isinstance(qz, (MultivariateNormalDiag, Normal)):
scale_var = get_variables(qz.variance())[0]
scale = 1.0 / tf.diag_part(hessian)
else: # qz is MultivariateNormalTriL
scale_var = get_variables(qz.covariance())[0]
scale = tf.matrix_inverse(tf.cholesky(hessian))
self.finalize_ops.append(scale_var.assign(scale))
self.latent_vars = latent_vars_normal.copy()
del latent_vars_normal
def _define_distance_to_clusters(self, data):
"""Defines the Mahalanobis distance to the assigned Gaussian."""
# TODO(xavigonzalvo): reuse (input - mean) * cov^-1 * (input -
# mean) from log probability function.
self._all_scores = []
for shard in data:
all_scores = []
shard = tf.expand_dims(shard, 0)
for c in xrange(self._num_classes):
if self._covariance_type == FULL_COVARIANCE:
cov = self._covs[c, :, :]
elif self._covariance_type == DIAG_COVARIANCE:
cov = tf.diag(self._covs[c, :])
inverse = tf.matrix_inverse(cov + self._min_var)
inv_cov = tf.tile(
tf.expand_dims(inverse, 0),
tf.pack([self._num_examples, 1, 1]))
diff = tf.transpose(shard - self._means[c, :, :], perm=[1, 0, 2])
m_left = tf.batch_matmul(diff, inv_cov)
all_scores.append(tf.sqrt(tf.batch_matmul(
m_left, tf.transpose(diff, perm=[0, 2, 1])
)))
self._all_scores.append(tf.reshape(
tf.concat(1, all_scores),
tf.pack([self._num_examples, self._num_classes])))
# Distance to the associated class.
self._all_scores = tf.concat(0, self._all_scores)
assignments = tf.concat(0, self.assignments())
rows = tf.to_int64(tf.range(0, self._num_examples))
indices = tf.concat(1, [tf.expand_dims(rows, 1),
tf.expand_dims(assignments, 1)])
self._scores = tf.gather_nd(self._all_scores, indices)
def update_centers(self, img_dataset):
'''
Optimize:
self.C = (U * hu^T + V * hv^T) (hu * hu^T + hv * hv^T)^{-1}
self.C^T = (hu * hu^T + hv * hv^T)^{-1} (hu * U^T + hv * V^T)
but all the C need to be replace with C^T :
self.C = (hu * hu^T + hv * hv^T)^{-1} (hu^T * U + hv^T * V)
'''
old_C_value = self.sess.run(self.C)
h = self.img_b_all
U = self.img_output_all
smallResidual = tf.constant(
np.eye(self.subcenter_num * self.subspace_num, dtype=np.float32) * 0.001)
Uh = tf.matmul(tf.transpose(h), U)
hh = tf.add(tf.matmul(tf.transpose(h), h), smallResidual)
compute_centers = tf.matmul(tf.matrix_inverse(hh), Uh)
update_C = self.C.assign(compute_centers)
C_value = self.sess.run(update_C, feed_dict={
self.img_output_all: img_dataset.output,
self.img_b_all: img_dataset.codes,
})
C_sums = np.sum(np.square(C_value), axis=1)
C_zeros_ids = np.where(C_sums < 1e-8)
C_value[C_zeros_ids, :] = old_C_value[C_zeros_ids, :]
self.sess.run(self.C.assign(C_value))
def invertible_1x1_conv(z, logdet, reverse=False, name=None, use_bias=False):
with tf.variable_scope(name, "invconv"):
shape = z.get_shape().as_list()
w_shape = [shape[3], shape[3]]
# Sample a random orthogonal matrix:
w_init = np.linalg.qr(np.random.randn(*w_shape))[0].astype('float32')
w = tf.get_variable("W", dtype=tf.float32, initializer=w_init)
det_w = tf.matrix_determinant(tf.cast(w, 'float64'))
dlogdet = tf.cast(tf.log(abs(det_w)), 'float32') * shape[1] * shape[2]
if use_bias:
b = tf.get_variable("bias", [1, 1, 1, shape[3]])
if not reverse:
_w = w[tf.newaxis, tf.newaxis, ...]
z = tf.nn.conv2d(z, _w, [1, 1, 1, 1], 'SAME', data_format='NHWC')
logdet += dlogdet
if use_bias:
z += b
else:
if use_bias:
z -= b
w_inv = tf.matrix_inverse(w)
_w = w_inv[tf.newaxis, tf.newaxis, ...]
z = tf.nn.conv2d(z, _w, [1, 1, 1, 1], 'SAME', data_format='NHWC')
logdet -= dlogdet
return z, logdet
def logpdf(self, x, mean=None, cov=1):
"""Log of the probability density function.
Parameters
----------
x : tf.Tensor
A 1-D or 2-D tensor.
mean : tf.Tensor, optional
A 1-D tensor. Defaults to zero mean.
cov : tf.Tensor, optional
A 1-D or 2-D tensor. Defaults to identity matrix.
Returns
-------
tf.Tensor
A tensor of one dimension less than the input.
"""
x = tf.cast(x, dtype=tf.float32)
x_shape = get_dims(x)
if len(x_shape) == 1:
d = x_shape[0]
else:
d = x_shape[1]
if mean is None:
r = x
else:
mean = tf.cast(mean, dtype=tf.float32)
r = x - mean
if cov is 1:
L_inv = tf.diag(tf.ones([d]))
det_cov = tf.constant(1.0)
else:
cov = tf.cast(cov, dtype=tf.float32)
if len(cov.get_shape()) == 1: # vector
L_inv = tf.diag(1.0 / tf.sqrt(cov))
det_cov = tf.reduce_prod(cov)
else: # matrix
L = tf.cholesky(cov)
L_inv = tf.matrix_inverse(L)
det_cov = tf.pow(tf.reduce_prod(tf.diag_part(L)), 2)
lps = -0.5*d*tf.log(2*np.pi) - 0.5*tf.log(det_cov)
if len(x_shape) == 1: # vector
r = tf.reshape(r, shape=(d, 1))
inner = tf.matmul(L_inv, r)
lps -= 0.5 * tf.matmul(inner, inner, transpose_a=True)
return tf.squeeze(lps)
else: # matrix
# TODO vectorize further
out = []
for r_vec in tf.unpack(r):
r_vec = tf.reshape(r_vec, shape=(d, 1))
inner = tf.matmul(L_inv, r_vec)
out += [tf.squeeze(lps -
0.5 * tf.matmul(inner, inner, transpose_a=True))]
return tf.pack(out)
def predict2():
# predicitions
cov=h.Mul(K_mm_2,tf.matrix_inverse(K_mm_2+K_mnnm_2/tf.square(sigma_2)),K_mm_2)
cov_chol=tf.cholesky(cov)
mu=h.Mul(K_mm_2,tf.cholesky_solve(cov_chol,K_mn_2),Ytr)/tf.square(sigma_2)
mean=h.Mul(K_nm_2,tf.matrix_solve(K_mm_1,mu))
variance=K_nn_2-h.Mul(K_nm_2,h.safe_chol(K_mm_2,tf.transpose(K_nm_2)))
var_terms=2*tf.sqrt(tf.reshape(tf.diag_part(variance)+tf.square(sigma_2),[N,1]))
return mean, var_terms
def predict(K_mn,sigma,K_mm,K_nn):
# predicitions
K_nm=tf.transpose(K_mn)
K_mm_I=tf.matrix_inverse(K_mm)
Sig_Inv=1e-1*np.eye(M)+K_mm+K_mnnm_2/tf.square(sigma)
mu_post=h.Mul(tf.matrix_solve(Sig_Inv,K_mn),Ytr)/tf.square(sigma)
mean=h.Mul(K_nm,mu_post)
variance=K_nn-h.Mul(K_nm,h.safe_chol(K_mm,K_mn))+h.Mul(K_nm,tf.matrix_solve(Sig_Inv,K_mn))
var_terms=2*tf.sqrt(tf.reshape(tf.diag_part(variance)+tf.square(sigma),[N,1]))
return mean, var_terms
def tf_exponent_gauss(mu1, prec, n): ''' takes in precision, and returns cov ''' prec = tf.mul(n, prec) sig = tf.matrix_inverse(prec) mu = tf.matmul(prec, mu1) mu = tf.mul(n, mu) mu = tf.matmul(sig, mu) return mu, sig
def get_q(self): n = tf.constant(self.n, dtype = tf.float64) prec = tf.mul(self.V, n) prec = tf.add(prec, self.SEP_prior_prec) qsig = tf.matrix_inverse(prec) mu = tf.matmul(self.V, self.u) mu = tf.mul(n, mu) qmu = tf.matmul(qsig, mu) return qmu, qsig
def add_predict_op(self):
"""
get predict
todo: get variance of predictions
"""
tmp_K_star=self.get_covariance_x(self.test_x, self.train_x)
self.K_star=tf.reshape(
tf.tile(tf.reshape(self.K_c, [self.config.task_num, 1, self.config.task_num, 1]), [1, self.config.test_length, 1, self.config.train_length]) \
* tf.tile(tf.reshape(tmp_K_star, [1, self.config.test_length, 1, self.config.train_length]), [self.config.task_num, 1, self.config.task_num, 1]), \
[self.config.test_length*self.config.task_num, self.config.train_length*self.config.task_num])
self.predict_op=tf.matmul(tf.matmul(self.K_star, tf.matrix_inverse(self.sigma)), self.train_y)
def read_data(self):
"""Provides images and camera intrinsics."""
with tf.name_scope('data_loading'):
with tf.name_scope('enqueue_paths'):
seed = random.randint(0, 2**31 - 1)
self.file_lists = self.compile_file_list(self.data_dir, 'train')
image_paths_queue = tf.train.string_input_producer(
self.file_lists['image_file_list'], seed=seed, shuffle=True)
cam_paths_queue = tf.train.string_input_producer(
self.file_lists['cam_file_list'], seed=seed, shuffle=True)
img_reader = tf.WholeFileReader()
_, image_contents = img_reader.read(image_paths_queue)
image_seq = tf.image.decode_jpeg(image_contents)
with tf.name_scope('load_intrinsics'):
cam_reader = tf.TextLineReader()
_, raw_cam_contents = cam_reader.read(cam_paths_queue)
rec_def = []
for _ in range(9):
rec_def.append([1.0])
raw_cam_vec = tf.decode_csv(raw_cam_contents, record_defaults=rec_def)
raw_cam_vec = tf.stack(raw_cam_vec)
intrinsics = tf.reshape(raw_cam_vec, [3, 3])
with tf.name_scope('convert_image'):
image_seq = self.preprocess_image(image_seq) # Converts to float.
with tf.name_scope('image_augmentation'):
image_seq = self.augment_image_colorspace(image_seq)
image_stack = self.unpack_images(image_seq)
with tf.name_scope('image_augmentation_scale_crop'):
image_stack, intrinsics = self.augment_images_scale_crop(
image_stack, intrinsics, self.img_height, self.img_width)
with tf.name_scope('multi_scale_intrinsics'):
intrinsic_mat = self.get_multi_scale_intrinsics(intrinsics,
self.num_scales)
intrinsic_mat.set_shape([self.num_scales, 3, 3])
intrinsic_mat_inv = tf.matrix_inverse(intrinsic_mat)
intrinsic_mat_inv.set_shape([self.num_scales, 3, 3])
with tf.name_scope('batching'):
image_stack, intrinsic_mat, intrinsic_mat_inv = (
tf.train.shuffle_batch(
[image_stack, intrinsic_mat, intrinsic_mat_inv],
batch_size=self.batch_size,
capacity=QUEUE_SIZE + QUEUE_BUFFER * self.batch_size,
min_after_dequeue=QUEUE_SIZE))
logging.info('image_stack: %s', util.info(image_stack))
return image_stack, intrinsic_mat, intrinsic_mat_inv
def logpdf(self, x, mean=None, cov=1):
"""
Parameters
----------
x : np.array or tf.Tensor
vector or matrix
mean : np.array or tf.Tensor, optional
vector. Defaults to zero mean.
cov : np.array or tf.Tensor, optional
vector or matrix. Defaults to identity.
"""
x = tf.cast(tf.convert_to_tensor(x), dtype=tf.float32)
x_shape = get_dims(x)
if len(x_shape) == 1:
d = x_shape[0]
else:
d = x_shape[1]
if mean is None:
r = x
else:
mean = tf.cast(tf.convert_to_tensor(mean), dtype=tf.float32)
r = x - mean
if cov is 1:
cov_inv = tf.diag(tf.ones([d]))
det_cov = tf.constant(1.0)
else:
cov = tf.cast(tf.convert_to_tensor(cov), dtype=tf.float32)
if len(cov.get_shape()) == 1: # vector
cov_inv = tf.diag(1.0 / cov)
det_cov = tf.reduce_prod(cov)
else: # matrix
cov_inv = tf.matrix_inverse(cov)
det_cov = tf.matrix_determinant(cov)
lps = -0.5*d*tf.log(2*np.pi) - 0.5*tf.log(det_cov)
if len(x_shape) == 1:
r = tf.reshape(r, shape=(d, 1))
lps -= 0.5 * tf.matmul(tf.matmul(r, cov_inv, transpose_a=True), r)
return tf.squeeze(lps)
else:
# TODO vectorize further
out = []
for r_vec in tf.unpack(r):
r_vec = tf.reshape(r_vec, shape=(d, 1))
out += [tf.squeeze(lps - 0.5 * tf.matmul(
tf.matmul(r_vec, cov_inv, transpose_a=True),
r_vec))]
return tf.pack(out)
"""
def __init__(self):
self.NUM_CLASSES = 1
super().__init__(self.NUM_CLASSES)
trainDataWithBias = tf.cast(np.concatenate((np.ones((self.data.trainData.shape[0], 1)), self.data.trainData), axis=1), tf.float32)
self.normal_w = tf.matmul(tf.matrix_inverse(tf.matmul(tf.transpose(trainDataWithBias), trainDataWithBias)), tf.matmul(tf.transpose(trainDataWithBias), tf.cast(self.data.trainTarget, tf.float32)))
self.normal_y = tf.matmul(self.x_plus_bias, self.normal_w)
self.normal_mse = tf.reduce_mean(tf.square(self.y_ - self.normal_y))/2
self.loss_fcn = tf.reduce_mean(tf.square(self.y_ - self.y))/2
# this value is overriden in the case of normal equations
self.correct_prediction = tf.equal(self.y_, tf.round(self.y))
self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32))
def fit(self, X, y):
n, p = X.shape
q = 1 if len(y.shape) == 1 else y.shape[1]
W = tf.Variable(tf.random_uniform([p,q], -1., 1.))
X = tf.constant(X)
y = tf.constant(y, shape = (n,q))
W = tf.matmul(tf.matrix_inverse(tf.matmul(tf.transpose(X), X)),
tf.matmul(tf.transpose(X), y))
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
clf.W = sess.run(W)
return self
def make_train_tf(self):
x = tf.reshape(self.states, [-1, self.n_hidden_2])
y = tf.reshape(self.targets, [-1, self.n_outputs])
indices = tf.random_shuffle(tf.range(tf.shape(x)[0]))
x_ = tf.gather(x, indices)
y_ = tf.gather(y, indices)
r = tf.matmul(tf.transpose(x_), x_)
p = tf.matmul(tf.transpose(x_), y_)
i = tf.diag(tf.ones([self.n_hidden_2]))
w = tf.matmul(tf.matrix_inverse(r + self.ridge_beta * i), p)
return tf.assign(self.w_hy, w)
def logpdf(self, x, mean=None, cov=1):
"""
Arguments
----------
x: tf.Tensor
vector
mean: tf.Tensor, optional
vector. Defaults to zero mean.
cov: tf.Tensor, optional
vector or matrix. Defaults to identity.
Returns
-------
tf.Tensor
scalar
"""
x = tf.cast(tf.squeeze(x), dtype=tf.float32)
d = get_dims(x)[0]
if mean is None:
r = tf.ones([d]) * x
else:
mean = tf.cast(tf.squeeze(mean), dtype=tf.float32)
r = x - mean
if cov == 1:
cov_inv = tf.diag(tf.ones([d]))
det_cov = tf.constant(1.0)
else:
cov = tf.cast(tf.squeeze(cov), dtype=tf.float32)
if len(cov.get_shape()) == 1:
cov_inv = tf.diag(1.0 / cov)
det_cov = tf.reduce_prod(cov)
else:
cov_inv = tf.matrix_inverse(cov)
det_cov = tf.matrix_determinant(cov)
r = tf.reshape(r, shape=(d, 1))
lps = -0.5*d*tf.log(2*np.pi) - 0.5*tf.log(det_cov) - \
0.5 * tf.matmul(tf.matmul(r, cov_inv, transpose_a=True), r)
"""
# TensorFlow can't reverse-mode autodiff Cholesky
L = tf.cholesky(cov)
L_inv = tf.matrix_inverse(L)
det_cov = tf.pow(tf.matrix_determinant(L), 2)
inner = dot(L_inv, r)
out = -0.5*d*tf.log(2*np.pi) - \
0.5*tf.log(det_cov) - \
0.5*tf.matmul(tf.transpose(inner), inner)
"""
return tf.squeeze(lps)
def get_scg_energy_fn():
"""Get energy function for 2d strongly correlated Gaussian."""
# Avoid recreating tf constants on each invocation of gradients
mu = tf.constant([0., 0.])
sigma = tf.constant([[50.05, -49.95], [-49.95, 50.05]])
sigma_inv = tf.matrix_inverse(sigma)
def energy(x):
"""Unnormalized log density/energy of 2d strongly correlated Gaussian."""
xmmu = x - mu
return .5 * tf.diag_part(
tf.matmul(tf.matmul(xmmu, sigma_inv), tf.transpose(xmmu)))
return energy
def _verifyInverse(self, x):
for np_type in [np.float32, np.float64]:
for adjoint in False, True:
y = x.astype(np_type)
with self.test_session():
# Verify that x^{-1} * x == Identity matrix.
inv = tf.matrix_inverse(y, adjoint=adjoint)
tf_ans = tf.batch_matmul(inv, y, adj_y=adjoint)
np_ans = np.identity(y.shape[-1])
if x.ndim > 2:
tiling = list(y.shape)
tiling[-2:] = [1, 1]
np_ans = np.tile(np_ans, tiling)
out = tf_ans.eval()
self.assertAllClose(np_ans, out)
self.assertShapeEqual(y, tf_ans)
def solve_ridge(x, y, ridge_factor):
with tf.name_scope("solve_ridge"):
# Added a column of ones to the end of the feature matrix for bias
A = tf.concat([x, tf.ones((x.shape.as_list()[0], 1))], axis=1)
# Analytic solution for the ridge regression loss
inv_target = tf.matmul(A, A, transpose_a=True)
np_diag_penalty = ridge_factor * np.ones(
A.shape.as_list()[1], dtype="float32")
# Remove penalty on bias component of weights
np_diag_penalty[-1] = 0.
diag_penalty = tf.constant(np_diag_penalty)
inv_target += tf.diag(diag_penalty)
inv = tf.matrix_inverse(inv_target)
w = tf.matmul(inv, tf.matmul(A, y, transpose_a=True))
return w
def q2part3():
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
trainData, trainTarget, validData, validTarget, testData, testTarget = data_gen()
biasTrainData = tf.convert_to_tensor(trainData)
biasTrainData = tf.reshape(biasTrainData, [3500, -1])
biasVec = tf.convert_to_tensor([1 for i in range(3500)])
biasVec = tf.cast(biasVec, tf.float64)
biasVec = tf.reshape(biasVec, [-1, 3500])
biasTrainData = tf.transpose(biasTrainData)
biasTrainData = tf.concat([biasVec, biasTrainData], 0)
biasTrainData = tf.transpose(biasTrainData)
biasTrainTarget = tf.convert_to_tensor(trainTarget)
biasTrainTarget = tf.cast(biasTrainTarget, tf.float64)
biasValidData = tf.convert_to_tensor(validData)
biasValidData = tf.reshape(biasValidData, [100, -1])
biasVec = tf.convert_to_tensor([1 for i in range(100)])
biasVec = tf.cast(biasVec, tf.float64)
biasVec = tf.reshape(biasVec, [-1, 100])
biasValidData = tf.transpose(biasValidData)
biasValidData = tf.concat([biasVec, biasValidData], 0)
biasValidData = tf.transpose(biasValidData)
biasValidTarget = tf.convert_to_tensor(validTarget)
biasValidTarget = tf.cast(biasValidTarget, tf.float64)
biasTestData = tf.convert_to_tensor(testData)
biasTestData = tf.reshape(biasTestData, [145, -1])
biasVec = tf.convert_to_tensor([1 for i in range(145)])
biasVec = tf.cast(biasVec, tf.float64)
biasVec = tf.reshape(biasVec, [-1, 145])
biasTestData = tf.transpose(biasTestData)
biasTestData = tf.concat([biasVec, biasTestData], 0)
biasTestData = tf.transpose(biasTestData)
biasTestTarget = tf.convert_to_tensor(testTarget)
biasTestTarget = tf.cast(biasTestTarget, tf.float64)
W_norm = tf.matmul(tf.transpose(biasTrainData), biasTrainData)
W_norm = tf.matrix_inverse(W_norm)
W_norm = tf.matmul(W_norm, tf.transpose(biasTrainData))
W_norm = tf.matmul(W_norm, biasTrainTarget)
epoch_size = len(trainData)
batch_size = 500
numBatches = epoch_size / batch_size
# flatten training data
trainData = np.reshape(trainData, [epoch_size, -1])
# flatten validation data
validData = np.reshape(validData, [100, -1])
# flatten test data
testData = np.reshape(testData, [145, -1])
# reshape training target
trainTarget = np.reshape(trainTarget, [epoch_size, -1])
# reshape validation target
validTarget = np.reshape(validTarget, [100, -1])
# reshape test target
testTarget = np.reshape(testTarget, [145, -1])
# variable creation
W = tf.Variable(tf.random_normal(shape=[trainData.shape[1], 1], stddev=0.35, seed=521), name="weights")
# W = tf.Variable(tf.random_normal(shape=[trainData.shape[1], 1], stddev=0.1), name="weights")
# W = tf.Variable(tf.zeros([trainData.shape[1], 1]), dtype=tf.float32)
b = tf.Variable(0.0, name="biases")
X = tf.placeholder(tf.float32, name="input_x")
y_target = tf.placeholder(tf.float32, name="target_y")
# graph definition
y_pred = tf.matmul(X, W) + b
# need to define W_lambda, l_rate
l_rate = tf.placeholder(tf.float32, [], name="learning_rate")
W_lambda = tf.placeholder(tf.float32, [], name="weight_decay")
# error definition
logits_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=y_pred, labels=y_target) / 2.0,
name="logits_loss")
W_decay = tf.reduce_sum(tf.multiply(W, W)) * W_lambda / 2.0
loss = logits_loss + W_decay
# accuracy definition
y_pred_sigmoid = tf.sigmoid(y_pred)
accuracy = tf.reduce_mean(
tf.cast(tf.equal(tf.cast(tf.greater(y_pred_sigmoid, 0.5), tf.float32), y_target), tf.float32))
# training mechanism
optimizer = tf.train.AdamOptimizer(learning_rate=l_rate)
train = optimizer.minimize(loss=loss)
# specify learning rate
each_l_rate = 0.005
# specify weight decay coefficient
each_W_lambda = 0.0
train_error_list = []
# initialize session
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
sess.run(W)
sess.run(b)
for step in range(0, 5000):
batch_idx = step % numBatches
trainDataBatch = trainData[int(batch_idx * batch_size):int((batch_idx + 1) * batch_size)]
trainTargetBatch = trainTarget[int(batch_idx * batch_size):int((batch_idx + 1) * batch_size)]
_, currentW, currentb, yhat = sess.run([train, W, b, y_pred],
feed_dict={X: trainDataBatch, y_target: trainTargetBatch,
l_rate: each_l_rate, W_lambda: each_W_lambda})
if batch_idx == numBatches - 1:
train_error = sess.run(loss, feed_dict={X: trainDataBatch, y_target: trainTargetBatch,
W_lambda: each_W_lambda})
train_accuracy = sess.run(accuracy, feed_dict={X: trainDataBatch, y_target: trainTargetBatch})
training_error = sess.run(loss,
feed_dict={X: trainDataBatch, y_target: trainTargetBatch, W_lambda: each_W_lambda})
train_error_list.append(training_error)
# trainData accuracy
y = tf.convert_to_tensor(trainTarget)
y = tf.reshape(y, [3500,-1])
y = tf.cast(y, tf.float64)
yhat_norm = tf.matmul(biasTrainData, W_norm)
yhat_bound_norm = tf.cast(tf.greater(yhat_norm, 0.5), tf.float64)
equal_norm = tf.cast(tf.equal(y, yhat_bound_norm), tf.float64)
accuracy_norm = tf.reduce_sum(tf.reduce_sum(equal_norm, 0), 0)/3500
print("TrainData Accuracy (linear regression normal equation):", sess.run(accuracy_norm))
accuracy_SGD = sess.run(accuracy, feed_dict={X: trainData, y_target: trainTarget})
print("TrainData Accuracy (logistic regression SGD):", accuracy_SGD)
# validData accuracy
y = tf.convert_to_tensor(validTarget)
y = tf.reshape(y, [100,-1])
y = tf.cast(y, tf.float64)
yhat_norm = tf.matmul(biasValidData, W_norm)
yhat_bound_norm = tf.cast(tf.greater(yhat_norm, 0.5), tf.float64)
equal_norm = tf.cast(tf.equal(y, yhat_bound_norm), tf.float64)
accuracy_norm = tf.reduce_sum(tf.reduce_sum(equal_norm, 0), 0)/100
print("ValidData Accuracy (linear regression normal equation):", sess.run(accuracy_norm))
accuracy_SGD = sess.run(accuracy, feed_dict={X: validData, y_target: validTarget})
print("ValidData Accuracy (logistic regression SGD):", accuracy_SGD)
# testData accuracy
y = tf.convert_to_tensor(testTarget)
y = tf.reshape(y, [145,-1])
y = tf.cast(y, tf.float64)
yhat_norm = tf.matmul(biasTestData, W_norm)
yhat_bound_norm = tf.cast(tf.greater(yhat_norm, 0.5), tf.float64)
equal_norm = tf.cast(tf.equal(y, yhat_bound_norm), tf.float64)
accuracy_norm = tf.reduce_sum(tf.reduce_sum(equal_norm, 0), 0)/145
print("TestData Accuracy (linear regression normal equation):", sess.run(accuracy_norm))
accuracy_SGD = sess.run(accuracy, feed_dict={X: testData, y_target: testTarget})
print("TestData Accuracy (logistic regression SGD):", accuracy_SGD)
# specify learning rate
each_l_rate = 0.001
# specify weight decay coefficient
each_W_lambda = 0.0
train_error_list = []
train_accuracy_list = []
linear_train_error_list = []
linear_train_accuracy_list = []
# variable creation
linear_W = tf.Variable(tf.random_normal(shape=[int(np.shape(trainData)[1]), 1], stddev=0.5, seed=521), name="weights")
# linear_W = tf.Variable(tf.truncated_normal(shape=[int(np.shape(trainData)[1]), 1], stddev=0.1), name="weights")
# linear_W = tf.Variable(tf.zeros([trainData.shape[1], 1]), dtype=tf.float32)
linear_b = tf.Variable(0.0, name="biases")
linear_X = tf.placeholder(tf.float32, [batch_size, int(np.shape(trainData)[1])], name="input_x")
linear_y_target = tf.placeholder(tf.float32, [batch_size, 1], name="target_y")
# graph definition
linear_y_pred = tf.matmul(linear_X,linear_W) + linear_b
linear_l_rate = tf.placeholder(tf.float32, [], name="learning_rate")
linear_W_lambda = tf.placeholder(tf.float32, [], name="weight_decay")
# error definition
linear_mse = tf.reduce_sum(tf.square(linear_y_pred-linear_y_target), name="mse")/(2.0*batch_size)
linear_W_decay = tf.reduce_sum(tf.multiply(linear_W,linear_W))*linear_W_lambda/2
linear_loss = linear_mse + linear_W_decay
# training mechanism
linear_optimizer = tf.train.AdamOptimizer(learning_rate=linear_l_rate)
linear_train = linear_optimizer.minimize(loss=linear_loss)
linear_accuracy = tf.reduce_mean(
tf.cast(tf.equal(tf.cast(tf.greater(linear_y_pred, 0.5), tf.float32), linear_y_target), tf.float32))
# initialize session
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
sess.run(W)
sess.run(b)
sess.run(linear_W)
sess.run(linear_b)
for step in range(0, 5000):
batch_idx = step % numBatches
trainDataBatch = trainData[int(batch_idx * batch_size):int((batch_idx + 1) * batch_size)]
trainTargetBatch = trainTarget[int(batch_idx * batch_size):int((batch_idx + 1) * batch_size)]
_, currentW, currentb, yhat = sess.run([train, W, b, y_pred],
feed_dict={X: trainDataBatch, y_target: trainTargetBatch,
l_rate: each_l_rate, W_lambda: each_W_lambda})
_, currentW, currentb, yhat = sess.run([linear_train, linear_W, linear_b, linear_y_pred],
feed_dict={linear_X: trainDataBatch, linear_y_target: trainTargetBatch,
linear_l_rate: each_l_rate, linear_W_lambda: each_W_lambda})
if batch_idx == numBatches - 1:
train_error = sess.run(loss, feed_dict={X: trainDataBatch, y_target: trainTargetBatch,
W_lambda: each_W_lambda})
linear_train_error = sess.run(linear_loss, feed_dict={linear_X: trainDataBatch, linear_y_target: trainTargetBatch,
linear_W_lambda: each_W_lambda})
train_accuracy = sess.run(accuracy, feed_dict={X: trainDataBatch, y_target: trainTargetBatch})
linear_train_accuracy = sess.run(linear_accuracy, feed_dict={linear_X: trainDataBatch, linear_y_target: trainTargetBatch})
train_error_list.append(train_error)
train_accuracy_list.append(train_accuracy)
linear_train_error_list.append(linear_train_error)
linear_train_accuracy_list.append(linear_train_accuracy)
print("Logistic Regression Cross-Entropy Loss:", train_error_list[-1])
print("Linear Regression MSE Loss:", linear_train_error_list[-1])
print("Logistic Regression Accuracy:", train_accuracy_list[-1])
print("Linear Regression Accuracy:", linear_train_accuracy_list[-1])
plt.clf()
plt.plot(train_error_list)
plt.title('Training Cross-Entropy Loss - Logistic Regression')
plt.xlabel('Number of Epoch')
plt.ylabel('Cross-Entropy Loss')
x1, x2, x3, x4 = plt.axis()
plt.axis((x1, x2, 0, 0.5))
plt.savefig('logistic_regression_error.png')
plt.clf()
plt.plot(linear_train_error_list)
plt.title('Training MSE Loss - Linear Regression')
plt.xlabel('Number of Epoch')
plt.ylabel('MSE Loss')
x1, x2, x3, x4 = plt.axis()
plt.axis((x1, x2, 0, 2))
plt.savefig('linear_regression_loss.png')
plt.clf()
plt.plot(train_accuracy_list)
plt.title('Training Accuracy - Logistic Regression')
plt.xlabel('Number of Epoch')
plt.ylabel('Accuracy')
plt.savefig('logistic_regression_accuracy.png')
plt.clf()
plt.plot(linear_train_accuracy_list)
plt.title('Training Accuracy - Linear Regression')
plt.xlabel('Number of Epoch')
plt.ylabel('Accuracy')
plt.savefig('linear_regression_accuracy.png')
# Comparison between cross-entropy and MSE loss over y = [0,1]
y = 0
yhat = np.linspace(0,1,500)
cross_entropy_loss = -1 * y * np.log(yhat) - (1 - y) * np.log(1 - yhat)
mse_loss = np.power(y - yhat, 2)
plt.clf()
plt.plot(yhat, cross_entropy_loss, label='cross-entropy loss')
plt.plot(yhat, mse_loss, label='MSE loss')
plt.title('Cross-Entropy Loss vs MSE loss')
plt.xlabel('prediction yhat (given y = 0)')
plt.ylabel('Loss')
plt.legend()
plt.savefig('cross_entropy_vs_mse_loss.png')
validData = validData.reshape([validData.shape[0], 28 * 28])
testData = testData.reshape([testData.shape[0], 28 * 28])
#NORMAL EQUATIONS FOR LEAST SQUARE SOLUTION
# set parameters and adjust for bias term
x_bias_train = tf.ones([trainData.shape[0], 1])
x_bias_valid = tf.ones([validData.shape[0], 1])
x_bias_test = tf.ones([testData.shape[0], 1])
trainDataMatrix = tf.concat([x_bias_train, trainData], 1)
validDataMatrix = tf.concat([x_bias_valid, validData], 1)
testDataMatrix = tf.concat([x_bias_test, testData], 1)
# implement normal equations
w_ls = tf.matmul(
tf.matmul(
tf.matrix_inverse(
tf.matmul(tf.transpose(trainDataMatrix), trainDataMatrix)),
tf.transpose(trainDataMatrix)), tf.cast(trainTarget, tf.float32))
# compute MSE for training dataset
Y_pred_train = tf.matmul(trainDataMatrix, w_ls)
Y_pred_valid = tf.matmul(validDataMatrix, w_ls)
Y_pred_test = tf.matmul(testDataMatrix, w_ls)
# compute accuracy for training dataset
Y_pred_train_set = tf.sign(tf.sign(Y_pred_train - 0.5) + 1)
Y_pred_valid_set = tf.sign(tf.sign(Y_pred_valid - 0.5) + 1)
Y_pred_test_set = tf.sign(tf.sign(Y_pred_test - 0.5) + 1)
accuracy_train = tf.contrib.metrics.accuracy(tf.to_int32(Y_pred_train_set),
tf.to_int32(trainTarget))
accuracy_valid = tf.contrib.metrics.accuracy(tf.to_int32(Y_pred_valid_set),
tf.to_int32(validTarget))
all_variables[i].assign(all_variables[i] -
alpha * tf.squeeze(update[i])))
return tf.pack(optmize_variables)
x = tf.Variable(np.random.random_sample(), dtype=tf.float32)
y = tf.Variable(np.random.random_sample(), dtype=tf.float32)
alpha = cons(0.1)
# f = tf.pow(x, cons(2)) + cons(2) * x * y + cons(3) * tf.pow(y, cons(2)) + cons(4) * x + cons(5) * y + cons(6)
f = cons(0.5) * tf.pow(x, 2) + cons(2.5) * tf.pow(y, 2)
all_variables = [x, y]
hessian = compute_hessian(f, all_variables)
hessian_inv = tf.matrix_inverse(hessian)
g = compute_grads(f, all_variables)
update = tf.unpack(tf.matmul(hessian_inv, g))
optimize_op = optimize(all_variables, update)
sess = tf.Session()
sess.run(tf.initialize_all_variables())
func = np.inf
for i in range(10):
prev = func
v1, v2, func = sess.run([x, y, f])
print(v1, v2, func)
sess.run(optimize_op)
def main(_):
if FLAGS.input_type == 'PP': assert ('hrestgt' not in FLAGS.supervision)
tf.logging.set_verbosity(tf.logging.INFO)
tf.set_random_seed(FLAGS.random_seed)
FLAGS.checkpoint_dir += '/%s/' % FLAGS.experiment_name
if not tf.gfile.IsDirectory(FLAGS.checkpoint_dir):
tf.gfile.MakeDirs(FLAGS.checkpoint_dir)
tf.logging.info("Image dir: %s" % FLAGS.image_dir)
if 'hrestgt' in FLAGS.supervision:
tf.logging.info("High-resolution image dir: %s" % FLAGS.image_dir)
if FLAGS.input_type == 'REALESTATE_PP':
data_loader = RealEstateSequenceDataLoader(FLAGS.cameras_glob,
FLAGS.image_dir, True, 1,
FLAGS.shuffle_seq_length,
FLAGS.random_seed)
else:
data_loader = ReplicaSequenceDataLoader(FLAGS.cameras_glob,
FLAGS.image_dir,
FLAGS.hres_image_dir, True,
FLAGS.shuffle_seq_length,
FLAGS.random_seed)
train_batch = data_loader.sample_batch()
# additional input tensors
if 'PP' in FLAGS.input_type:
train_batch['interp_pose'] = interpolate_pose(train_batch["ref_pose"],
train_batch["src_pose"])
train_batch['interp_pose_inv'] = tf.matrix_inverse(
train_batch["interp_pose"], name='interp_pose_inv')
train_batch['intrinsics_inv'] = tf.matrix_inverse(
train_batch['intrinsics'], name='intrinsics_inv')
train_batch['ref_pose_inv'] = tf.matrix_inverse(train_batch['ref_pose'],
name='ref_pose_inv')
train_batch['jitter_pose'] = tf.identity(tf_random_rotation(
FLAGS.rot_factor, FLAGS.tr_factor),
name='jitter_pose')
train_batch['jitter_pose_inv'] = tf.matrix_inverse(
train_batch['jitter_pose'], name='jitter_pose_inv')
if FLAGS.gcn:
coord, support, pix2vertex_lookup = load_mesh_input()
train_batch['p2v'] = tf.dtypes.cast(tf.identity(pix2vertex_lookup,
name='p2v'),
dtype=tf.float32)
train_batch['coord'] = tf.identity(coord, name='coord')
train_batch['support'] = [
tf.SparseTensor(indices=np.asarray(support[i][0],
dtype=np.float32),
values=np.asarray(support[i][1], dtype=np.float32),
dense_shape=np.asarray(support[i][2],
dtype=np.float32))
for i in range(len(support))
]
model = MSI()
train_op = model.build_train_graph(train_batch, FLAGS.min_depth,
FLAGS.max_depth, FLAGS.num_psv_planes,
FLAGS.num_msi_planes,
FLAGS.which_color_pred,
FLAGS.which_loss, FLAGS.learning_rate,
FLAGS.beta1)
model.train(train_op, FLAGS.checkpoint_dir, FLAGS.continue_train,
FLAGS.summary_freq, FLAGS.save_latest_freq, FLAGS.max_steps)
def __affine_shape(self,shapes,r,t,isinv=False):
if isinv:
r = tf.matrix_inverse(r)
t = tf.matmul(-t,r)
shapes = tf.matmul(shapes,r) + t
return shapes
def __call__(self, L, X, HtY, HtH, profile_x, regularization=None):
train_samples = tf.shape(HtY)[0]
X_ZF = tf.matmul(tf.expand_dims(HtY, 1), tf.matrix_inverse(HtH))
X_ZF = tf.squeeze(X_ZF, 1)
loss_ZF = tf.reduce_mean(tf.square(X - X_ZF))
ber_ZF = tf.reduce_mean(
tf.cast(tf.not_equal(X, tf.sign(X_ZF)), tf.float32))
v_size = 2 * self.Nt ## size of the second sub-layer (see table )
hl_size = 8 * self.Nt ## size of the first sub-layer
S = []
S.append(tf.zeros([train_samples, self.Nt]))
a = []
a.append(tf.zeros([train_samples, v_size]))
LOSS = []
LOSS.append(tf.zeros([]))
BER = []
BER.append(tf.zeros([]))
## Regularizing coefficients
lam_da = tf.Variable(0.001, trainable=True)
l1_regularizer = tf.contrib.layers.l1_regularizer(scale=0.005,
scope=None)
weights = tf.trainable_variables() # all vars of your graph
L1_nrom_reg = tf.contrib.layers.apply_regularization(
l1_regularizer, weights)
## Sublayer weights
W1r = []
W2r = []
W3r = []
for i in range(1, L + 1):
u_1 = tf.matmul(tf.expand_dims(S[-1], 1), HtH)
u = tf.squeeze(u_1, 1)
x = tf.concat([HtY, S[-1], u, a[-1]], 1)
Z, W1, b1 = tf.multiply(
relu_layer(x, 3 * self.Nt + v_size, hl_size, 'relu' + str(i)),
profile_x) # 1st sub-layer (18)
W1r.append(W1)
a, W2, b2 = linear_layer(tf.multiply(Z,
profile_x), hl_size, v_size,
'aff' + str(i)) # 2nd sub-layer (19)
a.append(a)
W2r.append(W2)
a[i] = (1 - alpha) * a[i] + alpha * a[i - 1]
S, W3, b3 = sign_layer(tf.multiply(Z, profile_x), hl_size, self.Nt,
'sign' + str(i)) # 3rd sub-layer (10)
S.append(S)
W3r.append(W3)
S[i] = (1 - alpha) * S[i] + alpha * S[i - 1]
error_linear = tf.reduce_mean(
tf.square(X - X_ZF),
1) ## Linear receiver error symbol estimated
error_wesnet = tf.reduce_mean(
tf.square(X - S[-1]),
1) ## WeSNet symbol error symbol estimate
'''
Sparcity inforcing regularisation for for i = l to L;
where l is the layer from which the sparcity is initially enforced. See the paper for the details.
'''
if regularization:
if 10 <= i <= L + 1:
L1_nrom_reg = tf.norm(W1r[i], ord=1) + tf.norm(
W2r[i], ord=1) + tf.norm(
W3r[i], ord=1) ## L1 norm regularizer (23)
regularization_penalty = lam_da * np.log(
1 + (i - 1) * L1_nrom_reg) ## L1 norm regularizer (22)
loss = np.log(i) * tf.reduce_mean(
error_wesnet / error_linear
) + regularization_penalty ## Regularised loss function (21)
else:
loss = np.log(i) * tf.reduce_mean(
error_wesnet /
error_linear) ## Non-regularised loss function (20)
LOSS.append(loss)
BER.append(
tf.reduce_mean(
tf.cast(tf.not_equal(X, tf.sign(S[-1])), tf.float32)))
TOTAL_LOSS = tf.add_n(LOSS)
return TOTAL_LOSS, LOSS, BER
def eigendecomp(A):
d, P = tf.self_adjoint_eig(A)
return P, tf.diag(d), tf.matrix_inverse(P)
import tensorflow as tf
a = tf.constant([[1., 2.], [3., 4.]])
b = tf.constant([[5., 6.], [7., 8.]])
x1 = tf.matmul(a, b)
x2 = tf.matrix_determinant(a)
x3 = tf.matrix_inverse(a)
with tf.Session() as sess:
y1, y2, y3 = sess.run([x1, x2, x3])
print(y1)
print(y2)
print(y3)
def train_cholesky():
tf.reset_default_graph()
dtype = tf.float64
XY0 = np.genfromtxt('linear-cholesky-XY.csv', delimiter=",")
X0 = XY0[:-1, :] # 2 x d
Y0 = XY0[-1:, :] # 1 x d
dsize = X0.shape[1]
fsize = X0.shape[0]
X_ = tf.placeholder(dtype, shape=X0.shape)
Y_ = tf.placeholder(dtype, shape=Y0.shape)
X = tf.Variable(X_, trainable=False)
Y = tf.Variable(Y_, trainable=False)
W = tf.Variable([[2, 1]], dtype=dtype)
error = tf.matmul(W, X) - Y # 1 x d
loss = tf.reduce_sum(tf.square(error)) / dsize
sess = tf.Session()
cov_ = tf.matmul(X_, tf.transpose(X_)) / dsize
cov = tf.Variable(cov_, trainable=False)
covI = tf.Variable(tf.matrix_inverse(cov_), trainable=False)
init_dict = {X_: X0, Y_: Y0}
lr = tf.constant(1, dtype=dtype)
opt = tf.train.GradientDescentOptimizer(learning_rate=lr)
(grad, var) = opt.compute_gradients(loss, tf.trainable_variables())[0]
ones_list = tf.constant([1.0] * fsize, dtype=dtype)
# decomposition of covariance matrix
Ch = tf.cholesky(cov)
D1 = tf.diag(tf.diag_part(Ch))
L = Ch @ tf.matrix_inverse(D1)
cov2 = L @ D1 @ D1 @ tf.transpose(L)
# decomposition of precision matrix
T = tf.matrix_inverse(L)
covI2 = tf.transpose(T) @ tf.matrix_inverse(D1 @ D1) @ T
# 1/2 of precision matrix recovers inverse Hessian
pre = covI2 / 2
grad = tf.matmul(grad, pre)
train_op = opt.apply_gradients([(grad, var)])
sess.run(tf.global_variables_initializer(), feed_dict=init_dict)
assert sess.run(tf.norm(cov2 - cov)) == 0.
assert sess.run(tf.norm(covI2 - covI)) == 0.
observed_losses = []
for i in range(100):
loss0, _ = sess.run([loss, train_op])
observed_losses.append(loss0)
# from accompanying notebook
# [1.03475, 4.90422*10^-28, 0., 0., 0., 0., 0., 0., 0., 0....
expected_losses = np.loadtxt("linear-cholesky-losses1.csv")
np.testing.assert_allclose(expected_losses[:10],
observed_losses[:10],
rtol=1e-12,
atol=1e-12)
def body(Sin, i):
# We distinguish the case of diagonal and full covariance matrices
# because computations can be made more efficiently for diagonal ones.
n = self.enr_unique_counts[i[-1]] # Number of sessions
start = self.enr_counts_offset[i[-1]]
end = self.enr_counts_offset[i[-1] + 1]
M1 = self.M1[start:end, :]
I_1xn1 = tf.transpose(
tf.ones_like(M1[:, 0:1])
) # "0:1" is just so that shape becomes (?,1) instead of (?,)
if (self.kaldi_norm):
norm_vector = 1.0 / (B + 1.0 / tf.cast(n, self.floatX))
norm = tf.reduce_sum((M1**2) * norm_vector,
axis=1,
keep_dims=True)
norm = tf.sqrt(self.dim / norm) # Get dim!!!!!!!!!
M1 = M1 * norm
if self.diag_cov:
P0 = 1.0 / (W + B / W
) # Take this and the below out from "body" ??
C0 = 0.5 * tf.reduce_sum(
tf.log(P0)
) # We don't include 2pi^(0.5d) since it will be cancelled out.
G = B / (tf.cast(n, self.floatX) * B + W
) # ( d x d ) Need this at two places
M1n = tf.cast(
n, self.floatX
) * M1 * G # ( n1 x d ) The "normalized" enrollemnt means
P = 1.0 / (W + G) # Precision
C = 0.5 * tf.reduce_sum(tf.log(P))
# For each normalized enrolment ivector, m1n, and each test i-vector, m2, we would like
# to calculate (m1n - ,2)P(m1n - ,2)'. We do that with a couple of matrix products rather
# than a loop. It could be worth thinking about whether it can be done with one operation
# involving 3D tensors.
S1 = tf.matmul(M1n, tf.transpose(M2 * P))
S2_1 = tf.matmul(
tf.reduce_sum(((M1n * P) * M1n), axis=1)[:, None], I_1xn2)
S2_2 = tf.matmul(
tf.reduce_sum(((M2 * P) * M2), axis=1)[:, None], I_1xn1)
S2_2_0 = tf.matmul(
tf.reduce_sum(((M2 * P0) * M2), axis=1)[:, None], I_1xn1)
else:
# THIS IS WRONG
P0 = tf.matrix_inverse(W + tf.matmul(B, tf.matrix_inverse(W)))
C0 = 0.5 * tf.reduce_sum(
tf.log(tf.diag_part(P0))
) # We don't include 2pi^(0.5d) since it will be cancelled out.
#n = 1 # Number of sessions
G = tf.matmul(
B,
tf.matrix_inverse(n * B +
W)) # ( d x d ) Need this at two places
M1n = n * tf.matmul(
M1, G) # ( n1 x d ) The "normalized" enrollemnt means
P = tf.matrix_inverse(W + G) # Precision
#S, C = 0.5*tf.slogdet(P) # We don't include 2pi^(0.5d) since it will be cancelled out.
C = 0.5 * tf.reduce_sum(tf.log(tf.diag_part(P)))
# For each normalized enrolment ivector, m1n, and each test i-vector, m2, we would like
# to calculate (m1n - ,2)P(m1n - ,2)'. We do that with a couple of matrix products rather
# than a loop. It could be worth thinking about whether it can be done with one operation
# involving 3D tensors.
S1 = tf.matmul(M1n, tf.matmul(P, M2, transpose_b=True))
S2_1 = tf.matmul(
tf.reduce_sum(((tf.matmul(M1n, P)) * M1n),
axis=1)[:, None], I_1xn2)
S2_2 = tf.matmul(
tf.reduce_sum(((tf.matmul(M2, P)) * M2), axis=1)[:, None],
I_1xn1)
S2_2_0 = tf.matmul(
tf.reduce_sum(((tf.matmul(M2, P0)) * M2), axis=1)[:, None],
I_1xn1)
S = 0.5 * (2 * S1 - S2_1 - tf.transpose(S2_2) +
tf.transpose(S2_2_0)) + C - C0
S = tf.concat([Sin, S], axis=0)
return [S, tf.add(i, 1)]
def invertible_1x1_conv(name, z, logdet, reverse=False):
if True: # Set to "False" to use the LU-decomposed version
with tf.variable_scope(name):
shape = Z.int_shape(z)
w_shape = [shape[3], shape[3]]
# Sample a random orthogonal matrix:
w_init = np.linalg.qr(
np.random.randn(*w_shape))[0].astype('float32')
w = tf.get_variable("W", dtype=tf.float32, initializer=w_init)
# dlogdet = tf.linalg.LinearOperator(w).log_abs_determinant() * shape[1]*shape[2]
dlogdet = tf.cast(
tf.log(abs(tf.matrix_determinant(tf.cast(w, 'float64')))),
'float32') * shape[1] * shape[2]
if not reverse:
_w = tf.reshape(w, [1, 1] + w_shape)
z = tf.nn.conv2d(z,
_w, [1, 1, 1, 1],
'SAME',
data_format='NHWC')
logdet += dlogdet
return z, logdet
else:
_w = tf.matrix_inverse(w)
_w = tf.reshape(_w, [1, 1] + w_shape)
z = tf.nn.conv2d(z,
_w, [1, 1, 1, 1],
'SAME',
data_format='NHWC')
logdet -= dlogdet
return z, logdet
else:
# LU-decomposed version
shape = Z.int_shape(z)
with tf.variable_scope(name):
dtype = 'float64'
# Random orthogonal matrix:
import scipy
np_w = scipy.linalg.qr(np.random.randn(
shape[3], shape[3]))[0].astype('float32')
np_p, np_l, np_u = scipy.linalg.lu(np_w)
np_s = np.diag(np_u)
np_sign_s = np.sign(np_s)
np_log_s = np.log(abs(np_s))
np_u = np.triu(np_u, k=1)
p = tf.get_variable("P", initializer=np_p, trainable=False)
l = tf.get_variable("L", initializer=np_l)
sign_s = tf.get_variable("sign_S",
initializer=np_sign_s,
trainable=False)
log_s = tf.get_variable("log_S", initializer=np_log_s)
# S = tf.get_variable("S", initializer=np_s)
u = tf.get_variable("U", initializer=np_u)
p = tf.cast(p, dtype)
l = tf.cast(l, dtype)
sign_s = tf.cast(sign_s, dtype)
log_s = tf.cast(log_s, dtype)
u = tf.cast(u, dtype)
w_shape = [shape[3], shape[3]]
l_mask = np.tril(np.ones(w_shape, dtype=dtype), -1)
l = l * l_mask + tf.eye(*w_shape, dtype=dtype)
u = u * np.transpose(l_mask) + tf.diag(sign_s * tf.exp(log_s))
w = tf.matmul(p, tf.matmul(l, u))
if True:
u_inv = tf.matrix_inverse(u)
l_inv = tf.matrix_inverse(l)
p_inv = tf.matrix_inverse(p)
w_inv = tf.matmul(u_inv, tf.matmul(l_inv, p_inv))
else:
w_inv = tf.matrix_inverse(w)
w = tf.cast(w, tf.float32)
w_inv = tf.cast(w_inv, tf.float32)
log_s = tf.cast(log_s, tf.float32)
if not reverse:
w = tf.reshape(w, [1, 1] + w_shape)
z = tf.nn.conv2d(z,
w, [1, 1, 1, 1],
'SAME',
data_format='NHWC')
logdet += tf.reduce_sum(log_s) * (shape[1] * shape[2])
return z, logdet
else:
w_inv = tf.reshape(w_inv, [1, 1] + w_shape)
z = tf.nn.conv2d(z,
w_inv, [1, 1, 1, 1],
'SAME',
data_format='NHWC')
logdet -= tf.reduce_sum(log_s) * (shape[1] * shape[2])
return z, logdet
def batch_ll(Kt, t, Y, sigma2_n, sigma2_g, A, W, N, T, S):
"""
Approximates log-expected likelihood term using Monte-Carlo samples.
Args:
Kt: Tensor of shape N x N containing kernel values at observation times `t'.
t: Tensor of shape N x 1 containing observation times.
Y: Tensor of shape T x N containing T observations from N nodes.
sigma2_n: Tensor of shape () containing observation noise variance.
sigma2_g: Tensor of shape () containing connection noise variance
A: Tensor of shape S x N x N containing samples from A for each connection ij.
W: Tensor of shape S x N x N containing samples from W for each connection ij.
N: Number of nodes.
T: Number of observation per nodes.
S: Number of samples.
Returns:
Tesnro of shape () which contains approximated expected log-likelihood.
"""
matmul = tf.matmul
I = tf.constant(np.identity(N), dtype=Latnet.FLOAT)
B = tf.multiply(A, W)
IB = tf.subtract(I, B)
L = tf.matrix_inverse(IB)
Eg = matmul(matmul(L, matmul(B, tf.transpose(B, [0, 2, 1]))),
tf.transpose(L, [0, 2, 1]))
Sn = tf.add(tf.multiply(sigma2_g, Eg), tf.multiply(sigma2_n, I))
Kf = tf.matrix_inverse(matmul(tf.transpose(IB, [0, 2, 1]), IB))
lt, Qt = tf.self_adjoint_eig(Kt)
lt = tf.cast(lt, Latnet.FLOAT)
Qt = tf.cast(Qt, Latnet.FLOAT)
ln, Qn = tf.self_adjoint_eig(Sn)
Ln_inv = tf.matrix_diag(
tf.sqrt(tf.math.divide(tf.constant(1., dtype=Latnet.FLOAT), ln)))
Ln_inv_Qn = matmul(Ln_inv, tf.transpose(Qn, [0, 2, 1]))
Ktilda_f = matmul(matmul(Ln_inv_Qn, Kf),
tf.transpose(Ln_inv_Qn, [0, 2, 1]))
ltilda_f, Qtilda_f = tf.self_adjoint_eig(Ktilda_f)
Lt_Lf = tf.add(
matmul(
tf.transpose(
tf.expand_dims(tf.tile(tf.expand_dims(lt, -1), [1, S]),
-3)), tf.expand_dims(ltilda_f, -2)),
tf.constant(1.0, dtype=Latnet.FLOAT))
logdet = tf.multiply(tf.cast(T, Latnet.FLOAT), tf.reduce_sum(
tf.log(ln))) + tf.reduce_sum(tf.log(Lt_Lf))
Ytilda = matmul(matmul(tf.tile(tf.expand_dims(Y, -3), [S, 1, 1]), Qn),
Ln_inv)
Qt_expanded = tf.tile(tf.expand_dims(Qt, -3), [S, 1, 1])
Ytf = tf.multiply(
tf.math.divide(tf.constant(1.0, dtype=Latnet.FLOAT), Lt_Lf),
matmul(tf.transpose(Qt_expanded, [0, 2, 1]),
matmul(Ytilda, Qtilda_f)))
ySy = tf.reduce_sum(
tf.matrix_diag_part(
matmul(
matmul(
matmul(tf.transpose(Ytilda, [0, 2, 1]), Qt_expanded),
Ytf), tf.transpose(Qtilda_f, [0, 2, 1]))))
_half = tf.constant(0.5, dtype=Latnet.FLOAT)
return tf.multiply(-_half, tf.math.divide(logdet, S)) + tf.multiply(
-_half, tf.math.divide(
ySy, S)) - (N * T) / tf.constant(2.0, Latnet.FLOAT) * tf.log(
tf.constant(2.0, dtype=Latnet.FLOAT) * np.pi)
def get(self):
""" Provides input data to the graph. """
# calculate size of each record (this lists what is contained in the db and how many bytes are occupied)
record_bytes = 2
encoding_bytes = 4
kp_xyz_entries = 3 * self.num_kp
record_bytes += encoding_bytes * kp_xyz_entries
encoding_bytes = 4
kp_uv_entries = 2 * self.num_kp
record_bytes += encoding_bytes * kp_uv_entries
cam_matrix_entries = 9
record_bytes += encoding_bytes * cam_matrix_entries
image_bytes = self.image_size[0] * self.image_size[1] * 3
record_bytes += image_bytes
hand_parts_bytes = self.image_size[0] * self.image_size[1]
record_bytes += hand_parts_bytes
kp_vis_bytes = self.num_kp
record_bytes += kp_vis_bytes
""" READ DATA ITEMS"""
# Start reader
reader = tf.FixedLengthRecordReader(header_bytes=0,
record_bytes=record_bytes)
_, value = reader.read(
tf.train.string_input_producer([self.path_to_db]))
# decode to floats
bytes_read = 0
data_dict = dict()
record_bytes_float32 = tf.decode_raw(value, tf.float32)
# 1. Read keypoint xyz
keypoint_xyz = tf.reshape(
tf.slice(record_bytes_float32, [bytes_read // 4],
[kp_xyz_entries]), [self.num_kp, 3])
bytes_read += encoding_bytes * kp_xyz_entries
# calculate palm coord
if not self.use_wrist_coord:
palm_coord_l = tf.expand_dims(
0.5 * (keypoint_xyz[0, :] + keypoint_xyz[12, :]), 0)
palm_coord_r = tf.expand_dims(
0.5 * (keypoint_xyz[21, :] + keypoint_xyz[33, :]), 0)
keypoint_xyz = tf.concat([
palm_coord_l, keypoint_xyz[1:21, :], palm_coord_r,
keypoint_xyz[-20:, :]
], 0)
# 2. Read keypoint uv
keypoint_uv = tf.cast(
tf.reshape(
tf.slice(record_bytes_float32, [bytes_read // 4],
[kp_uv_entries]), [self.num_kp, 2]), tf.int32)
bytes_read += encoding_bytes * kp_uv_entries
keypoint_uv = tf.cast(keypoint_uv, tf.float32)
# calculate palm coord
if not self.use_wrist_coord:
palm_coord_uv_l = tf.expand_dims(
0.5 * (keypoint_uv[0, :] + keypoint_uv[12, :]), 0)
palm_coord_uv_r = tf.expand_dims(
0.5 * (keypoint_uv[21, :] + keypoint_uv[33, :]), 0)
keypoint_uv = tf.concat([
palm_coord_uv_l, keypoint_uv[1:21, :], palm_coord_uv_r,
keypoint_uv[-20:, :]
], 0)
if self.coord_uv_noise:
noise = tf.truncated_normal([42, 2],
mean=0.0,
stddev=self.coord_uv_noise_sigma)
keypoint_uv += noise
# 3. Read camera intrinsics
cam_mat = tf.reshape(
tf.slice(record_bytes_float32, [bytes_read // 4],
[cam_matrix_entries]), [3, 3])
bytes_read += encoding_bytes * cam_matrix_entries
data_dict['cam_mat'] = cam_mat
# decode to uint8
bytes_read += 2
record_bytes_uint8 = tf.decode_raw(value, tf.uint8)
# 4. Read image
image = tf.reshape(
tf.slice(record_bytes_uint8, [bytes_read], [image_bytes]),
[self.image_size[0], self.image_size[1], 3])
image = tf.cast(image, tf.float32)
bytes_read += image_bytes
# subtract mean
image = image / 255.0 - 0.5
#image = image / 255.0
if self.hue_aug:
image = tf.image.random_hue(image, self.hue_aug_max)
data_dict['image'] = image
# 5. Read mask
hand_parts_mask = tf.reshape(
tf.slice(record_bytes_uint8, [bytes_read], [hand_parts_bytes]),
[self.image_size[0], self.image_size[1]])
hand_parts_mask = tf.cast(hand_parts_mask, tf.int32)
bytes_read += hand_parts_bytes
'''
data_dict['hand_parts'] = hand_parts_mask
hand_mask = tf.greater(hand_parts_mask, 1)#@@yafei: 1->0
bg_mask = tf.logical_not(hand_mask)
data_dict['hand_mask'] = tf.cast(tf.stack([bg_mask, hand_mask], 2), tf.int32)
'''
# 6. Read visibilty
keypoint_vis_val = tf.reshape(
tf.slice(record_bytes_uint8, [bytes_read], [kp_vis_bytes]),
[self.num_kp])
keypoint_vis = tf.cast(keypoint_vis_val, tf.bool)
bytes_read += kp_vis_bytes
data_dict['vis_val'] = keypoint_vis_val
# calculate palm visibility
if not self.use_wrist_coord:
palm_vis_l = tf.expand_dims(
tf.logical_or(keypoint_vis[0], keypoint_vis[12]), 0)
palm_vis_r = tf.expand_dims(
tf.logical_or(keypoint_vis[21], keypoint_vis[33]), 0)
keypoint_vis = tf.concat([
palm_vis_l, keypoint_vis[1:21], palm_vis_r, keypoint_vis[-20:]
], 0)
#data_dict['keypoint_vis'] = keypoint_vis
assert bytes_read == record_bytes, "Doesnt add up."
""" DEPENDENT DATA ITEMS: SUBSET of 21 keypoints"""
'''
# figure out dominant hand by analysis of the segmentation mask
one_map, zero_map = tf.ones_like(hand_parts_mask), tf.zeros_like(hand_parts_mask)
cond_l = tf.logical_and(tf.greater(hand_parts_mask, one_map), tf.less(hand_parts_mask, one_map*18))#@@yafei_0607:onemap*18 -> onemap
cond_r = tf.greater(hand_parts_mask, one_map*17)#@@yafei_0607:onemap*17 -> zeromap
hand_map_l = tf.where(cond_l, one_map, zero_map)
hand_map_r = tf.where(cond_r, one_map, zero_map)
num_px_left_hand = tf.reduce_sum(hand_map_l)
num_px_right_hand = tf.reduce_sum(hand_map_r)
# PRODUCE the 21 subset using the segmentation masks
# We only deal with the more prominent hand for each frame and discard the second set of keypoints
kp_coord_xyz_left = keypoint_xyz[:21, :]
kp_coord_xyz_right = keypoint_xyz[-21:, :]
cond_left = tf.logical_and(tf.cast(tf.ones_like(kp_coord_xyz_left), tf.bool), tf.greater(num_px_left_hand, num_px_right_hand))
kp_coord_xyz21 = tf.where(cond_left, kp_coord_xyz_left, kp_coord_xyz_right)
hand_side = tf.where(tf.greater(num_px_left_hand, num_px_right_hand),
tf.constant(0, dtype=tf.int32),
tf.constant(1, dtype=tf.int32)) # left hand = 0; right hand = 1
data_dict['hand_side'] = tf.one_hot(hand_side, depth=2, on_value=1.0, off_value=0.0, dtype=tf.float32)
data_dict['keypoint_xyz21'] = kp_coord_xyz21
'''
# GANerated Dataset has only left hand
data_dict['hand_side'] = tf.constant([1.0, 0.0])
data_dict['keypoint_xyz21'] = keypoint_xyz[:21, :]
kp_coord_xyz21 = keypoint_xyz[:21, :]
# make coords relative to root joint
kp_coord_xyz_root = kp_coord_xyz21[0, :] # this is the palm coord
kp_coord_xyz21_rel = kp_coord_xyz21 - kp_coord_xyz_root # relative coords in metric coords
'''
index_root_bone_length = tf.sqrt(tf.reduce_sum(tf.square(kp_coord_xyz21_rel[12, :] - kp_coord_xyz21_rel[11, :])))
data_dict['keypoint_scale'] = index_root_bone_length
data_dict['keypoint_xyz21_normed'] = kp_coord_xyz21_rel / index_root_bone_length # normalized by length of 12->11
# calculate local coordinates
kp_coord_xyz21_local = bone_rel_trafo(data_dict['keypoint_xyz21_normed'])
kp_coord_xyz21_local = tf.squeeze(kp_coord_xyz21_local)
data_dict['keypoint_xyz21_local'] = kp_coord_xyz21_local
'''
# calculate viewpoint and coords in canonical coordinates
data_dict[
'keypoint_xyz21_normed'] = kp_coord_xyz21_rel #keypoint_xyz[:21, :]
kp_coord_xyz21_rel_can, rot_mat = canonical_trafo(
data_dict['keypoint_xyz21_normed'])
kp_coord_xyz21_rel_can, rot_mat = tf.squeeze(
kp_coord_xyz21_rel_can), tf.squeeze(rot_mat)
#kp_coord_xyz21_rel_can = flip_right_hand(kp_coord_xyz21_rel_can, tf.logical_not(cond_left))
data_dict['keypoint_xyz21_can'] = kp_coord_xyz21_rel_can
data_dict['rot_mat'] = tf.matrix_inverse(rot_mat)
'''
# Set of 21 for visibility
keypoint_vis_left = keypoint_vis[:21]
keypoint_vis_right = keypoint_vis[-21:]
keypoint_vis21 = tf.where(cond_left[:, 0], keypoint_vis_left, keypoint_vis_right)
data_dict['keypint_vis21'] = keypoint_vis21
'''
keypoint_vis21 = keypoint_vis[:21]
data_dict['keypoint_vis21'] = keypoint_vis21
#data_dict['keypoint_vis21'] = tf.cast(tf.ones_like(keypoint_vis21),tf.bool)
'''
# Set of 21 for UV coordinates
keypoint_uv_left = keypoint_uv[:21, :]
keypoint_uv_right = keypoint_uv[-21:, :]
keypoint_uv21 = tf.where(cond_left[:, :2], keypoint_uv_left, keypoint_uv_right)
data_dict['keypoint_uv21'] = keypoint_uv21
'''
keypoint_uv21 = keypoint_uv[:21, :]
data_dict['keypoint_uv21'] = keypoint_uv21
""" DEPENDENT DATA ITEMS: HAND CROP """
if self.hand_crop:
crop_center = keypoint_uv21[12, ::-1]
# catch problem, when no valid kp available (happens almost never)
crop_center = tf.cond(tf.reduce_all(tf.is_finite(crop_center)),
lambda: crop_center,
lambda: tf.constant([0.0, 0.0]))
crop_center.set_shape([
2,
])
if self.crop_center_noise:
noise = tf.truncated_normal(
[2], mean=0.0, stddev=self.crop_center_noise_sigma)
crop_center += noise
crop_scale_noise = tf.constant(1.0)
if self.crop_scale_noise:
crop_scale_noise = tf.squeeze(
tf.random_uniform([1], minval=1.0, maxval=1.2))
# select visible coords only
kp_coord_h = tf.boolean_mask(keypoint_uv21[:, 1], keypoint_vis21)
kp_coord_w = tf.boolean_mask(keypoint_uv21[:, 0], keypoint_vis21)
kp_coord_hw = tf.stack([kp_coord_h, kp_coord_w], 1)
# determine size of crop (measure spatial extend of hw coords first)
min_coord = tf.maximum(tf.reduce_min(kp_coord_hw, 0), 0.0)
max_coord = tf.minimum(tf.reduce_max(kp_coord_hw, 0),
self.image_size)
# find out larger distance wrt the center of crop
crop_size_best = 2 * tf.maximum(max_coord - crop_center,
crop_center - min_coord)
crop_size_best = tf.reduce_max(crop_size_best)
crop_size_best = tf.minimum(tf.maximum(crop_size_best, 50.0),
500.0)
# catch problem, when no valid kp available
crop_size_best = tf.cond(
tf.reduce_all(tf.is_finite(crop_size_best)),
lambda: crop_size_best, lambda: tf.constant(200.0))
crop_size_best.set_shape([])
# calculate necessary scaling
scale = tf.cast(self.crop_size, tf.float32) / crop_size_best
scale = tf.minimum(tf.maximum(scale, 1.0), 10.0)
scale *= crop_scale_noise
data_dict['crop_scale'] = scale
if self.crop_offset_noise:
noise = tf.truncated_normal(
[2], mean=0.0, stddev=self.crop_offset_noise_sigma)
crop_center += noise
# Crop image
img_crop = crop_image_from_xy(tf.expand_dims(image, 0),
crop_center, self.crop_size, scale)
#data_dict['image_crop'] = tf.squeeze(img_crop)
# Modify uv21 coordinates
crop_center_float = tf.cast(crop_center, tf.float32)
keypoint_uv21_u = (keypoint_uv21[:, 0] - crop_center_float[1]
) * scale + self.crop_size // 2
keypoint_uv21_v = (keypoint_uv21[:, 1] - crop_center_float[0]
) * scale + self.crop_size // 2
keypoint_uv21 = tf.stack([keypoint_uv21_u, keypoint_uv21_v], 1)
data_dict['keypoint_uv21'] = keypoint_uv21
# Modify camera intrinsics
scale = tf.reshape(scale, [
1,
])
scale_matrix = tf.dynamic_stitch([
[0], [1], [2], [3], [4], [5], [6], [7], [8]
], [scale, [0.0], [0.0], [0.0], scale, [0.0], [0.0], [0.0], [1.0]])
scale_matrix = tf.reshape(scale_matrix, [3, 3])
crop_center_float = tf.cast(crop_center, tf.float32)
trans1 = crop_center_float[0] * scale - self.crop_size // 2
trans2 = crop_center_float[1] * scale - self.crop_size // 2
trans1 = tf.reshape(trans1, [
1,
])
trans2 = tf.reshape(trans2, [
1,
])
trans_matrix = tf.dynamic_stitch(
[[0], [1], [2], [3], [4], [5], [6], [7], [8]],
[[1.0], [0.0], -trans2, [0.0], [1.0], -trans1, [0.0], [0.0],
[1.0]])
trans_matrix = tf.reshape(trans_matrix, [3, 3])
data_dict['cam_mat'] = tf.matmul(trans_matrix,
tf.matmul(scale_matrix, cam_mat))
""" DEPENDENT DATA ITEMS: Scoremap from the SUBSET of 21 keypoints"""
# create scoremaps from the subset of 2D annoataion
keypoint_hw21 = tf.stack([keypoint_uv21[:, 1], keypoint_uv21[:, 0]],
-1)
scoremap_size = self.image_size
if self.hand_crop:
scoremap_size = (self.crop_size, self.crop_size)
scoremap = self.create_multiple_gaussian_map(keypoint_hw21,
scoremap_size,
self.sigma,
valid_vec=keypoint_vis21)
if self.scoremap_dropout:
scoremap = tf.nn.dropout(scoremap,
self.scoremap_dropout_prob,
noise_shape=[1, 1, 21])
scoremap *= self.scoremap_dropout_prob
data_dict['scoremap'] = scoremap
if self.scale_to_size:
image, keypoint_uv21, keypoint_vis21 = data_dict[
'image'], data_dict['keypoint_uv21'], data_dict[
'keypoint_vis21']
s = image.get_shape().as_list()
image = tf.image.resize_images(image, self.scale_target_size)
scale = (self.scale_target_size[0] / float(s[0]),
self.scale_target_size[1] / float(s[1]))
keypoint_uv21 = tf.stack([
keypoint_uv21[:, 0] * scale[1], keypoint_uv21[:, 1] * scale[0]
], 1)
data_dict = dict(
) # delete everything else because the scaling makes the data invalid anyway
data_dict['image'] = image
data_dict['keypoint_uv21'] = keypoint_uv21
data_dict['keypoint_vis21'] = keypoint_vis21
elif self.random_crop_to_size:
tensor_stack = tf.concat([
data_dict['image'],
tf.expand_dims(tf.cast(data_dict['hand_parts'], tf.float32),
-1),
tf.cast(data_dict['hand_mask'], tf.float32)
], 2)
s = tensor_stack.get_shape().as_list()
tensor_stack_cropped = tf.random_crop(
tensor_stack,
[self.random_crop_size, self.random_crop_size, s[2]])
data_dict = dict(
) # delete everything else because the random cropping makes the data invalid anyway
data_dict['image'], data_dict['scoremap'], data_dict['keypoint_vis21'] = tensor_stack_cropped[:, :, :3],\
tf.cast(scoremap, tf.float32),\
tf.cast(keypoint_vis21, tf.int32)
names, tensors = zip(*data_dict.items())
print(names)
if self.shuffle:
tensors = tf.train.shuffle_batch_join([tensors],
batch_size=self.batch_size,
capacity=100,
min_after_dequeue=50,
enqueue_many=False)
else:
tensors = tf.train.batch_join([tensors],
batch_size=self.batch_size,
capacity=100,
enqueue_many=False)
return dict(zip(names, tensors))
combination2 = tf.matmul(tf.matmul(tf.transpose(H), H), xk)
concatenation = tf.concat([combination1, xk, combination2, vk],
0,
name="Concatenate")
Cal1 = tf.matmul(w1k, concatenation) + b1k
zk = tf.nn.relu(Cal1, name="Zk")
cal2 = tf.matmul(w2k, zk) + b2k
t = 0.5
xk_1 = -1 + tf.nn.relu(cal2 + t) / abs(t) - tf.nn.relu(cal2 - t) / abs(t)
vk_1 = tf.matmul(w3k, zk) + b3k
#Define loss function and backpropagation algorithm
xwave_part1 = tf.matrix_inverse(tf.matmul(tf.transpose(H), H))
xwave_part2 = tf.matmul(xwave_part1, tf.transpose(H))
xwave = tf.matmul(xwave_part2, y)
lossfunction = tf.log(
tf.reduce_sum(tf.squared_difference(x, xk_1)) /
tf.reduce_sum(tf.squared_difference(x, xwave)))
train_step = tf.train.AdamOptimizer(0.001).minimize(lossfunction)
#Create a session to run Tensorflow
sess = tf.Session()
init_op = tf.global_variables_initializer()
sess.run(init_op)
print(sess.run(w1k))
print(sess.run(w2k))
print(sess.run(w3k))
avg_dim_variance,
shape=[COMPONENTS, DIMENSIONS, DIMENSIONS])
initial_weights = tf.placeholder_with_default(tf.cast(
tf.constant(1.0 / COMPONENTS, shape=[COMPONENTS]), tf.float64),
shape=[COMPONENTS])
# trainable variables: component means, covariances, and weights
means = tf.Variable(initial_means, dtype=tf.float64)
covariances = tf.Variable(initial_covariances, dtype=tf.float64)
weights = tf.Variable(initial_weights, dtype=tf.float64)
# E-step: recomputing responsibilities with respect to the current parameter values
differences = tf.subtract(tf.expand_dims(input, 0), tf.expand_dims(means, 1))
diff_times_inv_cov = tf.matmul(differences, tf.matrix_inverse(covariances))
sum_sq_dist_times_inv_cov = tf.reduce_sum(diff_times_inv_cov * differences, 2)
log_coefficients = tf.expand_dims(
ln2piD + tf.log(tf.matrix_determinant(covariances)), 1)
log_components = -0.5 * (log_coefficients + sum_sq_dist_times_inv_cov)
log_weighted = log_components + tf.expand_dims(tf.log(weights),
1) # Numerator
# print("Log weighted", log_weighted.shape)
log_shift = tf.expand_dims(tf.reduce_max(log_weighted, 0), 0)
# print('Log shift shape', log_shift.shape)
exp_log_shifted = tf.exp(log_weighted - log_shift)
# print('Exp log shift shape', exp_log_shifted.shape)
exp_log_shifted_sum = tf.reduce_sum(exp_log_shifted, 0)
# print('Exp log shifted sum', exp_log_shifted_sum.shape)
def get_update_UVW_ops_for_2sgd_sals(self, rho):
'''
See 2SGD/SALS algorithms in Expected Tensor Decomp paper
'''
def gamma(A, B):
ATA = tf.matmul(A, A, transpose_a=True) # A^T * A
BTB = tf.matmul(B, B, transpose_a=True) # B^T * B
return tf.multiply(ATA, BTB) # hadamard product of A^T*A and B^T*B
X = self.X_t
t = self.global_step + 1
alpha = .25 # smaller => decays slower (more quickly get updates from the gradients)
#batch_size = 1. / 500. * tf.sqrt(tf.cast(X.shape[0], tf.float32))
batch_size = 1.
eta_t = batch_size / (1. + t**alpha)
X_VW = tf.Variable(tf.constant(0.0, shape=[self.shape[0], self.rank]))
XU_W = tf.Variable(tf.constant(0.0, shape=[self.shape[1], self.rank]))
XUV_ = tf.Variable(tf.constant(0.0, shape=[self.shape[2], self.rank]))
def assign_zeroes():
a1 = tf.assign(X_VW,
tf.constant(0.0, shape=[self.shape[0], self.rank]))
a2 = tf.assign(XU_W,
tf.constant(0.0, shape=[self.shape[1], self.rank]))
a3 = tf.assign(XUV_,
tf.constant(0.0, shape=[self.shape[2], self.rank]))
return tf.group(a1, a2, a3)
def body(
ix,
X_VW2,
XU_W2,
XUV_2,
):
ijk = tf.gather(X.indices, ix)
val = tf.gather(X.values, ix)
Ui = tf.gather(self.U, ijk[0]) # R-dimensional
Vj = tf.gather(self.V, ijk[1])
Wk = tf.gather(self.W, ijk[2])
Xijk_Vj_Wk = val * tf.multiply(Vj, Wk)
Xijk_Ui_Wk = val * tf.multiply(Ui, Wk)
Xijk_Ui_Vj = val * tf.multiply(Ui, Vj)
r1 = tf.scatter_add(
X_VW, ijk[0], Xijk_Vj_Wk
) # add Xijk(Vj*Wk) to X_VW(i,:) (as vectors in \mathbb{R}^R)
r2 = tf.scatter_add(XU_W, ijk[1], Xijk_Ui_Wk)
r3 = tf.scatter_add(XUV_, ijk[2], Xijk_Ui_Vj)
new_ix = tf.add(ix, tf.constant(1))
return new_ix, r1, r2, r3
N = tf.shape(X.values)[0]
def cond(ix, *args):
ass = tf.cond(tf.equal(ix, tf.constant(0)),
lambda: assign_zeroes(), lambda: tf.no_op())
with tf.control_dependencies([ass]):
return tf.less(ix, N)
printer = tf.Print(XUV_, [X_VW, XU_W, XUV_])
ix = tf.Variable(tf.constant(
0, dtype=tf.int32)) # which index/value we're looking at
_, X_VW, XU_W, XUV_ = tf.while_loop(cond, body, [ix, X_VW, XU_W, XUV_])
U = self.U
V = self.V
W = self.W
self.D1 = X_VW
self.D2 = XU_W
self.D3 = XUV_
gamma_rho = gamma(V, W) + rho * tf.eye(self.rank)
self.gr1 = gamma_rho
inv_gamma_rho = tf.matrix_inverse(gamma_rho)
self.gr1inv = gamma_rho
self.grad_value_U = tf.matmul(X_VW, inv_gamma_rho)
gamma_rho = gamma(U, W) + rho * tf.eye(self.rank)
self.gr2 = gamma_rho
inv_gamma_rho = tf.matrix_inverse(gamma_rho)
self.grad_value_V = tf.matmul(XU_W, inv_gamma_rho)
gamma_rho = gamma(U, V) + rho * tf.eye(self.rank)
self.gr3 = gamma_rho
inv_gamma_rho = tf.matrix_inverse(gamma_rho)
self.grad_value_W = tf.matmul(XUV_, inv_gamma_rho)
update_U_op = tf.assign(U, (1 - eta_t) * U + eta_t * self.grad_value_U)
update_V_op = tf.assign(V, (1 - eta_t) * V + eta_t * self.grad_value_V)
update_W_op = tf.assign(W, (1 - eta_t) * W + eta_t * self.grad_value_W)
return [update_U_op, update_V_op, update_W_op]
def __init__(self, seq_length, emb_dim, hidden_dim, embeddings, emb_train):
## Define hyperparameters
## note: embedding_dim and hidden_dim are both 300, used interchangeably
self.embedding_dim = emb_dim
self.dim = hidden_dim
self.sequence_length = seq_length
## Define the placeholders
self.premise_x = tf.placeholder(tf.int32, [None, self.sequence_length])
self.hypothesis_x = tf.placeholder(tf.int32,
[None, self.sequence_length])
self.y = tf.placeholder(tf.int32, [None])
self.keep_rate_ph = tf.placeholder(tf.float32, [])
## Define parameters
self.E = tf.Variable(embeddings, trainable=emb_train)
self.W_mlp = tf.Variable(
tf.random_normal([self.dim * 8, self.dim], stddev=0.1))
self.b_mlp = tf.Variable(tf.random_normal([self.dim], stddev=0.1))
self.W_cl = tf.Variable(tf.random_normal([self.dim, 3], stddev=0.1))
self.b_cl = tf.Variable(tf.random_normal([3], stddev=0.1))
## Function for embedding lookup and dropout at embedding layer
def emb_drop(x):
emb = tf.nn.embedding_lookup(self.E, x)
emb_drop = tf.nn.dropout(emb, self.keep_rate_ph)
return emb_drop
# Get lengths of unpadded sentences
prem_seq_lengths, mask_prem = blocks.length(self.premise_x)
hyp_seq_lengths, mask_hyp = blocks.length(self.hypothesis_x)
### First biLSTM layer ###
premise_in = emb_drop(self.premise_x)
hypothesis_in = emb_drop(self.hypothesis_x)
premise_outs, c1 = blocks.biLSTM(premise_in,
dim=self.dim,
seq_len=prem_seq_lengths,
name='premise')
hypothesis_outs, c2 = blocks.biLSTM(hypothesis_in,
dim=self.dim,
seq_len=hyp_seq_lengths,
name='hypothesis')
premise_bi = tf.concat(premise_outs, axis=2)
hypothesis_bi = tf.concat(hypothesis_outs, axis=2)
premise_list = tf.unstack(premise_bi, axis=1)
hypothesis_list = tf.unstack(hypothesis_bi, axis=1)
### Attention ###
scores_all = []
premise_attn = []
alphas = []
for i in range(self.sequence_length):
scores_i_list = []
for j in range(self.sequence_length):
score_ij = tf.reduce_sum(tf.multiply(premise_list[i],
hypothesis_list[j]),
1,
keep_dims=True)
scores_i_list.append(score_ij)
scores_i = tf.stack(scores_i_list, axis=1)
alpha_i = blocks.masked_softmax(scores_i, mask_hyp)
a_tilde_i = tf.reduce_sum(tf.multiply(alpha_i, hypothesis_bi), 1)
premise_attn.append(a_tilde_i)
scores_all.append(scores_i)
alphas.append(alpha_i)
scores_stack = tf.stack(scores_all, axis=2)
scores_list = tf.unstack(scores_stack, axis=1)
hypothesis_attn = []
betas = []
for j in range(self.sequence_length):
scores_j = scores_list[j]
beta_j = blocks.masked_softmax(scores_j, mask_prem)
b_tilde_j = tf.reduce_sum(tf.multiply(beta_j, premise_bi), 1)
hypothesis_attn.append(b_tilde_j)
betas.append(beta_j)
# Make attention-weighted sentence representations into one tensor,
premise_attns = tf.stack(premise_attn, axis=1)
hypothesis_attns = tf.stack(hypothesis_attn, axis=1)
# For making attention plots,
self.alpha_s = tf.stack(alphas, axis=2)
self.beta_s = tf.stack(betas, axis=2)
### Subcomponent Inference ###
prem_diff = tf.subtract(premise_bi, premise_attns)
prem_mul = tf.multiply(premise_bi, premise_attns)
hyp_diff = tf.subtract(hypothesis_bi, hypothesis_attns)
hyp_mul = tf.multiply(hypothesis_bi, hypothesis_attns)
m_a = tf.concat([premise_bi, premise_attns, prem_diff, prem_mul], 2)
m_b = tf.concat([hypothesis_bi, hypothesis_attns, hyp_diff, hyp_mul],
2)
### Inference Composition ###
v1_outs, c3 = blocks.biLSTM(m_a,
dim=self.dim,
seq_len=prem_seq_lengths,
name='v1')
v2_outs, c4 = blocks.biLSTM(m_b,
dim=self.dim,
seq_len=hyp_seq_lengths,
name='v2')
v1_bi = tf.concat(v1_outs, axis=2)
v2_bi = tf.concat(v2_outs, axis=2)
### Pooling Layer ###
v_1_sum = tf.reduce_sum(v1_bi, 1)
v_1_ave = tf.div(
v_1_sum, tf.expand_dims(tf.cast(prem_seq_lengths, tf.float32), -1))
v_2_sum = tf.reduce_sum(v2_bi, 1)
v_2_ave = tf.div(
v_2_sum, tf.expand_dims(tf.cast(hyp_seq_lengths, tf.float32), -1))
v_1_max = tf.reduce_max(v1_bi, 1)
v_2_max = tf.reduce_max(v2_bi, 1)
v = tf.concat([v_1_ave, v_2_ave, v_1_max, v_2_max], 1)
# MLP layer
h_mlp = tf.nn.tanh(tf.matmul(v, self.W_mlp) + self.b_mlp)
############### MY CODE STARTS #####
# Define layer size
self.bow_layer_size = 300
# LSTM layer (final layer of the original esim model)
h_fc1 = h_mlp
# Bag-of-word input (concat word embeddings)
self.sentence_dim = self.dim * self.sequence_length
self.input_dim = self.sentence_dim * 2
bow_pre = premise_in
bow_hyp = hypothesis_in
bag_of_word_in = tf.concat([bow_pre, bow_hyp], 1)
bag_of_word_in = tf.reshape(bag_of_word_in, [-1, self.input_dim])
# Bag-of-word input layer params
W_fc2 = tf.Variable(
tf.random_normal([self.input_dim, self.bow_layer_size],
stddev=0.1))
b_fc2 = tf.Variable(tf.zeros([self.bow_layer_size]))
h_fc2 = tf.nn.sigmoid(tf.matmul(bag_of_word_in, W_fc2) + b_fc2)
# Bag-of-word output layer params
self.W_cl = tf.Variable(
tf.random_normal([self.dim + self.bow_layer_size, 3], stddev=0.1))
self.b_cl = tf.Variable(tf.random_normal([3], stddev=0.1))
# Compute prediction using [h_fc1, 0(pad)]
pad = tf.zeros_like(h_fc2, tf.float32)
yconv_contact_pred = tf.nn.dropout(tf.concat([h_fc1, pad], 1),
self.keep_rate_ph)
y_conv_pred = tf.matmul(yconv_contact_pred, self.W_cl) + self.b_cl
self.logits = y_conv_pred # Prediction
# Compute loss using [h_fc1, h_fc2] and [0(pad2), h_fc2]
pad2 = tf.zeros_like(h_fc1, tf.float32)
yconv_contact_H = tf.nn.dropout(tf.concat([pad2, h_fc2], 1),
self.keep_rate_ph)
y_conv_H = tf.matmul(yconv_contact_H, self.W_cl) + self.b_cl # get Fg
yconv_contact_loss = tf.nn.dropout(tf.concat([h_fc1, h_fc2], 1),
self.keep_rate_ph)
y_conv_loss = tf.matmul(yconv_contact_loss,
self.W_cl) + self.b_cl # get Fb
##########################################################
# Warning: Here it may enconter invertible matrix issue #
##########################################################
y_conv_loss = y_conv_loss - tf.matmul(
tf.matmul(tf.matmul(
y_conv_H,
tf.matrix_inverse(
tf.matmul(y_conv_H, y_conv_H, transpose_a=True))),
y_conv_H,
transpose_b=True), y_conv_loss) # get loss
cost_logits = y_conv_loss
self.total_cost = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=self.y, logits=y_conv_loss)) # Cost
import tensorflow as tf
import numpy as np
A = tf.placeholder(dtype=tf.float64, shape=[2, 2])
b = tf.placeholder(dtype=tf.float64, shape=[2])
A_pow = tf.sin(A)
A_relu = tf.nn.relu(A)
A_inverse = tf.matrix_inverse(A)
A_T = tf.transpose(A)
b_diag = tf.diag(b)
I = tf.eye(6)
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
print(sess.run(A_pow, feed_dict={A: [[1, 2], [-1, 1]]}))
print(sess.run(A_relu, feed_dict={A: [[1, 2], [-1, 1]], b: [1, 1]}))
print(sess.run(A_inverse, feed_dict={A: [[1, 2], [-1, 1]], b: [1, 1]}))
print(sess.run([b_diag, I], feed_dict={A: [[1, 2], [-1, 1]], b: [1, 1]}))
# Matrix rank
print("rank(A) = ")
print(np.linalg.matrix_rank(A.eval()))
print("rank(C) = ")
print(np.linalg.matrix_rank(C.eval()))
# Transpose
print("tran(A) = ")
print(tf.matrix_transpose(A).eval())
print("tran(B) = ")
print(tf.matrix_transpose(B).eval())
# Inverse
print("inv(A) = ")
print(tf.matrix_inverse(A).eval())
# Inverse of diagonal matrix has diag elements of the reciprocal of diag elements B
print("inv(B) = ")
print(tf.matrix_inverse(B).eval())
print("inv(C) = ") # since C has rank 1, this will cause error
try:
print(tf.matrix_inverse(C).eval())
except:
print("C is not invertible")
# Product of a matrix and its inverse is an identity (non-singular)
print("A*inv(A) = Eye(2)")
print(tf.matmul(A, tf.matrix_inverse(A)).eval())
# Element-wise multiplication
print("elem(A)*elem(B) = ")
def QuantizedActiv(x, nbit=2):
"""
Quantize activation.
Args:
x (tf.Tensor): a 4D tensor.
nbit (int): number of bits of quantized activation. Defaults to 2.
Returns:
tf.Tensor with attribute `variables`.
Variable Names:
* ``basis``: basis of quantized activation.
Note:
About multi-GPU training: moving averages across GPUs are not aggregated.
Batch statistics are computed by main training tower. This is consistent with most frameworks.
"""
init_basis = [(NORM_PPF_0_75 * 2 / (2**nbit - 1)) * (2.**i)
for i in range(nbit)]
init_basis = tf.constant_initializer(init_basis)
bit_dims = [nbit, 1]
num_levels = 2**nbit
# initialize level multiplier
init_level_multiplier = []
for i in range(0, num_levels):
level_multiplier_i = [0. for j in range(nbit)]
level_number = i
for j in range(nbit):
level_multiplier_i[j] = float(level_number % 2)
level_number = level_number // 2
init_level_multiplier.append(level_multiplier_i)
# initialize threshold multiplier
init_thrs_multiplier = []
for i in range(1, num_levels):
thrs_multiplier_i = [0. for j in range(num_levels)]
thrs_multiplier_i[i - 1] = 0.5
thrs_multiplier_i[i] = 0.5
init_thrs_multiplier.append(thrs_multiplier_i)
with tf.variable_scope('ActivationQuantization'):
basis = tf.get_variable('basis',
bit_dims,
tf.float32,
initializer=init_basis,
trainable=False)
ctx = get_current_tower_context() # current tower context
# calculate levels and sort
level_codes = tf.constant(init_level_multiplier)
levels = tf.matmul(level_codes, basis)
levels, sort_id = tf.nn.top_k(tf.transpose(levels, [1, 0]), num_levels)
levels = tf.reverse(levels, [-1])
sort_id = tf.reverse(sort_id, [-1])
levels = tf.transpose(levels, [1, 0])
sort_id = tf.transpose(sort_id, [1, 0])
# calculate threshold
thrs_multiplier = tf.constant(init_thrs_multiplier)
thrs = tf.matmul(thrs_multiplier, levels)
# calculate output y and its binary code
y = tf.zeros_like(x) # output
reshape_x = tf.reshape(x, [-1])
zero_dims = tf.stack([tf.shape(reshape_x)[0], nbit])
bits_y = tf.fill(zero_dims, 0.)
zero_y = tf.zeros_like(x)
zero_bits_y = tf.fill(zero_dims, 0.)
for i in range(num_levels - 1):
g = tf.greater(x, thrs[i])
y = tf.where(g, zero_y + levels[i + 1], y)
bits_y = tf.where(tf.reshape(g, [-1]),
zero_bits_y + level_codes[sort_id[i + 1][0]],
bits_y)
# training
if ctx.is_main_training_tower:
BT = tf.matrix_transpose(bits_y)
# calculate BTxB
BTxB = []
for i in range(nbit):
for j in range(nbit):
BTxBij = tf.multiply(BT[i], BT[j])
BTxBij = tf.reduce_sum(BTxBij)
BTxB.append(BTxBij)
BTxB = tf.reshape(tf.stack(values=BTxB), [nbit, nbit])
BTxB_inv = tf.matrix_inverse(BTxB)
# calculate BTxX
BTxX = []
for i in range(nbit):
BTxXi0 = tf.multiply(BT[i], reshape_x)
BTxXi0 = tf.reduce_sum(BTxXi0)
BTxX.append(BTxXi0)
BTxX = tf.reshape(tf.stack(values=BTxX), [nbit, 1])
new_basis = tf.matmul(BTxB_inv, BTxX) # calculate new basis
# create moving averages op
updata_moving_basis = moving_averages.assign_moving_average(
basis, new_basis, MOVING_AVERAGES_FACTOR)
add_model_variable(basis)
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, updata_moving_basis)
for i in range(nbit):
tf.summary.scalar('basis%d' % i, new_basis[i][0])
x_clip = tf.minimum(x, levels[num_levels - 1]) # gradient clip
y = x_clip + tf.stop_gradient(-x_clip) + tf.stop_gradient(
y) # gradient: y=clip(x)
y.variables = VariableHolder(basis=basis)
return y
import numpy as np
import tensorflow as tf
from sklearn.datasets import fetch_california_housing
housing = fetch_california_housing()
m , n = housing.data.shape
housing_data_plus_bias = np.c_[np.ones((m , 1)), housing.data]
X = tf.constant(housing_data_plus_bias, dtype=tf.float32, name='X')
y = tf.constant(housing.target.reshape(-1,1), dtype=tf.float32, name='y')
XT = tf.transpose(X)
theta = tf.matmul(tf.matmul(tf.matrix_inverse(tf.matmul(XT, X)), XT), y)
with tf.Session() as sess:
theta_value = theta.eval()
def build_model(self):
# Define prediction rnn
rnn_outputs_hypo, final_state_hypo = lstm_layer(
self.e_hypo, self.lstm_size, self.batch_size, self.seq_len_hypo,
"hypo")
rnn_outputs_prem, final_state_prem = lstm_layer(
self.e_prem, self.lstm_size, self.batch_size, self.seq_len_prem,
"prem")
last_output_hypo, alphas_hypo = attention_layer(self.attention_size,
rnn_outputs_hypo,
"encoder_hypo",
sparse=self.sparse)
last_output_prem, alphas_prem = attention_layer(self.attention_size,
rnn_outputs_prem,
"encoder_prem",
sparse=self.sparse)
self.alphas_hypo = alphas_hypo
self.alphas_prem = alphas_prem
# last_output = tf.nn.dropout(last_output, self.keep_probs)
# Define key-word model rnn
kwm_rnn_outputs_hypo, kwm_final_state_hypo = lstm_layer(
self.e_hypo,
self.lstm_size,
self.batch_size,
self.seq_len_hypo,
scope="kwm_hypo")
kwm_rnn_outputs_prem, kwm_final_state_prem = lstm_layer(
self.e_prem,
self.lstm_size,
self.batch_size,
self.seq_len_prem,
scope="kwm_prem")
kwm_last_output_hypo, kwm_alphas_hypo = attention_layer(
self.attention_size,
kwm_rnn_outputs_hypo,
"kwm_encoder_hypo",
sparse=self.sparse)
kwm_last_output_prem, kwm_alphas_prem = attention_layer(
self.attention_size,
kwm_rnn_outputs_prem,
"kwm_encoder_prem",
sparse=self.sparse)
last_output = tf.concat([last_output_hypo, last_output_prem], axis=1)
kwm_last_output = tf.concat(
[kwm_last_output_hypo, kwm_last_output_prem], axis=1)
############################
# Hex #########################
h_fc1 = last_output
h_fc2 = kwm_last_output
# Hex layer definition
"""
self.W_cl_1 = tf.Variable(tf.random_normal([self.dim, 3], stddev=0.1))
self.W_cl_2 = tf.Variable(tf.random_normal([1200, 3]), trainable=True)
self.b_cl = tf.Variable(tf.random_normal((3,)), trainable=True)
self.W_cl = tf.concat([self.W_cl_1, self.W_cl_2], 0)
"""
# Compute prediction using [h_fc1, 0(pad)]
pad = tf.zeros_like(h_fc2, tf.float32)
# print(pad.shape) -> (?, 600)
yconv_contact_pred = tf.nn.dropout(tf.concat([h_fc1, pad], 1),
self.keep_probs)
# y_conv_pred = tf.matmul(yconv_contact_pred, self.W_cl) + self.b_cl
y_conv_pred = dense_layer(yconv_contact_pred, 3, name="conv_pred")
self.logits = y_conv_pred # Prediction
# Compute loss using [h_fc1, h_fc2] and [0(pad2), h_fc2]
pad2 = tf.zeros_like(h_fc1, tf.float32)
yconv_contact_H = tf.concat([pad2, h_fc2], 1)
# Get Fg
# y_conv_H = tf.matmul(yconv_contact_H, self.W_cl) + self.b_cl # get Fg
y_conv_H = dense_layer(yconv_contact_H, 3, name="conv_H")
yconv_contact_loss = tf.nn.dropout(tf.concat([h_fc1, h_fc2], 1),
self.keep_probs)
# Get Fb
# y_conv_loss = tf.matmul(yconv_contact_loss, self.W_cl) + self.b_cl # get Fb
y_conv_loss = dense_layer(yconv_contact_loss, 3, name="conv_loss")
temp = tf.matmul(y_conv_H, y_conv_H, transpose_a=True)
self.temp = temp
y_conv_loss = y_conv_loss - tf.matmul(
tf.matmul(tf.matmul(y_conv_H, tf.matrix_inverse(temp)),
y_conv_H,
transpose_b=True), y_conv_loss) # get loss
self.logits = y_conv_loss
self.y = tf.nn.softmax(self.logits)
self.cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=tf.one_hot(
self.y_holder, depth=3),
logits=self.logits))
# Regularize kwm attention
reg1 = get_reg(kwm_alphas_hypo, lam=self.lam, type=self.reg)
reg2 = get_reg(kwm_alphas_prem, lam=self.lam, type=self.reg)
self.cost += reg1 + reg2
self.optimizer = tf.train.GradientDescentOptimizer(self.learning_rate)
self.train_op = self.optimizer.minimize(self.cost)
self.accuracy = tf.reduce_mean(
tf.cast(tf.equal(self.y_holder, tf.argmax(self.y, 1)), tf.float32))
def forward(self, inputs, grid, is_training=True, reuse=False):
def preprocessing(inputs):
dims = inputs.get_shape()
if len(dims) == 3:
inputs = tf.expand_dims(inputs, dim=0)
mean_BGR = tf.reshape(self.mean_BGR, [1, 1, 1, 3])
inputs = inputs[:, :, :, ::-1] + mean_BGR
return inputs
## -----------------------depth and normal FCN--------------------------
inputs = preprocessing(inputs)
with slim.arg_scope(
[slim.conv2d, slim.conv2d_transpose],
activation_fn=tf.nn.relu,
stride=1,
padding='SAME',
weights_initializer=weight_from_caffe(self.pretrain_weight),
biases_initializer=bias_from_caffe(self.pretrain_weight)):
with tf.variable_scope('fcn', reuse=reuse):
##---------------------vgg depth------------------------------------
conv1 = slim.repeat(inputs,
2,
slim.conv2d,
64, [3, 3],
scope='conv1')
pool1 = slim.max_pool2d(conv1, [3, 3],
stride=2,
padding='SAME',
scope='pool1')
conv2 = slim.repeat(pool1,
2,
slim.conv2d,
128, [3, 3],
scope='conv2')
pool2 = slim.max_pool2d(conv2, [3, 3],
stride=2,
padding='SAME',
scope='pool2')
conv3 = slim.repeat(pool2,
3,
slim.conv2d,
256, [3, 3],
scope='conv3')
pool3 = slim.max_pool2d(conv3, [3, 3],
stride=2,
padding='SAME',
scope='pool3')
conv4 = slim.repeat(pool3,
3,
slim.conv2d,
512, [3, 3],
scope='conv4')
pool4 = slim.max_pool2d(conv4, [3, 3],
stride=1,
padding='SAME',
scope='pool4')
conv5 = slim.repeat(pool4,
3,
slim.conv2d,
512, [3, 3],
rate=2,
scope='conv5')
pool5 = slim.max_pool2d(conv5, [3, 3],
stride=1,
padding='SAME',
scope='pool5')
pool5a = slim.avg_pool2d(pool5, [3, 3],
stride=1,
padding='SAME',
scope='pool5a')
fc6 = slim.conv2d(pool5a,
1024, [3, 3],
stride=1,
rate=12,
scope='fc6')
fc6 = slim.dropout(fc6,
0.5,
is_training=is_training,
scope='drop6')
fc7 = slim.conv2d(fc6, 1024, [1, 1], scope='fc7')
fc7 = slim.dropout(fc7,
0.5,
is_training=is_training,
scope='drop7')
pool6_1x1 = slim.avg_pool2d(fc7, [61, 81],
stride=[61, 81],
padding='SAME',
scope='pool6_1x1')
pool6_1x1_norm = slim.unit_norm(pool6_1x1,
dim=3,
scope='pool6_1x1_norm_new')
pool6_1x1_norm_scale = pool6_1x1_norm * 10
pool6_1x1_norm_upsample = tf.tile(
pool6_1x1_norm_scale, [1, 61, 81, 1],
name='pool6_1x1_norm_upsample')
out = tf.concat([fc7, pool6_1x1_norm_upsample],
axis=-1,
name='out')
out_reduce = slim.conv2d(out,
256, [1, 1],
activation_fn=tf.nn.relu,
stride=1,
scope='out_reduce',
padding='SAME',
weights_initializer=weight_from_caffe(
self.pretrain_weight),
biases_initializer=bias_from_caffe(
self.pretrain_weight))
out_conv = slim.conv2d(out_reduce,
256, [3, 3],
activation_fn=tf.nn.relu,
stride=1,
scope='out_conv',
padding='SAME',
weights_initializer=weight_from_caffe(
self.pretrain_weight),
biases_initializer=bias_from_caffe(
self.pretrain_weight))
out_conv_increase = slim.conv2d(
out_conv,
1024, [1, 1],
activation_fn=tf.nn.relu,
stride=1,
scope='out_conv_increase',
padding='SAME',
weights_initializer=weight_from_caffe(
self.pretrain_weight),
biases_initializer=bias_from_caffe(self.pretrain_weight))
fc8_nyu_depth = slim.conv2d(out_conv_increase,
1, [1, 1],
activation_fn=None,
scope='fc8_nyu_depth')
fc8_upsample = tf.image.resize_images(
fc8_nyu_depth, [self.crop_size_h, self.crop_size_w],
method=0,
align_corners=True)
# ---------------------------------------vgg depth end ---------------------------------------
## ----------------- vgg norm---------------------------------------------------------------
conv1_norm = slim.repeat(inputs,
2,
slim.conv2d,
64, [3, 3],
scope='conv1_norm')
pool1_norm = slim.max_pool2d(conv1_norm, [3, 3],
stride=2,
padding='SAME',
scope='pool1_norm')
conv2_norm = slim.repeat(pool1_norm,
2,
slim.conv2d,
128, [3, 3],
scope='conv2_norm')
pool2_norm = slim.max_pool2d(conv2_norm, [3, 3],
stride=2,
padding='SAME',
scope='pool2_norm')
conv3_norm = slim.repeat(pool2_norm,
3,
slim.conv2d,
256, [3, 3],
scope='conv3_norm')
pool3_norm = slim.max_pool2d(conv3_norm, [3, 3],
stride=2,
padding='SAME',
scope='pool3_norm')
conv4_norm = slim.repeat(pool3_norm,
3,
slim.conv2d,
512, [3, 3],
scope='conv4_norm')
pool4_norm = slim.max_pool2d(conv4_norm, [3, 3],
stride=1,
padding='SAME',
scope='pool4_norm')
conv5_norm = slim.repeat(pool4_norm,
3,
slim.conv2d,
512, [3, 3],
rate=2,
scope='conv5_norm')
pool5_norm = slim.max_pool2d(conv5_norm, [3, 3],
stride=1,
padding='SAME',
scope='pool5_norm')
pool5a_norm = slim.avg_pool2d(pool5_norm, [3, 3],
stride=1,
padding='SAME',
scope='pool5a_norm')
fc6_norm = slim.conv2d(pool5a_norm,
1024, [3, 3],
stride=1,
rate=12,
scope='fc6_norm')
fc6_norm = slim.dropout(fc6_norm,
0.5,
is_training=is_training,
scope='drop6_norm')
fc7_norm = slim.conv2d(fc6_norm,
1024, [1, 1],
scope='fc7_norm')
fc7_norm = slim.dropout(fc7_norm,
0.5,
is_training=is_training,
scope='drop7_norm')
pool6_1x1_norm_new = slim.avg_pool2d(
fc7_norm, [61, 81],
stride=[61, 81],
padding='SAME',
scope='pool6_1x1_norm_new')
pool6_1x1_norm_norm = slim.unit_norm(
pool6_1x1_norm_new, dim=3, scope='pool6_1x1_norm_new')
pool6_1x1_norm_scale_norm = pool6_1x1_norm_norm * 10
pool6_1x1_norm_upsample_norm = tf.tile(
pool6_1x1_norm_scale_norm, [1, 61, 81, 1],
name='pool6_1x1_norm_upsample')
out_norm = tf.concat([fc7_norm, pool6_1x1_norm_upsample_norm],
axis=-1,
name='out_norm')
fc8_nyu_norm_norm = slim.conv2d(out_norm,
3, [1, 1],
activation_fn=None,
scope='fc8_nyu_norm_norm')
fc8_upsample_norm = tf.image.resize_images(
fc8_nyu_norm_norm, [self.crop_size_h, self.crop_size_w],
method=0,
align_corners=True)
fc8_upsample_norm = slim.unit_norm(fc8_upsample_norm, dim=3)
# -------------------------------------vgg norm end---------------------------------------------
# ------------- depth to normal + norm refinement---------------------------------------------------
with tf.variable_scope('noise', reuse=reuse):
fc8_upsample_norm = tf.squeeze(fc8_upsample_norm)
fc8_upsample_norm = tf.reshape(
fc8_upsample_norm,
[self.batch_size, self.crop_size_h, self.crop_size_w, 3])
norm_matrix = tf.extract_image_patches(
images=fc8_upsample_norm,
ksizes=[1, self.k, self.k, 1],
strides=[1, 1, 1, 1],
rates=[1, self.rate, self.rate, 1],
padding='SAME')
matrix_c = tf.reshape(norm_matrix, [
self.batch_size, self.crop_size_h, self.crop_size_w,
self.k * self.k, 3
])
fc8_upsample_norm = tf.expand_dims(fc8_upsample_norm, axis=4)
angle = tf.matmul(matrix_c, fc8_upsample_norm)
valid_condition = tf.greater(angle, self.thresh)
valid_condition_all = tf.tile(valid_condition, [1, 1, 1, 1, 3])
exp_depth = tf.exp(fc8_upsample * 0.69314718056)
depth_repeat = tf.tile(exp_depth, [1, 1, 1, 3])
points = tf.multiply(grid, depth_repeat)
point_matrix = tf.extract_image_patches(
images=points,
ksizes=[1, self.k, self.k, 1],
strides=[1, 1, 1, 1],
rates=[1, self.rate, self.rate, 1],
padding='SAME')
matrix_a = tf.reshape(point_matrix, [
self.batch_size, self.crop_size_h, self.crop_size_w,
self.k * self.k, 3
])
matrix_a_zero = tf.zeros_like(matrix_a, dtype=tf.float32)
matrix_a_valid = tf.where(valid_condition_all, matrix_a,
matrix_a_zero)
matrix_a_trans = tf.matrix_transpose(matrix_a_valid,
name='matrix_transpose')
matrix_b = tf.ones(shape=[
self.batch_size, self.crop_size_h, self.crop_size_w,
self.k * self.k, 1
])
point_multi = tf.matmul(matrix_a_trans,
matrix_a_valid,
name='matrix_multiplication')
with tf.device('cpu:0'):
matrix_deter = tf.matrix_determinant(point_multi)
inverse_condition = tf.greater(matrix_deter, 1e-5)
inverse_condition = tf.expand_dims(inverse_condition, axis=3)
inverse_condition = tf.expand_dims(inverse_condition, axis=4)
inverse_condition_all = tf.tile(inverse_condition,
[1, 1, 1, 3, 3])
diag_constant = tf.ones([3], dtype=tf.float32)
diag_element = tf.diag(diag_constant)
diag_element = tf.expand_dims(diag_element, axis=0)
diag_element = tf.expand_dims(diag_element, axis=0)
diag_element = tf.expand_dims(diag_element, axis=0)
diag_matrix = tf.tile(diag_element, [
self.batch_size, self.crop_size_h, self.crop_size_w, 1, 1
])
inversible_matrix = tf.where(inverse_condition_all,
point_multi, diag_matrix)
with tf.device('cpu:0'):
inv_matrix = tf.matrix_inverse(inversible_matrix)
generated_norm = tf.matmul(
tf.matmul(inv_matrix, matrix_a_trans), matrix_b)
norm_normalize = slim.unit_norm((generated_norm), dim=3)
norm_normalize = tf.reshape(
norm_normalize,
[self.batch_size, self.crop_size_h, self.crop_size_w, 3])
norm_scale = norm_normalize * 10.0
conv1_noise = slim.repeat(norm_scale,
2,
slim.conv2d,
64, [3, 3],
scope='conv1_noise')
pool1_noise = slim.max_pool2d(conv1_noise, [3, 3],
stride=2,
padding='SAME',
scope='pool1_noise') #
conv2_noise = slim.repeat(pool1_noise,
2,
slim.conv2d,
128, [3, 3],
scope='conv2_noise')
conv3_noise = slim.repeat(conv2_noise,
3,
slim.conv2d,
256, [3, 3],
scope='conv3_noise')
fc1_noise = slim.conv2d(conv3_noise,
512, [1, 1],
activation_fn=tf.nn.relu,
stride=1,
scope='fc1_noise',
padding='SAME')
encode_norm_noise = slim.conv2d(fc1_noise,
3, [3, 3],
activation_fn=None,
stride=1,
scope='encode_norm_noise',
padding='SAME')
encode_norm_upsample_noise = tf.image.resize_images(
encode_norm_noise, [self.crop_size_h, self.crop_size_w],
method=0,
align_corners=True)
sum_norm_noise = tf.add(norm_normalize,
encode_norm_upsample_noise)
norm_pred_noise = slim.unit_norm(sum_norm_noise, dim=3)
norm_pred_all = tf.concat([
tf.expand_dims(tf.squeeze(fc8_upsample_norm), axis=0),
norm_pred_noise, inputs * 0.00392156862
],
axis=3)
norm_pred_all = slim.repeat(
norm_pred_all,
3,
slim.conv2d,
128, [3, 3],
rate=2,
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='conv1_norm_noise_new')
norm_pred_all = slim.repeat(
norm_pred_all,
3,
slim.conv2d,
128, [3, 3],
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='conv2_norm_noise_new')
norm_pred_final = slim.conv2d(
norm_pred_all,
3, [3, 3],
activation_fn=None,
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='norm_conv3_noise_new')
norm_pred_final = slim.unit_norm((norm_pred_final), dim=3)
# ------------- normal to depth + depth refinement---------------------------------------------------
with tf.variable_scope('norm_depth', reuse=reuse):
grid_patch = tf.extract_image_patches(
images=grid,
ksizes=[1, self.k, self.k, 1],
strides=[1, 1, 1, 1],
rates=[1, self.rate, self.rate, 1],
padding='SAME')
grid_patch = tf.reshape(grid_patch, [
self.batch_size, self.crop_size_h, self.crop_size_w,
self.k * self.k, 3
])
_, _, depth_data = tf.split(value=matrix_a,
num_or_size_splits=3,
axis=4)
tmp_matrix_zero = tf.zeros_like(angle, dtype=tf.float32)
valid_angle = tf.where(valid_condition, angle, tmp_matrix_zero)
lower_matrix = tf.matmul(matrix_c, tf.expand_dims(grid,
axis=4))
condition = tf.greater(lower_matrix, 1e-5)
tmp_matrix = tf.ones_like(lower_matrix)
lower_matrix = tf.where(condition, lower_matrix, tmp_matrix)
lower = tf.reciprocal(lower_matrix)
valid_angle = tf.where(condition, valid_angle, tmp_matrix_zero)
upper = tf.reduce_sum(tf.multiply(matrix_c, grid_patch), [4])
ratio = tf.multiply(lower, tf.expand_dims(upper, axis=4))
estimate_depth = tf.multiply(ratio, depth_data)
valid_angle = tf.multiply(
valid_angle,
tf.reciprocal(
tf.tile(
tf.reduce_sum(valid_angle, [3, 4], keep_dims=True)
+ 1e-5, [1, 1, 1, 81, 1])))
depth_stage1 = tf.reduce_sum(
tf.multiply(estimate_depth, valid_angle), [3, 4])
depth_stage1 = tf.expand_dims(tf.squeeze(depth_stage1), axis=2)
depth_stage1 = tf.clip_by_value(depth_stage1, 0, 10.0)
exp_depth = tf.expand_dims(tf.squeeze(exp_depth), axis=2)
depth_all = tf.expand_dims(tf.concat([
depth_stage1, exp_depth,
tf.squeeze(inputs) * 0.00392156862
],
axis=2),
axis=0)
depth_pred_all = slim.repeat(
depth_all,
3,
slim.conv2d,
128, [3, 3],
rate=2,
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='conv1_depth_noise_new')
depth_pred_all = slim.repeat(
depth_pred_all,
3,
slim.conv2d,
128, [3, 3],
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='conv2_depth_noise_new')
final_depth = slim.conv2d(
depth_pred_all,
1, [3, 3],
activation_fn=None,
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='depth_conv3_noise_new')
with tf.variable_scope('edge_refinemet', reuse=reuse):
print(inputs.shape)
edges = tf.py_func(myfunc_canny, [inputs], tf.float32)
edges = tf.reshape(edges,
[1, self.crop_size_h, self.crop_size_w, 1])
edge_input_depth = final_depth
edge_input_norm = norm_pred_final
# edge prediction for depth
edge_inputs = tf.concat([edges, inputs * 0.00784], axis=3)
edges_encoder = slim.repeat(
edge_inputs,
3,
slim.conv2d,
32, [3, 3],
rate=2,
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='conv1_edge_refinement')
edges_encoder = slim.repeat(
edges_encoder,
3,
slim.conv2d,
32, [3, 3],
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='conv2_edge_refinement')
edges_predictor = slim.conv2d(
edges_encoder,
8, [3, 3],
activation_fn=None,
weights_initializer=tf.contrib.layers.xavier_initializer(
uniform=False),
biases_initializer=tf.constant_initializer(0.0),
scope='edge_weight')
edges_all = edges_predictor + tf.tile(edges, [1, 1, 1, 8])
edges_all = tf.clip_by_value(edges_all, 0.0, 1.0)
dlr, drl, dud, ddu, nlr, nrl, nud, ndu = tf.split(
edges_all, num_or_size_splits=8, axis=3)
# 4 iteration depth
final_depth = propagate(edge_input_depth, dlr, drl, dud, ddu,
1)
final_depth = propagate(final_depth, dlr, drl, dud, ddu, 1)
final_depth = propagate(final_depth, dlr, drl, dud, ddu, 1)
final_depth = propagate(final_depth, dlr, drl, dud, ddu, 1)
# 4 iteration norm
norm_pred_final = propagate(edge_input_norm, nlr, nrl, nud,
ndu, 3)
norm_pred_final = slim.unit_norm((norm_pred_final), dim=3)
norm_pred_final = propagate(norm_pred_final, nlr, nrl, nud,
ndu, 3)
norm_pred_final = slim.unit_norm((norm_pred_final), dim=3)
norm_pred_final = propagate(norm_pred_final, nlr, nrl, nud,
ndu, 3)
norm_pred_final = slim.unit_norm((norm_pred_final), dim=3)
norm_pred_final = propagate(norm_pred_final, nlr, nrl, nud,
ndu, 3)
norm_pred_final = slim.unit_norm((norm_pred_final), dim=3)
return final_depth, fc8_upsample_norm, norm_pred_final, fc8_upsample
def invertible_1x1_conv(name, x, reverse=False):
"""1X1 convolution on x.
The 1X1 convolution is parametrized as P*L*(U + sign(s)*exp(log(s))) where
1. P is a permutation matrix.
2. L is a lower triangular matrix with diagonal entries unity.
3. U is a upper triangular matrix where the diagonal entries zero.
4. s is a vector.
sign(s) and P are fixed and the remaining are optimized. P, L, U and s are
initialized by the PLU decomposition of a random rotation matrix.
Args:
name: scope
x: Input Tensor.
reverse: whether the pass is from z -> x or x -> z.
Returns:
x_conv: x after a 1X1 convolution is applied on x.
objective: sum(log(s))
"""
_, height, width, channels = common_layers.shape_list(x)
w_shape = [channels, channels]
# Random rotation-matrix Q
random_matrix = np.random.rand(channels, channels)
np_w = scipy.linalg.qr(random_matrix)[0].astype("float32")
# Initialize P,L,U and s from the LU decomposition of a random rotation matrix
np_p, np_l, np_u = scipy.linalg.lu(np_w)
np_s = np.diag(np_u)
np_sign_s = np.sign(np_s)
np_log_s = np.log(np.abs(np_s))
np_u = np.triu(np_u, k=1)
with tf.variable_scope(name, reuse=tf.AUTO_REUSE):
p = tf.get_variable("P", initializer=np_p, trainable=False)
l = tf.get_variable("L", initializer=np_l)
sign_s = tf.get_variable(
"sign_S", initializer=np_sign_s, trainable=False)
log_s = tf.get_variable("log_S", initializer=np_log_s)
u = tf.get_variable("U", initializer=np_u)
# W = P * L * (U + sign_s * exp(log_s))
l_mask = np.tril(np.ones([channels, channels], dtype=np.float32), -1)
l = l * l_mask + tf.eye(channels, channels)
u = u * np.transpose(l_mask) + tf.diag(sign_s * tf.exp(log_s))
w = tf.matmul(p, tf.matmul(l, u))
# If height or width cannot be statically determined then they end up as
# tf.int32 tensors, which cannot be directly multiplied with a floating
# point tensor without a cast.
objective = tf.reduce_sum(log_s) * tf.cast(height * width, log_s.dtype)
if not reverse:
w = tf.reshape(w, [1, 1] + w_shape)
x = tf.nn.conv2d(x, w, [1, 1, 1, 1], "SAME", data_format="NHWC")
else:
u_inv = tf.matrix_inverse(u)
l_inv = tf.matrix_inverse(l)
p_inv = tf.matrix_inverse(p)
w_inv = tf.matmul(u_inv, tf.matmul(l_inv, p_inv))
w_inv = tf.reshape(w_inv, [1, 1]+w_shape)
x = tf.nn.conv2d(
x, w_inv, [1, 1, 1, 1], "SAME", data_format="NHWC")
objective *= -1
return x, objective
def handle_empty_marg(params, all_vars):
print_op = tf.Print(tf.constant(1), [], "Inside handling empty marg")
# Set Q_s as [[0]]
with tf.control_dependencies([print_op]):
all_vars = all_vars._replace(Q_s=tf.constant([[0.]], dtype=tf.float64))
# Calculate min b for transition
g_c_ = all_vars.g_c
x_c_ = all_vars.x_c
y_c_ = all_vars.y_c
trans_c = -1 * g_c_
# calc trans b for err and rem
# For err_vec, err_vec_y should have same sign as y_c_
elems = (all_vars.err_vec_y, all_vars.g_err)
trans_err = tf.map_fn(
lambda x: tf.cond(tf.greater(x[0] * y_c_, 0), lambda: -1 * x[1],
lambda: tf.constant(float("Inf"), dtype=tf.float64)),
elems,
dtype=tf.float64)
# For rem_vec, rem_vec_y should have opposite sign as y_c_
elems = (all_vars.rem_vec_y, all_vars.g_rem)
trans_rem = tf.map_fn(
lambda x: tf.cond(tf.less(x[0] * y_c_, 0), lambda: x[1], lambda: tf.
constant(float("Inf"), dtype=tf.float64)),
elems,
dtype=tf.float64)
# Find the min transition_val for b
all_trans = tf.concat([trans_err, trans_rem,
tf.reshape(trans_c, [
1,
])], 0)
abs_min_del_b = tf.reduce_min(all_trans)
# change the sign of min_del_b to the sign of y_c_ as
# this is the dirn of update
min_del_b = tf.cast(y_c_ * abs_min_del_b, dtype=tf.float64)
# update values
b_ = all_vars.b + min_del_b
g_c_ = g_c_ + (min_del_b * y_c_)
# update g_err
g_err_ = all_vars.g_err + (all_vars.err_vec_y * min_del_b)
# update g_rem
g_rem_ = all_vars.g_rem + (all_vars.rem_vec_y * min_del_b)
# print_op = tf.Print(tf.constant(1.), [all_vars.g_c, all_vars.Q_s, min_del_b],
# "Line 351")
# with tf.control_dependencies([print_op]):
all_vars = all_vars._replace(b=b_, g_c=g_c_, g_err=g_err_, g_rem=g_rem_)
# Transition vectors if g is zero and dirn of update is same as candidate
# Check if min is in err sup vectors
min_err_indx = tf.cond(
tf.equal(tf.shape(trans_err)[0], 0),
lambda: tf.constant(1),
lambda: check_min(trans_err, abs_min_del_b),
)
all_vars = tf.cond(
tf.less(min_err_indx,
tf.shape(trans_err)[0]),
lambda: handle_empty_err(params, min_err_indx, all_vars),
lambda: all_vars)
# Check if min is in rem vectors
min_rem_indx = tf.cond(
tf.equal(tf.shape(trans_rem)[0], 0),
lambda: tf.constant(1),
lambda: check_min(trans_rem, abs_min_del_b),
)
all_vars = tf.cond(
tf.less(min_rem_indx,
tf.shape(trans_rem)[0]),
lambda: handle_empty_rem(params, min_rem_indx, all_vars),
lambda: all_vars)
# Check if min is in trans_c
all_vars = tf.cond(
tf.equal(trans_c, abs_min_del_b),
lambda: free_add_to_marg(params["kernel"], all_vars.alpha_c, all_vars.
x_c, all_vars.y_c, all_vars),
lambda: all_vars)
# update R as inverse of Q
R_ = tf.matrix_inverse(all_vars.Q_s)
return all_vars._replace(R=R_)
Tr_Knn, K_nm_2, K_mnnm_2 = KS.build_psi_stats_rbf(X_m_2,tf.square(noise_2), len_sc_2, h_mu, h_S) K_mm_2=h.tf_SE_K(X_m_2,X_m_2,len_sc_2,noise_2) K_nn_2=h.tf_SE_K(h_mu,h_mu,len_sc_2,noise_2) K_mn_2=h.tf.transpose(K_nm_2) ''' opti_mu1= opti_Sig1= opti_mu2= opti_sig2= ''' a_mn=tf.matrix_solve(K_mm_1,K_mn_1) a_nm=tf.transpose(a_mn) sig_inv=tf.matrix_inverse(K_mm_1)+h.Mul(a_mn,a_nm)/tf.square(sigma_1) mu1=h.Mul(tf.matrix_inverse(sig_inv),a_mn,Ytr)/tf.square(sigma_1) F_v_1=GPLVM.Bound1(h_mu,h_S,K_mm_1,K_nm_1,Tr_Knn_1,sigma_1) s.run(tf.initialize_all_variables()) F_v_2=GPLVM.Bound2(Tr_Knn,K_nm_2,K_mnnm_2,sigma_2,K_mm_2,Ytr) mean,var_terms=predict(K_mn_2,sigma_2,K_mm_2,K_nn_2) mean=tf.reshape(mean,[-1]) var_terms=tf.reshape(var_terms,[-1]) global_step = tf.Variable(0, trainable=False) starter_learning_rate = 0.1 lrate=tf.train.exponential_decay(starter_learning_rate, global_step,
def evaluate_task(inputs):
features_train, train_outputs, features_test, test_outputs, representation, mean_features = inputs
q_representation = multihead_attention(features_test, mean_features,
mean_features)
with tf.variable_scope('inference_network'):
r_mu = inference_block(representation, args.d_theta, args.d_theta,
'infer_mean')
r_log_var = inference_block(representation, args.d_theta,
args.d_theta, 'infer_var')
p_mu = inference_block(q_representation, args.d_theta,
args.d_theta, 'p_mean')
p_log_var = inference_block(q_representation, args.d_theta,
args.d_theta, 'p_var')
# Infer classification layer from q
with tf.variable_scope('classifier'):
# compute bases
rs = tf.squeeze(
sample_normal(r_mu, r_log_var, args.d_rn_f, eps_=eps))
# compute the support kernel
x_supp_phi_t = rand_features(rs,
tf.transpose(features_train, [1, 0]),
bias) # (d_rn_f , w*s)
support_kernel = dotp_kernel(tf.transpose(x_supp_phi_t),
x_supp_phi_t) # (w*s , w*s)
# closed-form solution with trainable lmd
alpha = tf.matmul(
tf.matrix_inverse(support_kernel + (lmd_abs + 0.01) *
tf.eye(tf.shape(support_kernel)[0])),
train_outputs)
x_que_phi_t = rand_features(
rs, tf.transpose(features_test, [1, 0]), bias
) # tf.cos(tf.matmul(rs, tf.transpose(features_test, [1, 0])))
# compute the cross kernel
cross_kernel = dotp_kernel(tf.transpose(x_supp_phi_t), x_que_phi_t)
# prediction with calibration params
logits = gamma * tf.matmul(cross_kernel, alpha,
transpose_a=True) + beta
# align loss
target_kernel = dotp_kernel(train_outputs,
tf.transpose(train_outputs))
target_kernel = 0.99 * (target_kernel + 0.01)
kernel_align_loss = cosine_dist(target_kernel, support_kernel)
# kl loss
kl_loss = KL_divergence(r_mu, r_log_var, p_mu, p_log_var)
# cross entry loss
cross_entry_loss = mse(logits, test_outputs)
task_loss = cross_entry_loss + args.zeta * kernel_align_loss + args.tau * kl_loss
return [
task_loss, cross_entry_loss, kernel_align_loss, kl_loss, logits
]
def __init__(self,
sess,
batch_size,
input_len,
hidden_num,
output_len,
task='main'):
'''
Args:
sess : TensorFlow session.
batch_size : The batch size (N)
input_len : The length of input. (L)
hidden_num : The number of hidden node. (K)
output_len : The length of output. (O)
'''
self._sess = sess
self._batch_size = batch_size
self._input_len = input_len
self._hidden_num = hidden_num
self._output_len = output_len
# for train
self._x0 = tf.placeholder(tf.float32,
[self._batch_size, self._input_len])
self._t0 = tf.placeholder(tf.float32,
[self._batch_size, self._output_len])
# for test
self._x1 = tf.placeholder(tf.float32, [None, self._input_len])
self._t1 = tf.placeholder(tf.float32, [None, self._output_len])
self._W = tf.Variable(tf.random_normal(
[self._input_len, self._hidden_num]),
trainable=False,
dtype=tf.float32,
name=task + '_w')
self._b = tf.Variable(tf.random_normal([self._hidden_num]),
trainable=False,
dtype=tf.float32,
name=task + '_b')
self._beta = tf.Variable(tf.zeros([self._hidden_num,
self._output_len]),
trainable=False,
dtype=tf.float32,
name=task + '_beta')
self._var_list = [self._W, self._b, self._beta]
self.H0 = tf.matmul(self._x0, self._W) + self._b # N x L
self.H0_T = tf.transpose(self.H0)
self.H1 = tf.matmul(self._x1, self._W) + self._b # N x L
self.H1_T = tf.transpose(self.H1)
# beta analytic solution : self._beta_s (K x O)
if self._input_len < self._hidden_num: # L < K
identity = tf.constant(np.identity(self._hidden_num),
dtype=tf.float32)
self._beta_s = tf.matmul(
tf.matmul(
tf.matrix_inverse(
tf.matmul(self.H0_T, self.H0) + identity / omega),
self.H0_T), self._t0)
# _beta_s = (H_T*H + I/om)^(-1)*H_T*T
else:
identity = tf.constant(np.identity(self._batch_size),
dtype=tf.float32)
self._beta_s = tf.matmul(
tf.matmul(
self.H0_T,
tf.matrix_inverse(
tf.matmul(self.H0, self.H0_T) + identity / omega)),
self._t0)
# _beta_s = H_T*(H*H_T + I/om)^(-1)*T
self._assign_beta = self._beta.assign(self._beta_s)
self._fx0 = tf.matmul(self.H0, self._beta)
self._fx1 = tf.matmul(self.H1, self._beta)
self._cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(self._fx0, self._t0))
self._init = False
self._feed = False
# for the mnist test
self._correct_prediction = tf.equal(tf.argmax(self._fx1, 1),
tf.argmax(self._t1, 1))
self._accuracy = tf.reduce_mean(
tf.cast(self._correct_prediction, tf.float32))
def __init__(self, params, is_training=True):
self.is_training = is_training
batch_size = params["batch_size"]
num_layers = params['nlayer']
rnn_size = params['n_hidden']
grad_clip = params["grad_clip"]
self.output_keep_prob = tf.placeholder(tf.float32)
NOUT = params['n_output']
# Transition LSTM
cell_lst = []
for i in range(num_layers):
cell = rnncell.ModifiedLSTMCell(
rnn_size,
forget_bias=1,
initializer=tf.contrib.layers.xavier_initializer(),
num_proj=None,
is_training=self.is_training)
if i > 10 and is_training == True:
cell_drop = rnncell.DropoutWrapper(
cell, output_keep_prob=self.output_keep_prob)
cell = cell_drop
cell_lst.append(cell)
self.cell = rnncell.MultiRNNCell(cell_lst)
# LSTM for Q noise
cell_lst = []
for i in range(params['Qnlayer']):
cell_Q_noise = rnncell.ModifiedLSTMCell(
params['Qn_hidden'],
forget_bias=1,
initializer=tf.contrib.layers.xavier_initializer(),
num_proj=None,
is_training=self.is_training)
if i > 10 and is_training == True:
cell_drop = rnncell.DropoutWrapper(
cell_Q_noise, output_keep_prob=self.output_keep_prob)
cell_Q_noise = cell_drop
cell_lst.append(cell_Q_noise)
self.cell_Q_noise = rnncell.MultiRNNCell(cell_lst)
# LSTM for R noise
cell_lst = []
for i in range(params['Rnlayer']):
cell_R_noise = rnncell.ModifiedLSTMCell(
params['Rn_hidden'],
forget_bias=1,
initializer=tf.contrib.layers.xavier_initializer(),
num_proj=None,
is_training=self.is_training)
if i > 10 and is_training == True:
cell_drop = rnncell.DropoutWrapper(
cell_R_noise, output_keep_prob=self.output_keep_prob)
cell_R_noise = cell_drop
cell_lst.append(cell_R_noise)
self.cell_R_noise = rnncell.MultiRNNCell(cell_lst)
self.initial_state = self.cell.zero_state(
batch_size=params['batch_size'], dtype=tf.float32)
self.initial_state_Q_noise = self.cell_Q_noise.zero_state(
batch_size=params['batch_size'], dtype=tf.float32)
self.initial_state_R_noise = self.cell_Q_noise.zero_state(
batch_size=params['batch_size'], dtype=tf.float32)
self.initial_state_R_noise = self.cell_R_noise.zero_state(
batch_size=params['batch_size'], dtype=tf.float32)
self.repeat_data = tf.placeholder(
dtype=tf.int32, shape=[params["batch_size"], params['seq_length']])
#Measurements
self._z = tf.placeholder(dtype=tf.float32,
shape=[None, params['seq_length'], NOUT
]) # batch size, seqlength, feature
self.target_data = tf.placeholder(
dtype=tf.float32, shape=[None, params['seq_length'],
NOUT]) # batch size, seqlength, feature
self._P_inp = tf.placeholder(dtype=tf.float32,
shape=[None, NOUT, NOUT],
name='P')
self._F = 0.0 # state transition matrix
self._alpha_sq = 1. # fading memory control
self.M = 0.0 # process-measurement cross correlation
self._I = tf.placeholder(dtype=tf.float32,
shape=[None, NOUT, NOUT],
name='I')
self.u = 0.0
xres_lst = []
xpred_lst = []
pres_lst = []
tres_lst = []
kres_lst = []
with tf.variable_scope('rnnlm'):
output_w1 = tf.get_variable(
"output_w", [rnn_size, NOUT],
initializer=tf.contrib.layers.xavier_initializer())
output_b1 = tf.get_variable("output_b", [NOUT])
output_w1_Q_noise = tf.get_variable(
"output_w_Q_noise", [params['Qn_hidden'], NOUT],
initializer=tf.contrib.layers.xavier_initializer())
output_b1_Q_noise = tf.get_variable("output_b_Q_noise", [NOUT])
output_w1_R_noise = tf.get_variable(
"output_w_R_noise", [params['Rn_hidden'], NOUT],
initializer=tf.contrib.layers.xavier_initializer())
output_b1_R_noise = tf.get_variable("output_b_R_noise", [NOUT])
state_F = self.initial_state
state_Q = self.initial_state_Q_noise
state_R = self.initial_state_R_noise
with tf.variable_scope("rnnlm"):
for time_step in range(params['seq_length']):
if time_step > 0: tf.get_variable_scope().reuse_variables()
z = self._z[:, time_step, :] #bs,features
if time_step == 0:
self._x = z
self._P = self._P_inp
with tf.variable_scope("transitionF"):
(pred, state_F, ls_internals) = self.cell(self._x, state_F)
pred = tf.matmul(pred, output_w1) + output_b1
with tf.variable_scope("noiseQ"):
(pred_Q_noise, state_Q,
ls_internals) = self.cell_Q_noise(self._x, state_Q)
pred_Q_noise = tf.matmul(
pred_Q_noise, output_w1_Q_noise) + output_b1_Q_noise
with tf.variable_scope("noiseR"):
(pred_R_noise, state_R,
ls_internals) = self.cell_R_noise(z, state_R)
pred_R_noise = tf.matmul(
pred_R_noise, output_w1_R_noise) + output_b1_R_noise
self._x = pred
# lst=tf.unpack(pred, axis=1)
Q = tf.matrix_diag(tf.exp(pred_Q_noise))
# R=tf.matrix_diag(tf.exp(pred_R_noise))
#predict
P = self._P
self._P = P + Q
#update
P = self._P
x = self._x
self._y = z - x
# S = HPH' + R
# project system uncertainty into measurement space
# S = P + R
S = P
# K = PH'inv(S)
# map system uncertainty into kalman gain
K = tf.matmul(P, tf.matrix_inverse(S))
# x = x + Ky
# predict new x with residual scaled by the kalman gain
self._x = x + tf.squeeze(
tf.matmul(K, tf.expand_dims(self._y, 2)))
xpred_lst.append(x)
xres_lst.append(self._x)
tres_lst.append(x)
kres_lst.append(K)
# P = (I-KH)P(I-KH)' + KRK'
I_KH = self._I - K
# self._P = tf.matmul(I_KH, tf.matmul(P, tf.matrix_transpose(I_KH))) + tf.matmul(K, tf.matmul(R, tf.matrix_transpose(K)))
self._P = tf.matmul(
I_KH, tf.matmul(P, tf.matrix_transpose(I_KH))) + tf.matmul(
K, tf.matrix_transpose(K))
self._S = S
self._K = K
final_output = tf.reshape(tf.transpose(tf.pack(xres_lst), [1, 0, 2]),
[-1, params['n_output']])
final_pred_output = tf.reshape(
tf.transpose(tf.pack(xpred_lst), [1, 0, 2]),
[-1, params['n_output']])
flt = tf.squeeze(tf.reshape(self.repeat_data, [-1, 1]), [1])
where_flt = tf.not_equal(flt, 0)
indices = tf.where(where_flt)
y = tf.reshape(self.target_data, [-1, params["n_output"]])
self.final_output = tf.gather(final_output, tf.squeeze(indices, [1]))
self.final_pred_output = tf.gather(final_pred_output,
tf.squeeze(indices, [1]))
self.y = tf.gather(y, tf.squeeze(indices, [1]))
tmp = self.final_output - self.y
loss = tf.nn.l2_loss(tmp)
tmp_pred = self.final_pred_output - self.y
loss_pred = tf.nn.l2_loss(tmp_pred)
self.tvars = tf.trainable_variables()
l2_reg = tf.reduce_sum([tf.nn.l2_loss(var) for var in self.tvars])
l2_reg = tf.mul(l2_reg, 1e-4)
self.cost = tf.reduce_mean(loss) + l2_reg
self.lr = tf.Variable(0.0, trainable=False)
tvars = tf.trainable_variables()
total_parameters = 0
for variable in self.tvars:
# shape is an array of tf.Dimension
shape = variable.get_shape()
variable_parametes = 1
for dim in shape:
variable_parametes *= dim.value
total_parameters += variable_parametes
self.total_parameters = total_parameters
grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, tvars),
grad_clip)
optimizer = tf.train.AdamOptimizer(self.lr)
self.train_op = optimizer.apply_gradients(zip(grads, tvars))
self.states = {}
self.states["F_t"] = state_F
self.states["Q_t"] = state_Q
self.states["R_t"] = state_R
self.states["PCov_t"] = self._P
self.xres_lst = xres_lst
self.pres_lst = pres_lst
self.tres_lst = tres_lst
self.kres_lst = kres_lst
def QuantizedWeight(name, x, n, nbit=2):
"""
Quantize weight.
Args:
x (tf.Tensor): a 4D tensor.
Must have known number of channels, but can have other unknown dimensions.
name (str): operator's name.
n (int or double): variance of weight initialization.
nbit (int): number of bits of quantized weight. Defaults to 2.
Returns:
tf.Tensor with attribute `variables`.
Variable Names:
* ``basis``: basis of quantized weight.
Note:
About multi-GPU training: moving averages across GPUs are not aggregated.
Batch statistics are computed by main training tower. This is consistent with most frameworks.
"""
num_filters = x.get_shape().as_list()[-1]
init_basis = []
base = NORM_PPF_0_75 * ((2. / n)**0.5) / (2**(nbit - 1))
for j in range(nbit):
init_basis.append([(2**j) * base for i in range(num_filters)])
init_basis = tf.constant_initializer(init_basis)
bit_dims = [nbit, num_filters]
num_levels = 2**nbit
delta = EPS
# initialize level multiplier
init_level_multiplier = []
for i in range(num_levels):
level_multiplier_i = [0. for j in range(nbit)]
level_number = i
for j in range(nbit):
binary_code = level_number % 2
if binary_code == 0:
binary_code = -1
level_multiplier_i[j] = float(binary_code)
level_number = level_number // 2
init_level_multiplier.append(level_multiplier_i)
# initialize threshold multiplier
init_thrs_multiplier = []
for i in range(1, num_levels):
thrs_multiplier_i = [0. for j in range(num_levels)]
thrs_multiplier_i[i - 1] = 0.5
thrs_multiplier_i[i] = 0.5
init_thrs_multiplier.append(thrs_multiplier_i)
with tf.variable_scope(name):
basis = tf.get_variable('basis',
bit_dims,
tf.float32,
initializer=init_basis,
trainable=False)
level_codes = tf.constant(init_level_multiplier)
thrs_multiplier = tf.constant(init_thrs_multiplier)
sum_multiplier = tf.constant(
1., shape=[1, tf.reshape(x, [-1, num_filters]).get_shape()[0]])
sum_multiplier_basis = tf.constant(1., shape=[1, nbit])
ctx = get_current_tower_context() # current tower context
# calculate levels and sort
levels = tf.matmul(level_codes, basis)
levels, sort_id = tf.nn.top_k(tf.transpose(levels, [1, 0]), num_levels)
levels = tf.reverse(levels, [-1])
sort_id = tf.reverse(sort_id, [-1])
levels = tf.transpose(levels, [1, 0])
sort_id = tf.transpose(sort_id, [1, 0])
# calculate threshold
thrs = tf.matmul(thrs_multiplier, levels)
# calculate level codes per channel
reshape_x = tf.reshape(x, [-1, num_filters])
level_codes_channelwise_dims = tf.stack(
[num_levels * num_filters, nbit])
level_codes_channelwise = tf.fill(level_codes_channelwise_dims, 0.)
for i in range(num_levels):
eq = tf.equal(sort_id, i)
level_codes_channelwise = tf.where(
tf.reshape(eq, [-1]), level_codes_channelwise + level_codes[i],
level_codes_channelwise)
level_codes_channelwise = tf.reshape(level_codes_channelwise,
[num_levels, num_filters, nbit])
# calculate output y and its binary code
y = tf.zeros_like(x) + levels[0] # output
zero_dims = tf.stack([tf.shape(reshape_x)[0] * num_filters, nbit])
bits_y = tf.fill(zero_dims, -1.)
zero_y = tf.zeros_like(x)
zero_bits_y = tf.fill(zero_dims, 0.)
zero_bits_y = tf.reshape(zero_bits_y, [-1, num_filters, nbit])
for i in range(num_levels - 1):
g = tf.greater(x, thrs[i])
y = tf.where(g, zero_y + levels[i + 1], y)
bits_y = tf.where(
tf.reshape(g, [-1]),
tf.reshape(zero_bits_y + level_codes_channelwise[i + 1],
[-1, nbit]), bits_y)
bits_y = tf.reshape(bits_y, [-1, num_filters, nbit])
# training
if ctx.is_main_training_tower:
BT = tf.transpose(bits_y, [2, 0, 1])
# calculate BTxB
BTxB = []
for i in range(nbit):
for j in range(nbit):
BTxBij = tf.multiply(BT[i], BT[j])
BTxBij = tf.matmul(sum_multiplier, BTxBij)
if i == j:
mat_one = tf.ones([1, num_filters])
BTxBij = BTxBij + (delta * mat_one) # + E
BTxB.append(BTxBij)
BTxB = tf.reshape(tf.stack(values=BTxB), [nbit, nbit, num_filters])
# calculate inverse of BTxB
if nbit > 2:
BTxB_transpose = tf.transpose(BTxB, [2, 0, 1])
BTxB_inv = tf.matrix_inverse(BTxB_transpose)
BTxB_inv = tf.transpose(BTxB_inv, [1, 2, 0])
elif nbit == 2:
det = tf.multiply(BTxB[0][0], BTxB[1][1]) - tf.multiply(
BTxB[0][1], BTxB[1][0])
inv = []
inv.append(BTxB[1][1] / det)
inv.append(-BTxB[0][1] / det)
inv.append(-BTxB[1][0] / det)
inv.append(BTxB[0][0] / det)
BTxB_inv = tf.reshape(tf.stack(values=inv),
[nbit, nbit, num_filters])
elif nbit == 1:
BTxB_inv = tf.reciprocal(BTxB)
# calculate BTxX
BTxX = []
for i in range(nbit):
BTxXi0 = tf.multiply(BT[i], reshape_x)
BTxXi0 = tf.matmul(sum_multiplier, BTxXi0)
BTxX.append(BTxXi0)
BTxX = tf.reshape(tf.stack(values=BTxX), [nbit, num_filters])
BTxX = BTxX + (delta * basis) # + basis
# calculate new basis
new_basis = []
for i in range(nbit):
new_basis_i = tf.multiply(BTxB_inv[i], BTxX)
new_basis_i = tf.matmul(sum_multiplier_basis, new_basis_i)
new_basis.append(new_basis_i)
new_basis = tf.reshape(tf.stack(values=new_basis),
[nbit, num_filters])
# create moving averages op
updata_moving_basis = moving_averages.assign_moving_average(
basis, new_basis, MOVING_AVERAGES_FACTOR)
add_model_variable(basis)
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, updata_moving_basis)
y = x + tf.stop_gradient(-x) + tf.stop_gradient(y) # gradient: y=x
y.variables = VariableHolder(basis=basis)
return y
