def testWrongDimensions(self):
# The matrix and right-hand sides should have the same number of rows.
with self.test_session():
matrix = tf.constant([[1., 0.], [0., 1.]])
rhs = tf.constant([[1., 0.]])
with self.assertRaises(ValueError):
tf.matrix_solve_ls(matrix, rhs)
def genPerturbations(opt):
with tf.name_scope("genPerturbations"):
X = np.tile(opt.canon4pts[:,0],[opt.batchSize,1])
Y = np.tile(opt.canon4pts[:,1],[opt.batchSize,1])
dX = tf.random_normal([opt.batchSize,4])*opt.warpScale["pert"] \
+tf.random_normal([opt.batchSize,1])*opt.warpScale["trans"]
dY = tf.random_normal([opt.batchSize,4])*opt.warpScale["pert"] \
+tf.random_normal([opt.batchSize,1])*opt.warpScale["trans"]
O = np.zeros([opt.batchSize,4],dtype=np.float32)
I = np.ones([opt.batchSize,4],dtype=np.float32)
# fit warp parameters to generated displacements
if opt.warpType == "affine":
J = np.concatenate([np.stack([X,Y,I,O,O,O],axis=-1),
np.stack([O,O,O,X,Y,I],axis=-1)],axis=1)
dXY = tf.expand_dims(tf.concat(1,[dX,dY]),-1)
dpBatch = tf.matrix_solve_ls(J,dXY)
elif opt.warpType == "homography":
A = tf.concat([tf.stack([X,Y,I,O,O,O,-X*(X+dX),-Y*(X+dX)],axis=-1),
tf.stack([O,O,O,X,Y,I,-X*(Y+dY),-Y*(Y+dY)],axis=-1)],1)
b = tf.expand_dims(tf.concat([X+dX,Y+dY],1),-1)
dpBatch = tf.matrix_solve_ls(A,b)
dpBatch -= tf.to_float(tf.reshape([1,0,0,0,1,0,0,0],[1,8,1]))
dpBatch = tf.reduce_sum(dpBatch,reduction_indices=-1)
dpMtrxBatch = warp.vec2mtrxBatch(dpBatch,opt)
return dpBatch
def testEmpty(self):
full = np.array([[1., 2.], [3., 4.], [5., 6.]])
empty0 = np.empty([3, 0])
empty1 = np.empty([0, 2])
for fast in [True, False]:
with self.test_session():
tf_ans = tf.matrix_solve_ls(empty0, empty0, fast=fast).eval()
self.assertEqual(tf_ans.shape, (0, 0))
tf_ans = tf.matrix_solve_ls(empty0, full, fast=fast).eval()
self.assertEqual(tf_ans.shape, (0, 2))
tf_ans = tf.matrix_solve_ls(full, empty0, fast=fast).eval()
self.assertEqual(tf_ans.shape, (2, 0))
tf_ans = tf.matrix_solve_ls(empty1, empty1, fast=fast).eval()
self.assertEqual(tf_ans.shape, (2, 2))
def solve_ls(A, B):
"""functional maps layer.
Args:
A: part descriptors projected onto part shape eigenvectors.
B: model descriptors projected onto model shape eigenvectors.
Returns:
Ct_est: estimated C (transposed), such that CA ~= B
safeguard_inverse:
"""
# transpose input matrices
At = tf.transpose(A, [0, 2, 1])
Bt = tf.transpose(B, [0, 2, 1])
# solve C via least-squares
Ct_est = tf.matrix_solve_ls(At, Bt)
C_est = tf.transpose(Ct_est, [0, 2, 1])
# calculate error for safeguarding
safeguard_inverse = tf.nn.l2_loss(tf.matmul(At, Ct_est) -
Bt) / tf.to_float(
tf.reduce_prod(tf.shape(A)))
return C_est, safeguard_inverse
def homography(x1s, x2s):
def ax(p, q):
return [p[0], p[1], 1, 0, 0, 0, -p[0] * q[0], -p[1] * q[0]]
def ay(p, q):
return [0, 0, 0, p[0], p[1], 1, -p[0] * q[1], -p[1] * q[1]]
p = []
p.append(ax(x1s[0], x2s[0]))
p.append(ay(x1s[0], x2s[0]))
p.append(ax(x1s[1], x2s[1]))
p.append(ay(x1s[1], x2s[1]))
p.append(ax(x1s[2], x2s[2]))
p.append(ay(x1s[2], x2s[2]))
p.append(ax(x1s[3], x2s[3]))
p.append(ay(x1s[3], x2s[3]))
# A is 8x8
A = tf.stack(p, axis=0)
m = [[
x2s[0][0], x2s[0][1], x2s[1][0], x2s[1][1], x2s[2][0], x2s[2][1],
x2s[3][0], x2s[3][1]
]]
# P is 8x1
P = tf.transpose(tf.stack(m, axis=0))
# here we solve the linear system
# we transpose the result for convenience
return tf.transpose(tf.matrix_solve_ls(A, P, fast=True))
def _verifyRegularized(self, x, y, l2_regularizer):
for np_type in [np.float32, np.float64]:
# Test with a single matrix.
a = x.astype(np_type)
b = y.astype(np_type)
np_ans = BatchRegularizedLeastSquares(a, b, l2_regularizer)
with self.test_session():
# Test with the batch version of matrix_solve_ls on regular matrices
tf_ans = tf.batch_matrix_solve_ls(
a, b, l2_regularizer=l2_regularizer, fast=True).eval()
self.assertAllClose(np_ans, tf_ans, atol=1e-5, rtol=1e-5)
# Test with the simple matrix_solve_ls on regular matrices
tf_ans = tf.matrix_solve_ls(
a, b, l2_regularizer=l2_regularizer, fast=True).eval()
self.assertAllClose(np_ans, tf_ans, atol=1e-5, rtol=1e-5)
# Test with a 2x3 batch of matrices.
a = np.tile(x.astype(np_type), [2, 3, 1, 1])
b = np.tile(y.astype(np_type), [2, 3, 1, 1])
np_ans = BatchRegularizedLeastSquares(a, b, l2_regularizer)
with self.test_session():
tf_ans = tf.batch_matrix_solve_ls(
a, b, l2_regularizer=l2_regularizer, fast=True).eval()
self.assertAllClose(np_ans, tf_ans, atol=1e-5, rtol=1e-5)
def _verifySolve(self, x, y):
for np_type in [np.float32, np.float64]:
a = x.astype(np_type)
b = y.astype(np_type)
np_ans, _, _, _ = np.linalg.lstsq(a, b)
for fast in [True, False]:
with self.test_session():
tf_ans = tf.matrix_solve_ls(a, b, fast=fast)
ans = tf_ans.eval()
self.assertEqual(np_ans.shape, tf_ans.get_shape())
self.assertEqual(np_ans.shape, ans.shape)
# Check residual norm.
tf_r = b - BatchMatMul(a, ans)
tf_r_norm = np.sum(tf_r * tf_r)
np_r = b - BatchMatMul(a, np_ans)
np_r_norm = np.sum(np_r * np_r)
self.assertAllClose(np_r_norm, tf_r_norm)
# Check solution.
if fast or a.shape[0] >= a.shape[1]:
# We skip this test for the underdetermined case when using the
# slow path, because Eigen does not return a minimum norm solution.
# TODO(rmlarsen): Enable this check for all paths if/when we fix
# Eigen's solver.
self.assertAllClose(np_ans, ans, atol=1e-5, rtol=1e-5)
def sup_penalty_surreal(C_est, source_evecs, target_evecs):
"""
Args: full_source_evecs and full_target_evecs are over batch..
"""
fmap = tf.matrix_solve_ls(tf.transpose(target_evecs, [0, 2, 1]),
tf.transpose(source_evecs, [0, 2, 1]))
return tf.nn.l2_loss(C_est - fmap)
def _random_transform(self, batch_size, spatial_rank):
"""
computes a relative transformation
mapping <-1..1, -1..1, -1..1> to <-1..1, -1..1, -1..1> (in 3D)
or <-1..1, -1..1> to <-1..1, -1..1> (in 2D).
:param batch_size: number of different random transformations
:param spatial_rank: number of spatial dimensions
:return:
"""
output_corners = get_relative_corners(spatial_rank)
output_corners = tf.tile([output_corners], [batch_size, 1, 1])
# make randomised output corners
random_size = [batch_size, 2**spatial_rank, spatial_rank]
random_scale = tf.random_uniform(random_size, 1. - self.scale, 1.0)
source_corners = output_corners * random_scale
# make homogeneous corners
batch_ones = tf.ones_like(output_corners[..., 0:1])
source_corners = tf.concat([source_corners, batch_ones], -1)
output_corners = tf.concat([output_corners, batch_ones], -1)
ls_transform = tf.matrix_solve_ls(output_corners, source_corners)
return tf.transpose(ls_transform, [0, 2, 1])
def genPerturbations(opt):
with tf.name_scope("genPerturbations"):
X = np.tile(opt.canon4pts[:,0],[opt.batchSize,1])
Y = np.tile(opt.canon4pts[:,1],[opt.batchSize,1])
dX = tf.random_normal([opt.batchSize,4])*opt.pertScale \
+tf.random_normal([opt.batchSize,1])*opt.transScale
dY = tf.random_normal([opt.batchSize,4])*opt.pertScale \
+tf.random_normal([opt.batchSize,1])*opt.transScale
O = np.zeros([opt.batchSize,4],dtype=np.float32)
I = np.ones([opt.batchSize,4],dtype=np.float32)
# fit warp parameters to generated displacements
if opt.warpType=="homography":
A = tf.concat([tf.stack([X,Y,I,O,O,O,-X*(X+dX),-Y*(X+dX)],axis=-1),
tf.stack([O,O,O,X,Y,I,-X*(Y+dY),-Y*(Y+dY)],axis=-1)],1)
b = tf.expand_dims(tf.concat([X+dX,Y+dY],1),-1)
pPert = tf.matrix_solve(A,b)[:,:,0]
pPert -= tf.to_float([[1,0,0,0,1,0,0,0]])
else:
if opt.warpType=="translation":
J = np.concatenate([np.stack([I,O],axis=-1),
np.stack([O,I],axis=-1)],axis=1)
if opt.warpType=="similarity":
J = np.concatenate([np.stack([X,Y,I,O],axis=-1),
np.stack([-Y,X,O,I],axis=-1)],axis=1)
if opt.warpType=="affine":
J = np.concatenate([np.stack([X,Y,I,O,O,O],axis=-1),
np.stack([O,O,O,X,Y,I],axis=-1)],axis=1)
dXY = tf.expand_dims(tf.concat([dX,dY],1),-1)
pPert = tf.matrix_solve_ls(J,dXY)[:,:,0]
return pPert
def __init__(self,
input_layer_size,
hidden_layer_size,
sigma,
rho,
eta=1e-3):
super(DecompositionRBFN,
self).__init__(input_layer_size, hidden_layer_size, sigma, rho,
eta)
# Redefine training error:
self.training_error = tf.reduce_mean(
tf.square(self.y_p - self.y_placeholder)
) / 2 + rho * tf.reduce_sum(tf.square(self.c))
# Redefine optimization algorithm:
self.train_step = tf.train.GradientDescentOptimizer(eta).minimize(
self.training_error, var_list=[self.c])
# LLSQ:
self.P = tf.placeholder(tf.float32)
self.llsq_matrix = 2.0 * (
tf.matmul(self.gaussian_f, self.gaussian_f, transpose_a=True) /
(2.0 * self.P) + rho * tf.eye(hidden_layer_size))
self.llsq_rhs = tf.matmul(self.gaussian_f,
tf.expand_dims(self.y_placeholder, 1),
transpose_a=True) / self.P
self.update_v = self.v.assign(
tf.squeeze(
tf.matrix_solve_ls(self.llsq_matrix, self.llsq_rhs,
fast=False)))
def _verifySolveBatch(self, x, y):
# Since numpy.linalg.lsqr does not support batch solves, as opposed
# to numpy.linalg.solve, we just perform this test for a fixed batch size
# of 2x3.
for np_type in [np.float32, np.float64]:
a = np.tile(x.astype(np_type), [2, 3, 1, 1])
b = np.tile(y.astype(np_type), [2, 3, 1, 1])
np_ans = np.empty([2, 3, a.shape[-1], b.shape[-1]])
for dim1 in range(2):
for dim2 in range(3):
np_ans[dim1, dim2, :, :], _, _, _ = np.linalg.lstsq(
a[dim1, dim2, :, :], b[dim1, dim2, :, :])
for fast in [True, False]:
with self.test_session():
tf_ans = tf.matrix_solve_ls(a, b, fast=fast).eval()
self.assertEqual(np_ans.shape, tf_ans.shape)
# Check residual norm.
tf_r = b - BatchMatMul(a, tf_ans)
tf_r_norm = np.sum(tf_r * tf_r)
np_r = b - BatchMatMul(a, np_ans)
np_r_norm = np.sum(np_r * np_r)
self.assertAllClose(np_r_norm, tf_r_norm)
# Check solution.
if fast or a.shape[-2] >= a.shape[-1]:
# We skip this test for the underdetermined case when using the
# slow path, because Eigen does not return a minimum norm solution.
# TODO(rmlarsen): Enable this check for all paths if/when we fix
# Eigen's solver.
self.assertAllClose(np_ans, tf_ans, atol=1e-5, rtol=1e-5)
def homography(x1s, x2s):
p = []
# we build matrix A by using only 4 point correspondence. The linear
# system is solved with the least square method, so here
# we could even pass more correspondence
p.append(ax(x1s[0], x2s[0]))
p.append(ay(x1s[0], x2s[0]))
p.append(ax(x1s[1], x2s[1]))
p.append(ay(x1s[1], x2s[1]))
p.append(ax(x1s[2], x2s[2]))
p.append(ay(x1s[2], x2s[2]))
p.append(ax(x1s[3], x2s[3]))
p.append(ay(x1s[3], x2s[3]))
# A is 8x8
A = tf.stack(p, axis=0)
m = [[
x2s[0][0], x2s[0][1], x2s[1][0], x2s[1][1], x2s[2][0], x2s[2][1],
x2s[3][0], x2s[3][1]
]]
# P is 8x1
P = tf.transpose(tf.stack(m, axis=0))
# here we solve the linear system
# we transpose the result for convenience
return tf.transpose(tf.matrix_solve_ls(A, P, fast=True))
def _verifySolve(self, x, y):
for np_type in [np.float32, np.float64]:
a = x.astype(np_type)
b = y.astype(np_type)
np_ans, _, _, _ = np.linalg.lstsq(a, b)
for fast in [True, False]:
with self.test_session():
tf_ans = tf.matrix_solve_ls(a, b, fast=fast)
ans = tf_ans.eval()
self.assertEqual(np_ans.shape, tf_ans.get_shape())
self.assertEqual(np_ans.shape, ans.shape)
# Check residual norm.
tf_r = b - BatchMatMul(a, ans)
tf_r_norm = np.sum(tf_r * tf_r)
np_r = b - BatchMatMul(a, np_ans)
np_r_norm = np.sum(np_r * np_r)
self.assertAllClose(np_r_norm, tf_r_norm)
# Check solution.
if fast or a.shape[0] >= a.shape[1]:
# We skip this test for the underdetermined case when using the
# slow path, because Eigen does not return a minimum norm solution.
# TODO(rmlarsen): Enable this check for all paths if/when we fix
# Eigen's solver.
self.assertAllClose(np_ans, ans, atol=1e-5, rtol=1e-5)
def _verifyRegularized(self, x, y, l2_regularizer):
for np_type in [np.float32, np.float64]:
# Test with a single matrix.
a = x.astype(np_type)
b = y.astype(np_type)
np_ans = BatchRegularizedLeastSquares(a, b, l2_regularizer)
with self.test_session():
# Test with the batch version of matrix_solve_ls on regular matrices
tf_ans = tf.batch_matrix_solve_ls(
a, b, l2_regularizer=l2_regularizer, fast=True).eval()
self.assertAllClose(np_ans, tf_ans, atol=1e-5, rtol=1e-5)
# Test with the simple matrix_solve_ls on regular matrices
tf_ans = tf.matrix_solve_ls(a,
b,
l2_regularizer=l2_regularizer,
fast=True).eval()
self.assertAllClose(np_ans, tf_ans, atol=1e-5, rtol=1e-5)
# Test with a 2x3 batch of matrices.
a = np.tile(x.astype(np_type), [2, 3, 1, 1])
b = np.tile(y.astype(np_type), [2, 3, 1, 1])
np_ans = BatchRegularizedLeastSquares(a, b, l2_regularizer)
with self.test_session():
tf_ans = tf.batch_matrix_solve_ls(
a, b, l2_regularizer=l2_regularizer, fast=True).eval()
self.assertAllClose(np_ans, tf_ans, atol=1e-5, rtol=1e-5)
def _verifySolveBatch(self, x, y):
# Since numpy.linalg.lsqr does not support batch solves, as opposed
# to numpy.linalg.solve, we just perform this test for a fixed batch size
# of 2x3.
for np_type in [np.float32, np.float64]:
a = np.tile(x.astype(np_type), [2, 3, 1, 1])
b = np.tile(y.astype(np_type), [2, 3, 1, 1])
np_ans = np.empty([2, 3, a.shape[-1], b.shape[-1]])
for dim1 in range(2):
for dim2 in range(3):
np_ans[dim1, dim2, :, :], _, _, _ = np.linalg.lstsq(
a[dim1, dim2, :, :], b[dim1, dim2, :, :])
for fast in [True, False]:
with self.test_session():
tf_ans = tf.matrix_solve_ls(a, b, fast=fast).eval()
self.assertEqual(np_ans.shape, tf_ans.shape)
# Check residual norm.
tf_r = b - BatchMatMul(a, tf_ans)
tf_r_norm = np.sum(tf_r * tf_r)
np_r = b - BatchMatMul(a, np_ans)
np_r_norm = np.sum(np_r * np_r)
self.assertAllClose(np_r_norm, tf_r_norm)
# Check solution.
if fast or a.shape[-2] >= a.shape[-1]:
# We skip this test for the underdetermined case when using the
# slow path, because Eigen does not return a minimum norm solution.
# TODO(rmlarsen): Enable this check for all paths if/when we fix
# Eigen's solver.
self.assertAllClose(np_ans, tf_ans, atol=1e-5, rtol=1e-5)
def testBatchResultSize(self): # 3x3x3 matrices, 3x3x1 right-hand sides. matrix = np.array([1., 2., 3., 4., 5., 6., 7., 8., 9.] * 3).reshape(3, 3, 3) rhs = np.array([1., 2., 3.] * 3).reshape(3, 3, 1) answer = tf.matrix_solve(matrix, rhs) ls_answer = tf.matrix_solve_ls(matrix, rhs) self.assertEqual(ls_answer.get_shape(), [3, 3, 1]) self.assertEqual(answer.get_shape(), [3, 3, 1])
def my_body_text_lines(loop_i, tmp_tf_input):
tp_indices_tf = self.final_group_tf_api(successions_pair,
sub_graphs_pair, loop_i)
text_line_boxes_tf = tf.gather(text_proposals_tf, tp_indices_tf)
X_tf = (text_line_boxes_tf[:, 0] +
text_line_boxes_tf[:, 2]) / 2 # 求每一个小框的中心x，y坐标
Y_tf = (text_line_boxes_tf[:, 1] + text_line_boxes_tf[:, 3]) / 2
X_tf = tf.reshape(X_tf, [-1, 1])
ones = tf.ones(shape=tf.shape(X_tf))
ones = tf.cast(ones, tf.float32)
X_tf = tf.concat([X_tf, ones], 1)
Y_tf = tf.reshape(Y_tf, [-1, 1])
z1 = tf.matrix_solve_ls(
X_tf, Y_tf) #求解多个线性方程的最小二乘问题 ---- np.polyfit(X, Y, 1)
x0_index = tf.argmin(text_line_boxes_tf[:, 0]) # 文本行x坐标最小值
x1_index = tf.argmax(text_line_boxes_tf[:, 2]) # 文本行x坐标最大值
x0 = tf.gather(text_line_boxes_tf[:, 0], x0_index)
x1 = tf.gather(text_line_boxes_tf[:, 2], x1_index)
offset = (text_line_boxes_tf[0, 2] -
text_line_boxes_tf[0, 0]) * 0.5 # 小框宽度的一半
# 以全部小框的左上角这个点去拟合一条直线，然后计算一下文本行x坐标的极左极右对应的y坐标
lt_y, rt_y = self.fit_y_tf_api(text_line_boxes_tf[:, 0],
text_line_boxes_tf[:, 1],
x0 + offset, x1 - offset)
# 以全部小框的左下角这个点去拟合一条直线，然后计算一下文本行x坐标的极左极右对应的y坐标
lb_y, rb_y = self.fit_y_tf_api(text_line_boxes_tf[:, 0],
text_line_boxes_tf[:, 3],
x0 + offset, x1 - offset)
tmp_scores = tf.gather(scores_tf, tp_indices_tf)
score = tf.reduce_mean(tmp_scores) # 求全部小框得分的均值作为文本行的均值
height = tf.reduce_mean((text_line_boxes_tf[:, 3] -
text_line_boxes_tf[:, 1])) # 小框平均高度
height = tf.add(height, 2.5)
x0 = tf.reshape(x0, [-1])
ty = tf.reshape(tf.minimum(lt_y, rt_y), [-1]) # 文本行上端 线段 的y坐标的小值
x1 = tf.reshape(x1, [-1])
by = tf.reshape(tf.maximum(lb_y, rb_y), [-1]) # 文本行下端 线段 的y坐标的大值
score = tf.reshape(score, [-1]) # 文本行得分
z1_0 = tf.reshape(z1[0], [-1]) # 根据中心点拟合的直线的k，b
z1_1 = tf.reshape(z1[1], [-1])
height = tf.reshape(height, [-1])
text_lines_tf_tmp = tf.concat(
[x0, ty, x1, by, score, z1_0, z1_1, height], 0)
result = tf.concat(
[tmp_tf_input,
tf.reshape(text_lines_tf_tmp, shape=[1, 8])], 0)
return loop_i + 1, result
def normal():
with np.load("notMNIST.npz") as data:
Data, Target = data["images"], data["labels"]
posClass = 2
negClass = 9
dataIndx = (Target == posClass) + (Target == negClass)
Data = Data[dataIndx] / 255.
Target = Target[dataIndx].reshape(-1, 1)
Target[Target == posClass] = 1
Target[Target == negClass] = 0
np.random.seed(521)
randIndx = np.arange(len(Data))
np.random.shuffle(randIndx)
Data, Target = Data[randIndx], Target[randIndx]
trainData, trainTarget = Data[:3500], Target[:3500]
validData, validTarget = Data[3500:3600], Target[3500:3600]
testData, testTarget = Data[3600:], Target[3600:]
trainData = trainData.reshape((-1, 28 * 28)).astype(np.float32)
validData = validData.reshape((-1, 28 * 28)).astype(np.float32)
testData = testData.reshape((-1, 28 * 28)).astype(np.float32)
trainTarget = trainTarget.astype(np.float32)
X = tf.Variable(trainData)
Y = tf.Variable(trainTarget)
start = time.time()
weight = tf.matrix_solve_ls(X, Y, l2_regularizer=0.0, fast=True)
end = time.time()
time_ms = (end - start) * 1000.0
prediction = tf.matmul(trainData, weight)
#squared_diff_sum = tf.reduce_sum(tf.pow((prediction - Y), 2))
#training_MSE_loss = squared_diff_sum / (2.0*500.0)
training_MSE_loss = tf.scalar_mul(
0.5, tf.reduce_mean(tf.pow(tf.subtract(prediction, Y), 2)))
validation_pred = tf.matmul(validData, weight)
val_acc_count = 0.0
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for i in list(range(0, len(validTarget))):
if sess.run(tf.greater(validation_pred[i],
0.5)) and validTarget[i] > 0.5:
val_acc_count += 1
elif sess.run(tf.less(validation_pred[i],
0.5)) and validTarget[i] < 0.5:
val_acc_count += 1
test_pred = tf.matmul(testData, weight)
test_acc_count = 0.0
for i in list(range(0, len(testTarget))):
if sess.run(tf.greater(test_pred[i], 0.5)) and testTarget[i] > 0.5:
test_acc_count += 1
elif sess.run(tf.less(test_pred[i], 0.5)) and testTarget[i] < 0.5:
test_acc_count += 1
return training_MSE_loss, val_acc_count / 100.0, test_acc_count / 145.0, time_ms
def newton_type_update_batched_global(self, lhs, rhs, psd):
delta_batched_t = tf.squeeze(tf.matrix_solve_ls(
lhs,
tf.expand_dims(rhs, axis=-1),
fast=psd and pkg_constants.CHOLESKY_LSTSQS),
axis=-1)
nr_update_batched = tf.transpose(delta_batched_t)
return nr_update_batched
def testBatchResultSize(self):
# 3x3x3 matrices, 3x3x1 right-hand sides.
matrix = np.array([1., 2., 3., 4., 5., 6., 7., 8., 9.] * 3).reshape(
3, 3, 3)
rhs = np.array([1., 2., 3.] * 3).reshape(3, 3, 1)
answer = tf.matrix_solve(matrix, rhs)
ls_answer = tf.matrix_solve_ls(matrix, rhs)
self.assertEqual(ls_answer.get_shape(), [3, 3, 1])
self.assertEqual(answer.get_shape(), [3, 3, 1])
def _solve_w_mean(self, new_z_mean, M):
"""Minimise the conditional KL-divergence between z wrt w."""
w_matrix = tf.matmul(M, M, transpose_b=True)
w_rhs = tf.einsum('bmc,sbc->bms', M, new_z_mean)
w_mean = tf.matrix_solve_ls(
matrix=w_matrix, rhs=w_rhs,
l2_regularizer=self._obs_noise_stddev ** 2 / self._w_prior_stddev ** 2)
w_mean = tf.einsum('bms->sbm', w_mean)
return w_mean
def get_filter_matrix_conj(
Y, inverse_power, correlation_matrix, correlation_vector,
K, delay, mode='solve'):
dyn_shape = tf.shape(Y)
D = dyn_shape[0]
correlation_vector = tf.reshape(correlation_vector, (D * D * K, 1))
selector = tf.reshape(tf.transpose(tf.reshape(
tf.range(D * D * K), (D, K, D)
), (1, 0, 2)), (-1,))
inv_selector = tf.reshape(tf.transpose(tf.reshape(
tf.range(D * D * K), (K, D, D)
), (1, 0, 2)), (-1,))
correlation_vector = tf.gather(correlation_vector, inv_selector)
# Idea is to solve matrix inversion independently for each block matrix.
# This should still be faster and more stable than np.linalg.inv().
# print(np.linalg.cond(correlation_matrix))
if mode == 'inv':
with tf.device('/cpu:0'):
inv_correlation_matrix = tf.matrix_inverse(correlation_matrix)
stacked_filter_conj = tf.einsum(
'ab,cb->ca',
inv_correlation_matrix, tf.reshape(correlation_vector, (D, D * K))
)
stacked_filter_conj = tf.reshape(stacked_filter_conj, (D * D * K, 1))
elif mode == 'solve':
with tf.device('/cpu:0'):
stacked_filter_conj = tf.reshape(
tf.matrix_solve(
tf.tile(correlation_matrix[None, ...], [D, 1, 1]),
tf.reshape(correlation_vector, (D, D * K, 1))
),
(D * D * K, 1)
)
elif mode == 'solve_ls':
g = tf.get_default_graph()
with tf.device('/cpu:0'), g.gradient_override_map(
{"MatrixSolveLs": "CustomMatrixSolveLs"}):
stacked_filter_conj = tf.reshape(
tf.matrix_solve_ls(
tf.tile(correlation_matrix[None, ...], [D, 1, 1]),
tf.reshape(correlation_vector, (D, D * K, 1)), fast=False
),
(D * D * K, 1)
)
else:
raise ValueError
stacked_filter_conj = tf.gather(stacked_filter_conj, selector)
filter_matrix_conj = tf.transpose(
tf.reshape(stacked_filter_conj, (K, D, D)),
(0, 2, 1)
)
return filter_matrix_conj
def embedding_inversion(gru_embedding, name, batch_size):
with tf.variable_scope("embedding_inversion"):
gru_embedding = tf.log(gru_embedding+1e-10) - tf.log(1-gru_embedding+1e-10)
image_embedding_weight = tf.contrib.framework.get_variables_by_name(name+"/weights")
image_embedding_bias = tf.contrib.framework.get_variables_by_name(name+"/biases")
embedding_inverse_tensor = tf.matrix_solve_ls(tf.transpose(tf.tile(image_embedding_weight,
[batch_size, 1, 1]), [0, 2, 1]),
tf.expand_dims(tf.subtract(gru_embedding,image_embedding_bias), 2),
fast=True, l2_regularizer=1e-5)
return tf.squeeze(embedding_inverse_tensor), image_embedding_weight
def solve(A, y):
t_A = tf.placeholder(dtype=floatX, shape=A.shape)
t_y = tf.placeholder(dtype=floatX, shape=y.shape)
t_x = tf.matrix_solve_ls(t_A, t_y, l2_regularizer=1e-3)
S = tf.Session()
S.run(tf.initialize_all_variables())
rez = S.run(t_x, feed_dict={t_A:A, t_y:y})
return rez
def random_transform(self,batch_size):
if self._transform is None:
corners = [[[-1.,-1.,-1.],[-1.,-1.,1.],[-1.,1.,-1.],[-1.,1.,1.],[1.,-1.,-1.],[1.,-1.,1.],[1.,1.,-1.],[1.,1.,1.]]]
corners = tf.tile(corners,[batch_size,1,1])
corners2 = corners * \
(1-tf.random_uniform([batch_size,8,3],0,self.scale))
corners_homog = tf.concat([corners,tf.ones([batch_size,8,1])],2)
corners2_homog = tf.concat([corners2,tf.ones([batch_size,8,1])],2)
_transform = tf.matrix_solve_ls(corners_homog,corners2_homog)
self._transform = tf.transpose(_transform,[0,2,1])
return self._transform
def solve_ls(self, A, B):
# Transpose input matrices
At = tf.transpose(A, [0, 2, 1])
Bt = tf.transpose(B, [0, 2, 1])
# Solve C via least-squares
Ct_est = tf.matrix_solve_ls(At, Bt)
#Ct_est = tf.matrix_solve_ls(At, Bt, l2_regularizer = 0.000001)
C_est = tf.transpose(Ct_est, [0, 2, 1], name='C_est')
return C_est
def random_transform(self,batch_size):
if self._transform is None:
corners = [[[-1.,-1.,-1.],[-1.,-1.,1.],[-1.,1.,-1.],[-1.,1.,1.],[1.,-1.,-1.],[1.,-1.,1.],[1.,1.,-1.],[1.,1.,1.]]]
corners = tf.tile(corners,[batch_size,1,1])
corners2 = corners * \
(1-tf.random_uniform([batch_size,8,3],0,self.scale))
corners_homog = tf.concat([corners,tf.ones([batch_size,8,1])],2)
corners2_homog = tf.concat([corners2,tf.ones([batch_size,8,1])],2)
_transform = tf.matrix_solve_ls(corners_homog,corners2_homog)
self._transform = tf.transpose(_transform,[0,2,1])
return self._transform
def newton_type_update_full_global(self, lhs, rhs, psd):
delta_t = tf.squeeze(
tf.matrix_solve_ls(
lhs,
# (full_data_model.hessians + tf.transpose(full_data_model.hessians, perm=[0, 2, 1])) / 2, # symmetrization, don't need this with closed forms
tf.expand_dims(rhs, axis=-1),
fast=psd and pkg_constants.CHOLESKY_LSTSQS),
axis=-1)
nr_update_full = tf.transpose(delta_t)
return nr_update_full
def _propagate_solution(self, args):
"""Perform least squares to get A- and B-matrices and propagate forward
Args:
args: Various arguments and specifications
"""
# Define X- and Y-matrices
X = tf.concat([self.g_vals[:, :-1], self.u[:, :(args.seq_length - 1)]],
axis=2)
Y = self.g_vals[:, 1:]
# Solve for A and B using least-squares
self.K = tf.matrix_solve_ls(X, Y, l2_regularizer=args.l2_regularizer)
self.A = self.K[:, :args.latent_dim]
self.B = self.K[:, args.latent_dim:]
# Perform least squares to find A-inverse
self.A_inv = tf.matrix_solve_ls(
Y - tf.matmul(self.u[:, :(args.seq_length - 1)], self.B),
self.g_vals[:, :-1],
l2_regularizer=args.l2_regularizer)
# Get predicted code at final time step
self.z_t = self.g_vals[:, -1]
# Create recursive predictions for z
z_t = tf.expand_dims(self.z_t, axis=1)
z_vals = [z_t]
for t in range(args.seq_length - 2, -1, -1):
u = self.u[:, t]
u = tf.expand_dims(u, axis=1)
z_t = tf.matmul(z_t - tf.matmul(u, self.B), self.A_inv)
z_vals.append(z_t)
self.z_vals_reshape = tf.stack(z_vals, axis=1)
# Flip order
self.z_vals_reshape = tf.squeeze(tf.reverse(self.z_vals_reshape, [1]))
# Reshape predicted z-values
self.z_vals = tf.reshape(
self.z_vals_reshape,
[args.batch_size * args.seq_length, args.latent_dim])
def solve(self, A, B):
"""
Solve a system of linear equations.
Finds X such that:
A X = B.
:param A: [BxMxM] TensorFlow Tensor.
:param B: [BxMxN] TensorFlow Tensor.
:return: X: [BxMxN] TensorFlow Tensor.
"""
return tf.matrix_solve_ls(matrix=A, rhs=B)
def _validation(embedding, batch_size, weight_name, bias_name):
with tf.variable_scope("embedding_inversion"):
embedding = tf.log(embedding) - tf.log(1 - embedding)
embedding_bias = tf.contrib.framework.\
get_variables_by_name(bias_name)
embedding_weight = tf.contrib.framework.\
get_variables_by_name(weight_name)
embedding_inverse_tensor = tf.matrix_solve_ls(
tf.transpose(tf.tile(embedding_weight, [batch_size, 1, 1]),
[0, 2, 1]),
tf.expand_dims(tf.subtract(embedding, embedding_bias), 2))
return tf.squeeze(embedding_inverse_tensor)
def normal():
with np.load("notMNIST.npz") as data:
Data, Target = data["images"], data["labels"]
posClass = 2
negClass = 9
dataIndx = (Target == posClass) + (Target == negClass)
Data = Data[dataIndx] / 255.
Target = Target[dataIndx].reshape(-1, 1)
Target[Target == posClass] = 1
Target[Target == negClass] = 0
np.random.seed(521)
randIndx = np.arange(len(Data))
np.random.shuffle(randIndx)
Data, Target = Data[randIndx], Target[randIndx]
trainData, trainTarget = Data[:3500], Target[:3500]
validData, validTarget = Data[3500:3600], Target[3500:3600]
testData, testTarget = Data[3600:], Target[3600:]
trainData = trainData.reshape((-1, 28 * 28)).astype(np.float32)
validData = validData.reshape((-1, 28 * 28)).astype(np.float32)
testData = testData.reshape((-1, 28 * 28)).astype(np.float32)
trainTarget_cp = copy.deepcopy(trainTarget)
trainTarget = trainTarget.astype(np.float32)
X = tf.Variable(trainData)
Y = tf.Variable(trainTarget)
weight = tf.matrix_solve_ls(X, Y, l2_regularizer=0.0, fast=True)
prediction = tf.matmul(trainData, weight)
#squared_diff_sum = tf.reduce_sum(tf.pow((prediction - Y), 2))
#training_MSE_loss = squared_diff_sum / (2.0*500.0)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
valid_pred = tf.matmul(validData, weight)
correct_num = tf.cast(tf.less(tf.abs(valid_pred - validTarget), 0.5),
tf.int32)
valid_accuracy = tf.reduce_mean(tf.cast(correct_num, tf.float32))
test_pred = tf.matmul(testData, weight)
correct_num = tf.cast(tf.less(tf.abs(test_pred - testTarget), 0.5),
tf.int32)
test_accuracy = tf.reduce_mean(tf.cast(correct_num, tf.float32))
train_pred = tf.matmul(trainData, weight)
correct_num = tf.cast(tf.less(tf.abs(train_pred - trainTarget), 0.5),
tf.int32)
train_accuracy = tf.reduce_mean(tf.cast(correct_num, tf.float32))
return sess.run(train_accuracy), sess.run(valid_accuracy), sess.run(
test_accuracy)
def _ir_solver(self, a, b, hu1, hu2):
x = tf.matrix_solve_ls(a,
b,
l2_regularizer=0.001,
fast=True,
name=None)
for i in range(3):
rec = tf.matmul(a, x)
r_e = tf.subtract(rec, b)
w = self._loss_function(r_e, hu1, hu2, w_gradient=True)
wa = tf.matmul(w, a)
wb = tf.matmul(w, b)
x = tf.matrix_solve_ls(wa,
wb,
l2_regularizer=0.001,
fast=True,
name=None)
rec_f = tf.matmul(a, x)
err_f = tf.abs(tf.subtract(rec_f, b))
error = self._loss_function(err_f, hu1, hu2)
return x, error
def solve(self, A, B):
"""
Solve a system of linear equations.
Finds X such that:
A X = B.
:param A: [BxMxM] TensorFlow Tensor.
:param B: [BxMxN] TensorFlow Tensor.
:return: X: [BxMxN] TensorFlow Tensor.
"""
A = tf.Print(A, [A], summarize=20)
B = tf.Print(B, [B], summarize=20)
return tf.matrix_solve_ls(matrix=A, rhs=B)
def solve_w_mean(self, memory_sample: tf.Tensor,
z: tf.Tensor) -> tf.Tensor:
if self.use_addressing_matrix:
raise NotImplementedError
w_matrix = tf.matmul(memory_sample, memory_sample, transpose_b=True)
w_rhs = tf.einsum("bmc,ebc->bme", memory_sample, z)
l2_regularizer = tf.square(
self._observational_noise_stddev) + tf.square(self._w_prior_stddev)
w_mean = tf.matrix_solve_ls(matrix=w_matrix,
rhs=w_rhs,
l2_regularizer=l2_regularizer)
w_mean = tf.transpose(w_mean, perm=[2, 0, 1], name="w_mean")
return w_mean
def solve_ls(A, B):
# Transpose input matrices
At = tf.transpose(A, [0, 2, 1])
Bt = tf.transpose(B, [0, 2, 1])
# Solve C via least-squares
Ct_est = tf.matrix_solve_ls(At, Bt)
#Ct_est = tf.matrix_solve_ls(At, Bt, l2_regularizer = 0.000001)
C_est = tf.transpose(Ct_est, [0, 2, 1], name='C_est')
# Calculate error for safeguarding
safeguard_inverse = tf.nn.l2_loss(tf.matmul(At, Ct_est) - Bt)
safeguard_inverse /= tf.to_float(tf.reduce_prod(tf.shape(A)))
return C_est, safeguard_inverse
def Phase_ini(g_y, B, M, N, y_abs, g_H):
delta = M / N
yy = np.reshape(np.reshape(y_abs, -1), [B, M])
yy_ = yy**2
ymean = np.mean(yy_, 1, keepdims=True)
yy1 = yy_ / ymean
Ta = (yy1 - 1) / (yy1 + np.sqrt(delta) - 1)
Tb = Ta * ymean
g_2 = np.reshape(g_H, [B, M, N])
x2_M_c = np.zeros([B, 2 * N])
for i in range(B):
def fn(xin):
return np.matmul((g_2[i, :, :].conj().T),
Tb[i, :] * np.matmul(g_2[i, :, :], xin)) / M
AA = LinearOperator((N, N), matvec=fn)
[eig_v, xin2] = eigs(AA, 1, which='LR', maxiter=1e3)
abs2_y = tf.concat([yy[i], yy[i]], 0) / np.sqrt(2)
xin2r = np.vstack((np.real(xin2), np.imag(xin2)))
Hr = np.vstack(
(np.hstack((np.real(g_2[i, :, :]), -np.imag(g_2[i, :, :]))),
np.hstack((np.imag(g_2[i, :, :]), np.real(g_2[i, :, :])))))
u11 = np.matmul(Hr, xin2r)
u_var_c = np.sqrt(np.square(u11[0:M, :]) +
np.square(u11[M:2 * M, :])) / np.sqrt(2)
u_var_r = tf.concat([u_var_c, u_var_c], 0)
u = tf.multiply(u_var_r[:, 0], abs2_y[:])
l = tf.multiply(u_var_r[:, 0], u_var_r[:, 0])
s = tf.divide(tf.norm(u, axis=0, ord=2), tf.norm(l, axis=0, ord=2))
x2 = tf.multiply(xin2r, s)
for iteration in range(3):
z_ob = tf.matmul(Hr, x2)
zhat_var_c = tf.sqrt(
tf.square(z_ob[0:M, :]) +
tf.square(z_ob[M:2 * M, :])) / np.sqrt(2)
zhat_var_r = tf.concat([zhat_var_c, zhat_var_c], 0)
z_abs = tf.expand_dims(
tf.multiply(abs2_y[:], tf.divide(z_ob[:, 0],
zhat_var_r[:, 0])), -1)
x2 = tf.matrix_solve_ls(matrix=Hr, rhs=z_abs)
with tf.Session() as sess:
x2_M_c[i, :] = sess.run(x2[:, 0])
return x2_M_c
def ica_reg(X, Y, alpha=1.0, lamda=1.0, ica_obj=tf_ica_obj):
p, n = X.shape
p_, r = Y.shape
if p != p_:
raise ValueError('X and Y must have the same number of rows.')
pca = PCA(whiten=True)
X_w = pca.fit_transform(X.T).T
D = pca.components_
P = np.eye(p) - np.ones(p)/p
Y_tilde = tf.constant(np.linalg.pinv(P.dot(np.linalg.pinv(D))).dot(P.dot(Y)), dtype=tf.float32)
#D_inv = tf.constant(np.linalg.pinv(pca.components_), dtype=tf.float32)
W = tf.placeholder(tf.float32, shape=[p, p])
#B = tf.placeholder(tf.float32, shape=[p, r])
#b = tf.placeholder(tf.float32, shape=[r])
B = tf.matmul(W, Y_tilde)
B_err = tf.placeholder(tf.float32, shape=[p, r])
W_ortho = tf_orthogonalize(W)
B_err_null = tf.matmul(B, tf.matrix_solve_ls(B, B_err))
#A = tf.matmul(D_inv, tf.transpose(W))
#Y_hat = tf.matmul(A, B) + tf.tile(tf.reshape(b, [1, r]), [p, 1])
J_ica = ica_obj(W, tf.constant(X_w, dtype=tf.float32))
#J_regression = 0.5*tf_l2_norm(Y - Y_hat)
J_sparse = tf_l1_norm(B + B_err)
#J = J_ica + alpha*J_regression + lamda*J_sparse
J = J_ica + lamda*J_sparse
with tf.Session() as sess:
bd_opt = BoldDriverOptimizer(sess, J, [W, B_err], \
[gen_orthonormal(p), np.zeros_like(Y)], \
[W_ortho, B_err_null])
while not bd_opt.converged:
bd_opt.run()
print(bd_opt.f)
_W, _B_err = bd_opt.x
feed_dict = {W: _W, B_err: _B_err}
_B = (B + B_err).eval(feed_dict=feed_dict)
print(sess.run([J_ica, J_sparse], feed_dict=feed_dict))
S = _W.dot(X_w)
return S, _W, _B
def random_transform(self, batch_size):
if self._transform is None:
corners_ = self.get_corners()
_batch_ones = tf.ones([batch_size, len(corners_[0]), 1])
corners = tf.tile(corners_, [batch_size, 1, 1])
random_size = [batch_size, len(corners_[0]), len(corners_[0][0])]
random_scale = tf.random_uniform(random_size, 0, self.scale)
corners2 = corners * (1 - random_scale)
corners_homog = tf.concat([corners, _batch_ones], 2)
corners2_homog = tf.concat([corners2, _batch_ones], 2)
_transform = tf.matrix_solve_ls(corners_homog, corners2_homog)
self._transform = tf.transpose(_transform, [0, 2, 1])
return self._transform
def newton_solver(con, var):
# shape info
var_shp = [x.get_shape() for x in var]
# derivatives
g = flatify(con)
G = jacobian(g, var)
# can be non-square so use least squares
U = squeeze(tf.matrix_solve_ls(T(G), -g[:, None], fast=False))
# updates
diffs = unpack(U, var_shp)
# operators
upds = increment(var, diffs)
return upds
def _verifySolve(self, x, y):
for np_type in [np.float32, np.float64]:
a = x.astype(np_type)
b = y.astype(np_type)
np_ans, _, _, _ = np.linalg.lstsq(a, b)
for fast in [True, False]:
with self.test_session():
tf_ans = tf.matrix_solve_ls(a, b, fast=fast)
ans = tf_ans.eval()
self.assertEqual(np_ans.shape, tf_ans.get_shape())
self.assertEqual(np_ans.shape, ans.shape)
# Check residual norm.
tf_r = b - BatchMatMul(a, ans)
tf_r_norm = np.sum(tf_r * tf_r)
np_r = b - BatchMatMul(a, np_ans)
np_r_norm = np.sum(np_r * np_r)
self.assertAllClose(np_r_norm, tf_r_norm)
# Check solution.
self.assertAllClose(np_ans, ans, atol=1e-5, rtol=1e-5)
def constrained_gradient_descent(obj, con, var, step=0.1):
# shape info
var_shp = [x.get_shape() for x in var]
# derivatives
g = flatify(con)
F = jacobian(obj, var)
G = jacobian(g, var)
# constrained gradient descent
L = tf.matrix_solve(
tf.matmul(T(G), G),
-tf.matmul(T(G), F)
)
Ugd = step*squeeze(F + tf.matmul(G, L))
# correction step (zangwill-garcia)
# can be non-square so use least squares
Ugz = squeeze(tf.matrix_solve_ls(
tf.concat([T(G), Ugd[None, :]], 0),
-tf.concat([g[:, None], [[0.0]]], 0),
fast=False
))
# updates
gd_diffs = unpack(Ugd, var_shp)
gz_diffs = unpack(Ugz, var_shp)
# operators
gd_upds = increment(var, gd_diffs)
gz_upds = increment(var, gz_diffs)
# slope
gain = tf.squeeze(tf.matmul(Ugd[None, :], F))
return gd_upds, gz_upds, gain
def fit_one_step(
model_matrix,
response,
model,
model_coefficients_start=None,
predicted_linear_response_start=None,
l2_regularizer=None,
dispersion=None,
offset=None,
learning_rate=None,
fast_unsafe_numerics=True,
name=None):
"""Runs one step of Fisher scoring.
Args:
model_matrix: (Batch of) `float`-like, matrix-shaped `Tensor` where each row
represents a sample's features.
response: (Batch of) vector-shaped `Tensor` where each element represents a
sample's observed response (to the corresponding row of features). Must
have same `dtype` as `model_matrix`.
model: `tfp.glm.ExponentialFamily`-like instance used to construct the
negative log-likelihood loss, gradient, and expected Hessian (i.e., the
Fisher information matrix).
model_coefficients_start: Optional (batch of) vector-shaped `Tensor`
representing the initial model coefficients, one for each column in
`model_matrix`. Must have same `dtype` as `model_matrix`.
Default value: Zeros.
predicted_linear_response_start: Optional `Tensor` with `shape`, `dtype`
matching `response`; represents `offset` shifted initial linear
predictions based on `model_coefficients_start`.
Default value: `offset` if `model_coefficients is None`, and
`tf.linalg.matvec(model_matrix, model_coefficients_start) + offset`
otherwise.
l2_regularizer: Optional scalar `Tensor` representing L2 regularization
penalty, i.e.,
`loss(w) = sum{-log p(y[i]|x[i],w) : i=1..n} + l2_regularizer ||w||_2^2`.
Default value: `None` (i.e., no L2 regularization).
dispersion: Optional (batch of) `Tensor` representing `response` dispersion,
i.e., as in, `p(y|theta) := exp((y theta - A(theta)) / dispersion)`.
Must broadcast with rows of `model_matrix`.
Default value: `None` (i.e., "no dispersion").
offset: Optional `Tensor` representing constant shift applied to
`predicted_linear_response`. Must broadcast to `response`.
Default value: `None` (i.e., `tf.zeros_like(response)`).
learning_rate: Optional (batch of) scalar `Tensor` used to dampen iterative
progress. Typically only needed if optimization diverges, should be no
larger than `1` and typically very close to `1`.
Default value: `None` (i.e., `1`).
fast_unsafe_numerics: Optional Python `bool` indicating if solve should be
based on Cholesky or QR decomposition.
Default value: `True` (i.e., "prefer speed via Cholesky decomposition").
name: Python `str` used as name prefix to ops created by this function.
Default value: `"fit_one_step"`.
Returns:
model_coefficients: (Batch of) vector-shaped `Tensor`; represents the
next estimate of the model coefficients, one for each column in
`model_matrix`.
predicted_linear_response: `response`-shaped `Tensor` representing linear
predictions based on new `model_coefficients`, i.e.,
`tf.linalg.matvec(model_matrix, model_coefficients_next) + offset`.
"""
graph_deps = [model_matrix, response, model_coefficients_start,
predicted_linear_response_start, dispersion, learning_rate]
with tf.name_scope(name, 'fit_one_step', graph_deps):
[
model_matrix,
response,
model_coefficients_start,
predicted_linear_response_start,
offset,
] = prepare_args(
model_matrix,
response,
model_coefficients_start,
predicted_linear_response_start,
offset)
# Compute: mean, grad(mean, predicted_linear_response_start), and variance.
mean, variance, grad_mean = model(predicted_linear_response_start)
# If either `grad_mean` or `variance is non-finite or zero, then we'll
# replace it with a value such that the row is zeroed out. Although this
# procedure may seem circuitous, it is necessary to ensure this algorithm is
# itself differentiable.
is_valid = (tf.is_finite(grad_mean) & tf.not_equal(grad_mean, 0.) &
tf.is_finite(variance) & (variance > 0.))
def mask_if_invalid(x, mask):
mask = tf.fill(tf.shape(x), value=np.array(mask, x.dtype.as_numpy_dtype))
return tf.where(is_valid, x, mask)
# Run one step of iteratively reweighted least-squares.
# Compute "`z`", the adjusted predicted linear response.
# z = predicted_linear_response_start
# + learning_rate * (response - mean) / grad_mean
z = (response - mean) / mask_if_invalid(grad_mean, 1.)
# TODO(jvdillon): Rather than use learning rate, we should consider using
# backtracking line search.
if learning_rate is not None:
z *= learning_rate[..., tf.newaxis]
z += predicted_linear_response_start
# Compute "`w`", the per-sample weight.
if dispersion is not None:
# For convenience, we'll now scale the variance by the dispersion factor.
variance *= dispersion
w = (mask_if_invalid(grad_mean, 0.) *
tf.rsqrt(mask_if_invalid(variance, np.inf)))
a = model_matrix * w[..., tf.newaxis]
b = z * w
# Solve `min{ || A @ model_coefficients - b ||_2**2 : model_coefficients }`
# where `@` denotes `matmul`.
if l2_regularizer is None:
l2_regularizer = np.array(0, a.dtype.as_numpy_dtype)
else:
l2_regularizer_ = distribution_util.maybe_get_static_value(
l2_regularizer, a.dtype.as_numpy_dtype)
if l2_regularizer_ is not None:
l2_regularizer = l2_regularizer_
def _embed_l2_regularization():
"""Adds synthetic observations to implement L2 regularization."""
# `tf.matrix_solve_ls` does not respect the `l2_regularization` argument
# when `fast_unsafe_numerics` is `False`. This function adds synthetic
# observations to the data to implement the regularization instead.
# Adding observations `sqrt(l2_regularizer) * I` is mathematically
# equivalent to adding the term
# `-l2_regularizer ||coefficients||_2**2` to the log-likelihood.
num_model_coefficients = num_cols(model_matrix)
batch_shape = tf.shape(model_matrix)[:-2]
eye = tf.eye(
num_model_coefficients, batch_shape=batch_shape, dtype=a.dtype)
a_ = tf.concat([a, tf.sqrt(l2_regularizer) * eye], axis=-2)
b_ = distribution_util.pad(
b, count=num_model_coefficients, axis=-1, back=True)
# Return l2_regularizer=0 since its now embedded.
l2_regularizer_ = np.array(0, a.dtype.as_numpy_dtype)
return a_, b_, l2_regularizer_
a, b, l2_regularizer = tf.contrib.framework.smart_cond(
smart_reduce_all([not(fast_unsafe_numerics),
l2_regularizer > 0.]),
_embed_l2_regularization,
lambda: (a, b, l2_regularizer))
model_coefficients_next = tf.matrix_solve_ls(
a, b[..., tf.newaxis],
fast=fast_unsafe_numerics,
l2_regularizer=l2_regularizer,
name='model_coefficients_next')
model_coefficients_next = model_coefficients_next[..., 0]
# TODO(b/79122261): The approach used in `matrix_solve_ls` could be made
# faster by avoiding explicitly forming Q and instead keeping the
# factorization in 'implicit' form with stacked (rescaled) Householder
# vectors underneath the 'R' and then applying the (accumulated)
# reflectors in the appropriate order to apply Q'. However, we don't
# presently do this because we lack core TF functionality. For reference,
# the vanilla QR approach is:
# q, r = tf.linalg.qr(a)
# c = tf.matmul(q, b, adjoint_a=True)
# model_coefficients_next = tf.matrix_triangular_solve(
# r, c, lower=False, name='model_coefficients_next')
predicted_linear_response_next = calculate_linear_predictor(
model_matrix,
model_coefficients_next,
offset,
name='predicted_linear_response_next')
return model_coefficients_next, predicted_linear_response_next
def test_MatrixSolveLs(self):
t = tf.matrix_solve_ls(*self.random((3, 3), (3, 1)))
self.check(t)
for extra in [(), (2,), (3,)] + [(3, 2)] * (size < 10):
shape = extra + (size, size)
name = '%s_%s' % (dtype.__name__, '_'.join(map(str, shape)))
_AddTest(MatrixUnaryFunctorGradientTest, 'MatrixInverseGradient', name,
_GetMatrixUnaryFunctorGradientTest(tf.matrix_inverse, dtype,
shape))
_AddTest(MatrixUnaryFunctorGradientTest, 'MatrixDeterminantGradient',
name,
_GetMatrixUnaryFunctorGradientTest(tf.matrix_determinant,
dtype, shape))
# Tests for gradients of matrix_solve_ls
for dtype in np.float32, np.float64:
for rows in 2, 5, 10:
for cols in 2, 5, 10:
for l2_regularization in 0.0, 0.001, 1.0:
shape = (rows, cols)
name = '%s_%s_%s' % (dtype.__name__, '_'.join(map(str, shape)),
l2_regularization)
_AddTest(
MatrixBinaryFunctorGradientTest,
'MatrixSolveLsGradient',
name,
_GetMatrixBinaryFunctorGradientTest(
lambda a, b, l=l2_regularization: tf.matrix_solve_ls(a, b, l),
dtype,
shape,
float32_tol_fudge=4.0))
tf.test.main()
