def testMainReadme(self):
# Just check that the readme examples do not raise exceptions.
# Create a random tensor of shape (3, 2, 2).
a = t3f.random_tensor((3, 2, 2), tt_rank=3)
norm = t3f.frobenius_norm(a)
# Convert TT-tensor into a dense tensor for printing.
a_full = t3f.full(a)
# Run a tensorflow session to run the operations.
with tf.Session() as sess:
# Run the operations. Note that if you run these
# two operations separetly (sess.run(a_full), sess.run(norm))
# the result will be different, since sess.run will
# generate a new random tensor a on each run because `a' is
# an operation 'generate me a random tensor'.
a_val, norm_val = sess.run([a_full, norm])
a = t3f.random_tensor((3, 2, 2), tt_rank=3)
b_dense = tf.random_normal((3, 2, 2))
# Use TT-SVD on b_dense.
b_tt = t3f.to_tt_tensor(b_dense, max_tt_rank=4)
sum_round = t3f.round(t3f.add(a, b_tt), max_tt_rank=2)
# Inner product (sum of products of all elements).
a = t3f.random_tensor((3, 2, 2), tt_rank=3)
b = t3f.random_tensor((3, 2, 2), tt_rank=4)
inner_prod = t3f.flat_inner(a, b)
A = t3f.random_matrix(((3, 2, 2), (2, 3, 3)), tt_rank=3)
b = t3f.random_matrix(((2, 3, 3), None), tt_rank=3)
# Matrix-by-vector
matvec = t3f.matmul(A, b)
# Matrix-by-dense matrix
b_dense = tf.random_normal((18, 1))
matvec2 = t3f.matmul(A, b_dense)
def call(self, x):
if self.bias:
h = t3f.matmul(x, self.W) + self.b
else:
h = t3f.matmul(x, self.W)
if self.activation is not None:
if self.activation in activations:
h = Activation(self.activation)(h)
else:
raise ValueError('Unknown activation "%s", only %s and None '
'are supported' %
(self.activation, activations))
return h
def call(self, x):
res = t3f.matmul(x, self.matrix)
if self.use_bias:
res += self.b
if self.activation is not None:
res = Activation(self.activation)(res)
return res
def LQ(niters, batch_size, lr, rank, X_data, Y_data, model):
tf.reset_default_graph()
sess = tf.Session()
losses = []
if model == 'riemt3f':
X = tf.placeholder(tf.float32, [None, dx])
Y = tf.placeholder(tf.float32, [None, dy])
#### T3F RGD ####
print('Starting T3F RGD...')
# using riemannian projection implemented by t3f, compute a separate update
# this requires projection/rounding which should be computationally intensive
initializer = t3f.glorot_initializer([nx, ny], tt_rank=rank)
W_t3f_rgd = t3f.get_variable('W_t3f_rgd', initializer=initializer)
cost_t3f_rgd = tf.reduce_mean(0.5 *
tf.square(Y - t3f.matmul(X, W_t3f_rgd)))
# least squares derivative
grad = t3f.to_tt_matrix(tf.matmul(
tf.transpose(Y - t3f.matmul(X, W_t3f_rgd)), -1 * X),
shape=[nx, ny],
max_tt_rank=rank)
riemannian_grad = t3f.riemannian.project(grad, W_t3f_rgd)
# norm_t3f_rgd = t3f.frobenius_norm(riemannian_grad, epsilon=1e-10)
### HARD CODED SLOWER RATE HERE BC OF DIVERGENCE
train_step = t3f.assign(
W_t3f_rgd,
t3f.round(W_t3f_rgd - 0.1 * lr * riemannian_grad,
max_tt_rank=rank))
sess.run(tf.global_variables_initializer())
nparams = np.sum([
np.prod(v.get_shape().as_list()) for v in tf.trainable_variables()
])
print('Total number of parameters: ', nparams)
t0 = time.time()
# while(sess.run(tf.less(mingradnorm, norm_t3f_rgd), feed_dict={X: X_data, Y: Y_data})):
i = 0
while (i <= niters):
i = i + 1
x_mb, y_mb = next_batch(X_data, Y_data, batch_size)
_, tmp = sess.run([train_step.op, cost_t3f_rgd],
feed_dict={
X: x_mb,
Y: y_mb
})
losses.append(tmp)
# print(sess.run(norm_t3f_rgd, feed_dict={X: X_data, Y: Y_data}))
print(i, tmp)
if tmp < mincost or np.isnan(tmp):
break
t1 = time.time()
myT = t1 - t0
elif model == 'ott':
X = tf.placeholder(tf.float32, [None, dx])
Y = tf.placeholder(tf.float32, [None, dy])
# #### Own EOTT GD ####
print('Starting OTT...')
W_EOTT_gd = aOTTtfVariable(shape=[ny, nx], r=rank)
cost_eott_gd = tf.reduce_mean(
0.5 * tf.square(Y - tf.transpose(W_EOTT_gd.mult(tf.transpose(X)))))
opt = tf.train.GradientDescentOptimizer(learning_rate=1.0)
# Manifold Update
gW1 = opt.compute_gradients(cost_eott_gd, W_EOTT_gd.getQ())
man_update = [v.assign(gradStep(X=v, G=g, lr=lr)) for g, v in gW1]
t0 = time.time()
sess.run(tf.global_variables_initializer())
nparams = np.sum([
np.prod(v.get_shape().as_list()) for v in tf.trainable_variables()
])
print('Total number of parameters: ', nparams)
wopt = sess.run(W_EOTT_gd.getQ())
i = 0
while (i <= niters):
i = i + 1
x_mb, y_mb = next_batch(X_data, Y_data, batch_size)
_, tmp = sess.run([man_update, cost_eott_gd],
feed_dict={
X: x_mb,
Y: y_mb
})
# _, tmp = sess.run([Eucupdate, cost_eott_gd], feed_dict={X: x_mb, Y: y_mb})
losses.append(tmp)
print(i, tmp)
if tmp < mincost or np.isnan(tmp):
break
t1 = time.time()
myT = t1 - t0
else:
print('what model is that? unknown')
return
t1 = time.time()
print('Took seconds:', myT)
return myT, losses
one_matrix = t3f.get_variable('one_matrix', initializer=matrices[0])
matrices = t3f.get_variable('matrices', initializer=matrices)
vecs = t3f.random_matrix_batch((shape, None), 10, batch_size=100)
vecs = t3f.cast(vecs, tf.float64)
one_vec = t3f.get_variable('one_vec', initializer=vecs[0])
vecs = t3f.get_variable('vecs', initializer=vecs)
vecs100 = t3f.random_matrix_batch((shape, None), 100, batch_size=100)
vecs100 = t3f.cast(vecs100, tf.float64)
one_vec100 = t3f.get_variable('one_vec100', initializer=vecs100[0])
vecs100 = t3f.get_variable('vecs100', initializer=vecs100)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
print(device_lib.list_local_devices())
logs = {}
matvec_op = t3f.matmul(one_matrix, one_vec).op
# Warmup.
timeit.timeit(lambda: sess.run(matvec_op), number=10)
matvec_time = timeit.timeit(lambda: sess.run(matvec_op), number=1000) / 1000
print('Multiplying %s by %s takes %f seconds.' %
(one_matrix, one_vec, matvec_time))
logs['matvec_time'] = matvec_time
batch_matvec_op = t3f.matmul(one_matrix, vecs).op
batch_matvec_time = timeit.timeit(lambda: sess.run(batch_matvec_op),
number=100) / 100
print('Multiplying %s by %s takes %f seconds.' %
(one_matrix, vecs, batch_matvec_time))
logs['batch_matvec_time'] = batch_matvec_time
matmul_op = t3f.matmul(one_matrix, one_matrix).op
def matmul(self, input_data):
activation = t3f.matmul(input_data, self.weights)
if self.use_bias:
activation = activation + self.bias
return activation