def test_learning_factorization(verbose=False):
n = 9
m = 8
y_mat = toy_factorization_problem(n=n, m=m, rk=3, noise=1)
rank = 2
batch_size = n * m
tuples = mat2tuples(y_mat)
tuple_iterable = data_to_batches(tuples, minibatch_size=batch_size)
# tuple_iterable = positive_and_negative_tuple_sampler(mat2tuples(y_mat), minibatch_size=batch_size)
sampler, (x, y) = feed_dict_sampler(tuple_iterable, types=[np.int64, np.float32])
loss_op = tf.reduce_mean(loss_func(multilinear_tuple_scorer(x, rank=rank, n_emb=n+m)[0], y, 'quadratic'))
initial_losses = [tf_eval(loss_op, f) for _, f in sampler]
if verbose:
print(initial_losses)
# hooks = [lambda s, e, it, l: it and ((it % 100) == 0) and print("%d) loss=%f" % (it, l))]
hooks = [lambda it, b, l: it and ((it % 1) == 0) and print("{0}) train loss={1}".format(it, l[0]))]
emb, = learn(loss_op, sampler, tf.train.AdamOptimizer(learning_rate=0.1), hooks, max_epochs=200)
# emb, = learn(l, sampler, tf.train.GradientDescentOptimizer(learning_rate=0.5), hooks, max_epochs=500)
# emb, = learn(l, sampler, tf.train.AdagradOptimizer(0.01, initial_accumulator_value=0.01), hooks, max_epochs=500)
mat0 = svd_factorize_matrix(y_mat, rank=2) # exact svd solution
mat = emb[:n, :].dot(emb[n:, :].T)
if verbose:
print(np.linalg.norm(mat - mat0))
assert(np.linalg.norm(mat - mat0) < 1e-3)
def test_learning_factorization(verbose=False):
n = 9
m = 8
y_mat = toy_factorization_problem(n=n, m=m, rk=3, noise=1)
rank = 2
batch_size = n * m
tuples = mat2tuples(y_mat)
tuple_iterable = data_to_batches(tuples, minibatch_size=batch_size)
# tuple_iterable = positive_and_negative_tuple_sampler(mat2tuples(y_mat), minibatch_size=batch_size)
sampler, (x, y) = feed_dict_sampler(tuple_iterable,
types=[np.int64, np.float32])
loss_op = tf.reduce_mean(
loss_func(
multilinear_tuple_scorer(x, rank=rank, n_emb=n + m)[0], y,
'quadratic'))
initial_losses = [tf_eval(loss_op, f) for _, f in sampler]
if verbose:
print(initial_losses)
# hooks = [lambda s, e, it, l: it and ((it % 100) == 0) and print("%d) loss=%f" % (it, l))]
hooks = [
lambda it, b, l: it and (
(it % 1) == 0) and print("{0}) train loss={1}".format(it, l[0]))
]
emb, = learn(loss_op,
sampler,
tf.train.AdamOptimizer(learning_rate=0.1),
hooks,
max_epochs=200)
# emb, = learn(l, sampler, tf.train.GradientDescentOptimizer(learning_rate=0.5), hooks, max_epochs=500)
# emb, = learn(l, sampler, tf.train.AdagradOptimizer(0.01, initial_accumulator_value=0.01), hooks, max_epochs=500)
mat0 = svd_factorize_matrix(y_mat, rank=2) # exact svd solution
mat = emb[:n, :].dot(emb[n:, :].T)
if verbose:
print(np.linalg.norm(mat - mat0))
assert (np.linalg.norm(mat - mat0) < 1e-3)
def factorize_tuples(tuples, rank=2, arity=None, minibatch_size=100, n_iter=1000, eval_freq=100,
loss_types=('quadratic',),
negative_prop=0.0, n_emb=None,
minibatch_generator=None, verbose=True,
scoring=None, negative_sample=False, tf_optim=None, emb0=None, n_ent = None,
bigram = False, dictionaries = None):
"""
Factorize a knowledge base using a TensorFlow model
:param tuples: list of tuples representing a knowledge base. Types can be used
:param scoring: TensorFlow operator accepting 2 tensors as input and one as output ():
- input 1 is the embedding tensor. It has size [n_ent, rank] and contains float values
- input 2 is a tensor containing tuples of integer from a minibatch. It has size [minibatch_size, order]
and values range from 0 to n_ent-1
- output is a list of continuous values
:param tf_optim: TensorFlow model performing the optimization
:return: embeddings
Note about sparse_hermitian_product:
To recover the predictions, you must average the real and imaginary part because it is learn using this formula.
See the sparse_hermitian_product function with the 'real' default option. This is simpler in the real case when we
use the multilinear function: we would replace hermitian_dot by dot(emb, emb.transpose())
# demo of a rectangular matrix factorization with square loss:
>>> y_mat = toy_factorization_problem(n=7, m=6, rk=4, noise=1)
>>> emb0 = svd_factorize_matrix(y_mat, rank=4) # exact svd solution
>>> u2 = factorize_tuples(mat2tuples(y_mat), 4, emb0=emb0, n_iter=500, verbose=False)[0]
>>> x_mat_est2 = np.dot(u2[:7], u2[7:].T) # the initial matrix
>>> np.linalg.norm(emb0[:7].dot(emb0[7:].T)-x_mat_est2) < 1e-3
True
# demo of a symmetric square matrix factorization with square loss:
# >>> n = 5
# >>> mat = random_symmetric_real_matrix(n, rank=4)
# >>> eig_sol = eig_approx(mat, rank=2)
# >>> embeddings = factorize_tuples(mat2tuples(mat, common_types=True), rank=2, verbose=False, n_iter=100)
# >>> h = hermitian_dot(embeddings, embeddings)
# >>> sgd_sol = 0.5 * (h[0] + h[1])
# >>> np.linalg.norm(eig_sol - sgd_sol)<1e3
# True
"""
if isinstance(tuples, tuple) and len(tuples) == 2:
warnings.warn('Providing tuples as (inputs, outputs) is deprecated. '
'Use [(input_1, output_1), ..., (input_n, output_n)] instead'
'You should provide them as zip(inputs, outputs)')
tuples = [x for x in zip(tuples[0], tuples[1])]
if scoring is None:
if n_emb is None:
n_emb = np.max([np.max(x) for x, y in tuples]) + 1
# the TensorFlow optimizer
tf_optim = tf_optim if tf_optim is not None else tf.train.AdamOptimizer(learning_rate=0.1)
if isinstance(tuples, list):
inputs, outputs, minibatch_generator = simple_tuple_generator(tuples, minibatch_size, n_iter, eval_freq,
negative_prop, n_ent, bigram, dictionaries)
# the scoring function is usually a dot product between embeddings
if scoring is None:
preds, params = multilinear_tuple_scorer(inputs, rank=rank, n_emb=n_emb, emb0=emb0)
#preds, params = multilinear_tuple_scorer(inputs, rank=rank, n_emb=n_emb, emb0=emb0)
# elif scoring == generalised_multilinear_dot_product_scorer: # commented because it can be done externally
# preds, params = scoring(inputs, rank=rank, n_emb=n_emb, emb0=emb0,
# norm_scalers=norm_scalers)
else:
preds, params = scoring(inputs, rank=rank, n_emb=n_emb, emb0=emb0)
# Minimize the loss
loss_ops = {}
train_ops = {}
for m in loss_types:
loss_ops[m] = tf.reduce_mean(loss_func(preds, outputs, m))
train_ops[m] = tf_optim.minimize(loss_ops[m])
# Launch the graph.
with tf.Session() as sess: # we close the session at the end of the training
sess.run(tf.initialize_all_variables())
for epoch, eval_step, (minibatch_inputs, minibatch_outputs, minibatch_type) in minibatch_generator():
feed = {inputs: minibatch_inputs, outputs: minibatch_outputs}
if eval_step:
train_losses = {}
for m in loss_types:
_, train_losses[m] = sess.run([train_ops[m], loss_ops[m]], feed_dict=feed)
if verbose:
print(epoch, train_losses)
else:
sess.run(train_ops[minibatch_type], feed_dict=feed)
final_params = sess.run(params)
return final_params
