def loss(self,
train_labels,
train_indices,
negative_labels,
embeddings=None):
if embeddings is None:
embeddings = self.V
embeddings = tf.nn.embedding_lookup(embeddings, train_indices)
# average
context_embs = tf.reduce_mean(embeddings, axis=1)
pos_embs = tf.nn.embedding_lookup(embeddings, train_labels)
neg_embs = tf.nn.embedding_lookup(embeddings, negative_labels)
score_pos = tf.multiply(context_embs, pos_embs)
score_neg = tf.multiply(context_embs, neg_embs)
ranking_loss = max_margin(score_pos, score_neg)
return ranking_loss
def create_graph(self):
print('Building Model')
# Translation Model initialisation
w_bound = np.sqrt(6. / self.embedding_size)
self.E = tf.Variable(tf.random_uniform(
(self.num_entities, self.embedding_size),
minval=-w_bound,
maxval=w_bound),
name="E")
self.R = tf.Variable(tf.random_uniform(
(self.num_relations, self.embedding_size),
minval=-w_bound,
maxval=w_bound),
name="R")
self.normalize_E = self.E.assign(tf.nn.l2_normalize(self.E, 1))
self.inpr = tf.placeholder(tf.int32, [self.batch_size_kg], name="rhs")
self.inpl = tf.placeholder(tf.int32, [self.batch_size_kg], name="lhs")
self.inpo = tf.placeholder(tf.int32, [self.batch_size_kg], name="rell")
self.inprn = tf.placeholder(tf.int32, [self.batch_size_kg],
name="rhsn")
self.inpln = tf.placeholder(tf.int32, [self.batch_size_kg],
name="lhsn")
self.inpon = tf.placeholder(tf.int32, [self.batch_size_kg],
name="relln")
lhs = tf.nn.embedding_lookup(self.E, self.inpl)
rhs = tf.nn.embedding_lookup(self.E, self.inpr)
rell = tf.nn.embedding_lookup(self.R, self.inpo)
lhsn = tf.nn.embedding_lookup(self.E, self.inpln)
rhsn = tf.nn.embedding_lookup(self.E, self.inprn)
relln = tf.nn.embedding_lookup(self.R, self.inpon)
if self.event_layer is not None:
self.event_layer.create_graph()
if not self.event_layer.shared:
# Option 1
# self.a = tf.Variable(tf.random_uniform([self.embedding_size], minval=-w_bound,
# maxval=w_bound), name="a")
# self.b = tf.Variable(tf.random_uniform([self.embedding_size], minval=-w_bound,
# maxval=w_bound), name="b")
# lhs = tf.multiply(self.a, lhs) + tf.multiply(self.b, tf.nn.embedding_lookup(self.event_layer.V,
# self.inpl))
# rhs = tf.multiply(self.a, rhs) + tf.multiply(self.b, tf.nn.embedding_lookup(self.event_layer.V,
# self.inpr))
#
# lhsn = tf.multiply(self.a, lhsn) + tf.multiply(self.b, tf.nn.embedding_lookup(self.event_layer.V,
# self.inpln))
# rhsn = tf.multiply(self.a, rhsn) + tf.multiply(self.b, tf.nn.embedding_lookup(self.event_layer.V,
# self.inprn))
# Option 2
self.a = tf.Variable(tf.random_uniform(
[self.num_relations, self.embedding_size],
minval=-w_bound,
maxval=w_bound),
name="a")
self.b = tf.Variable(tf.random_uniform(
[self.num_relations, self.embedding_size],
minval=-w_bound,
maxval=w_bound),
name="b")
a_r = tf.nn.embedding_lookup(self.a, self.inpo)
b_r = tf.nn.embedding_lookup(self.b, self.inpo)
lhs = tf.multiply(a_r, lhs) + tf.multiply(
b_r, tf.nn.embedding_lookup(self.event_layer.V, self.inpl))
rhs = tf.multiply(a_r, rhs) + tf.multiply(
b_r, tf.nn.embedding_lookup(self.event_layer.V, self.inpr))
lhsn = tf.multiply(a_r, lhsn) + tf.multiply(
b_r, tf.nn.embedding_lookup(self.event_layer.V,
self.inpln))
rhsn = tf.multiply(a_r, rhsn) + tf.multiply(
b_r, tf.nn.embedding_lookup(self.event_layer.V,
self.inprn))
# more flexible connections
if self.fnsim == dot_similarity:
simi = tf.diag_part(
self.fnsim(self.leftop(lhs, rell),
tf.transpose(self.rightop(rhs, rell)),
broadcast=False))
simin = tf.diag_part(
self.fnsim(self.leftop(lhsn, rell),
tf.transpose(self.rightop(rhsn, rell)),
broadcast=False))
else:
simi = self.fnsim(self.leftop(lhs, rell),
self.rightop(rhs, rell),
broadcast=False)
simin = self.fnsim(self.leftop(lhsn, relln),
self.rightop(rhsn, relln),
broadcast=False)
kg_loss = max_margin(simi, simin)
self.loss = kg_loss
if self.event_layer is not None:
if type(self.event_layer) == Skipgram:
self.train_inputs = tf.placeholder(tf.int32,
shape=[self.batch_size_sg])
else:
self.train_inputs = tf.placeholder(
tf.int32, shape=[self.batch_size_sg, None])
self.train_labels = tf.placeholder(tf.int32,
shape=[self.batch_size_sg, 1])
if not self.event_layer.shared:
self.loss += self.event_layer.alpha * self.event_layer.loss(
self.num_sampled,
self.train_labels,
self.train_inputs,
embeddings=None)
else:
self.loss += self.event_layer.alpha * self.event_layer.loss(
self.num_sampled,
self.train_labels,
self.train_inputs,
embeddings=self.E)
self.global_step = tf.Variable(0, trainable=False)
starter_learning_rate = self.init_lr
learning_rate = tf.constant(starter_learning_rate)
self.optimizer = tf.train.AdagradOptimizer(learning_rate).minimize(
self.loss)
def create_graph(self):
print('Building Model')
# Translation Model initialisation
w_bound = np.sqrt(6. / self.embedding_size)
self.E = tf.Variable(tf.random_uniform((self.num_entities, self.embedding_size), minval=-w_bound,
maxval=w_bound))
self.R = tf.Variable(tf.random_uniform((self.num_relations, self.embedding_size), minval=-w_bound,
maxval=w_bound))
self.W = tf.Variable(tf.random_uniform((self.num_relations, self.embedding_size), minval=-w_bound,
maxval=w_bound))
self.normalize_W = self.W.assign(tf.nn.l2_normalize(self.W, 1))
self.inpr = tf.placeholder(tf.int32, [self.batch_size_kg], name="rhs")
self.inpl = tf.placeholder(tf.int32, [self.batch_size_kg], name="lhs")
self.inpo = tf.placeholder(tf.int32, [self.batch_size_kg], name="rell")
self.inprn = tf.placeholder(tf.int32, [self.batch_size_kg], name="rhsn")
self.inpln = tf.placeholder(tf.int32, [self.batch_size_kg], name="lhsn")
self.inpon = tf.placeholder(tf.int32, [self.batch_size_kg], name="relln")
lhs = tf.nn.embedding_lookup(self.E, self.inpl)
rhs = tf.nn.embedding_lookup(self.E, self.inpr)
rell = tf.nn.embedding_lookup(self.R, self.inpo)
wr = tf.nn.embedding_lookup(self.W, self.inpo)
lhsn = tf.nn.embedding_lookup(self.E, self.inpln)
rhsn = tf.nn.embedding_lookup(self.E, self.inprn)
relln = tf.nn.embedding_lookup(self.R, self.inpon)
wrn = tf.nn.embedding_lookup(self.W, self.inpon)
if self.event_layer is not None:
self.event_layer.create_graph()
if not self.event_layer.shared:
self.a = tf.Variable(tf.random_uniform([self.embedding_size], minval=-w_bound,
maxval=w_bound), name="a")
self.b = tf.Variable(tf.random_uniform([self.embedding_size], minval=-w_bound,
maxval=w_bound), name="b")
lhs = tf.multiply(self.a, lhs) + tf.multiply(self.b, tf.nn.embedding_lookup(self.event_layer.V,
self.inpl))
rhs = tf.multiply(self.a, rhs) + tf.multiply(self.b, tf.nn.embedding_lookup(self.event_layer.V,
self.inpr))
lhsn = tf.multiply(self.a, lhsn) + tf.multiply(self.b, tf.nn.embedding_lookup(self.event_layer.V,
self.inpln))
rhsn = tf.multiply(self.a, rhsn) + tf.multiply(self.b, tf.nn.embedding_lookup(self.event_layer.V,
self.inprn))
lhs_proj = lhs - dot(lhs, wr) * wr # dot and elementwise mul => projection
rhs_proj = rhs - dot(rhs, wr) * wr
lhs_proj_n = lhsn - dot(lhsn, wrn) * wrn
rhs_proj_n = rhsn - dot(rhsn, wrn) * wrn
simi = l2_similarity(trans(lhs_proj, rell), ident_entity(rhs_proj, rell))
simin = l2_similarity(trans(lhs_proj_n, relln), ident_entity(rhs_proj_n, relln))
# TransH Loss
epsilon = tf.constant(0.0001)
reg1 = tf.maximum(0., tf.reduce_sum(tf.sqrt(tf.reduce_sum(self.E ** 2, axis=1)) - 1))
reg2_z = dot(self.W, self.R) ** 2
reg2_n = tf.expand_dims(tf.sqrt(tf.reduce_sum(self.R ** 2, axis=1)), 1)
reg2 = tf.reduce_sum(tf.maximum(0., (reg2_z / reg2_n) - epsilon))
kg_loss = max_margin(simi, simin) + self.lambd * (reg1 + reg2)
self.loss = kg_loss
if self.event_layer is not None:
if type(self.event_layer) == Skipgram:
self.train_inputs = tf.placeholder(tf.int32, shape=[self.batch_size_sg])
else:
self.train_inputs = tf.placeholder(tf.int32, shape=[self.batch_size_sg, None])
self.train_labels = tf.placeholder(tf.int32, shape=[self.batch_size_sg, 1])
if not self.event_layer.shared:
self.loss += self.event_layer.alpha * self.event_layer.loss(self.num_sampled, self.train_labels,
self.train_inputs, embeddings=None)
else:
self.loss += self.event_layer.alpha * self.event_layer.loss(self.num_sampled, self.train_labels,
self.train_inputs, embeddings=self.E)
self.global_step = tf.Variable(0, trainable=False)
starter_learning_rate = self.init_lr
learning_rate = tf.constant(starter_learning_rate)
self.optimizer = tf.train.AdagradOptimizer(learning_rate).minimize(self.loss)
def create_graph(self):
print('Building Model')
# Translation Model initialisation
self.w_bound = np.sqrt(6. / self.embedding_size)
self.E = tf.Variable(tf.random_uniform(
(self.num_entities, self.embedding_size),
minval=-self.w_bound,
maxval=self.w_bound),
name="E")
self.R = tf.Variable(tf.random_uniform(
(self.num_relations, self.embedding_size, self.embedding_size),
minval=-self.w_bound,
maxval=self.w_bound),
name="R")
self.inpr = tf.placeholder(tf.int32, [self.batch_size_kg], name="rhs")
self.inpl = tf.placeholder(tf.int32, [self.batch_size_kg], name="lhs")
self.inpo = tf.placeholder(tf.int32, [self.batch_size_kg], name="rell")
self.inprn = tf.placeholder(tf.int32, [self.batch_size_kg],
name="rhsn")
self.inpln = tf.placeholder(tf.int32, [self.batch_size_kg],
name="lhsn")
self.inpon = tf.placeholder(tf.int32, [self.batch_size_kg],
name="relln")
lhs = tf.nn.embedding_lookup(self.E, self.inpl)
rhs = tf.nn.embedding_lookup(self.E, self.inpr)
rell = tf.nn.embedding_lookup(self.R, self.inpo)
lhsn = tf.nn.embedding_lookup(self.E, self.inpln)
rhsn = tf.nn.embedding_lookup(self.E, self.inprn)
relln = tf.nn.embedding_lookup(self.R, self.inpon)
# RESCAL with ranking loss
lhs = tf.expand_dims(lhs, 1) # [batch, 1, d] # zeile mal zeile
lhs = tf.reduce_sum(tf.multiply(lhs, rell),
2) # [batch, 1, d] * [batch, d, d] ==> [batch, d]
lhsn = tf.expand_dims(lhsn, 1)
lhsn = tf.reduce_sum(tf.multiply(lhsn, relln), 2)
if self.event_layer is not None:
self.event_layer.create_graph()
if not self.event_layer.shared:
self.a = tf.Variable(tf.random_uniform([self.embedding_size],
minval=-self.w_bound,
maxval=self.w_bound),
name="a")
self.b = tf.Variable(tf.random_uniform([self.embedding_size],
minval=-self.w_bound,
maxval=self.w_bound),
name="b")
lhs = tf.multiply(self.a, lhs) + tf.multiply(
self.b,
tf.nn.embedding_lookup(self.event_layer.V, self.inpl))
rhs = tf.multiply(self.a, rhs) + tf.multiply(
self.b,
tf.nn.embedding_lookup(self.event_layer.V, self.inpr))
lhsn = tf.multiply(self.a, lhsn) + tf.multiply(
self.b,
tf.nn.embedding_lookup(self.event_layer.V, self.inpln))
rhsn = tf.multiply(self.a, rhsn) + tf.multiply(
self.b,
tf.nn.embedding_lookup(self.event_layer.V, self.inprn))
simi = tf.nn.sigmoid(tf.reduce_sum(tf.multiply(lhs, rhs),
1)) # [batch, d] * [batch, d]
simin = tf.nn.sigmoid(tf.reduce_sum(tf.multiply(lhsn, rhsn), 1))
kg_loss = max_margin(simi, simin) + self.lambd * (
tf.nn.l2_loss(self.E) + tf.nn.l2_loss(self.R))
self.loss = kg_loss
if self.event_layer is not None:
if type(self.event_layer) == Skipgram:
self.train_inputs = tf.placeholder(tf.int32,
shape=[self.batch_size_sg])
else:
self.train_inputs = tf.placeholder(
tf.int32, shape=[self.batch_size_sg, None])
self.train_labels = tf.placeholder(tf.int32,
shape=[self.batch_size_sg, 1])
if not self.event_layer.shared:
self.loss += self.event_layer.alpha * self.event_layer.loss(
self.num_sampled,
self.train_labels,
self.train_inputs,
embeddings=None)
else:
self.loss += self.event_layer.alpha * self.event_layer.loss(
self.num_sampled,
self.train_labels,
self.train_inputs,
embeddings=self.E)
self.global_step = tf.Variable(0, trainable=False)
starter_learning_rate = self.init_lr
# tf.train.exponential_decay(starter_learning_rate, global_step, 10, 0.98, staircase=True)
learning_rate = tf.constant(starter_learning_rate)
self.optimizer = tf.train.AdagradOptimizer(learning_rate).minimize(
self.loss)
def create_graph(self):
print('Building Model')
# Translation Model initialisation
w_bound = np.sqrt(6. / self.embedding_size)
self.E = tf.Variable(tf.random_uniform(
(self.num_entities, self.embedding_size),
minval=-w_bound,
maxval=w_bound),
name="E")
self.R = tf.Variable(tf.random_uniform(
(self.num_relations, self.embedding_size),
minval=-w_bound,
maxval=w_bound),
name="R")
self.A = tf.Variable(tf.random_uniform(
(self.embedding_size, self.embedding_size),
minval=-w_bound,
maxval=w_bound),
name="A")
self.N = tf.Variable(tf.constant(self.tk.get_pointwise(), tf.float32),
trainable=False)
self.normalize_E = self.E.assign(tf.nn.l2_normalize(self.E, 1))
self.normalize_R = self.R.assign(tf.nn.l2_normalize(self.R, 1))
self.inpr = tf.placeholder(tf.int32, [self.batch_size_kg], name="rhs")
self.inpl = tf.placeholder(tf.int32, [self.batch_size_kg], name="lhs")
self.inpo = tf.placeholder(tf.int32, [self.batch_size_kg], name="rell")
self.inprn = tf.placeholder(tf.int32, [self.batch_size_kg],
name="rhsn")
self.inpln = tf.placeholder(tf.int32, [self.batch_size_kg],
name="lhsn")
self.inpon = tf.placeholder(tf.int32, [self.batch_size_kg],
name="relln")
n_h = tf.nn.embedding_lookup(self.N, self.inpl)
n_t = tf.nn.embedding_lookup(self.N, self.inpr)
n_hn = tf.nn.embedding_lookup(self.N, self.inpln)
n_tn = tf.nn.embedding_lookup(self.N, self.inprn)
lhs = tf.nn.embedding_lookup(self.E, self.inpl)
rhs = tf.nn.embedding_lookup(self.E, self.inpr)
rell = tf.nn.embedding_lookup(self.R, self.inpo)
lhsn = tf.nn.embedding_lookup(self.E, self.inpln)
rhsn = tf.nn.embedding_lookup(self.E, self.inprn)
relln = tf.nn.embedding_lookup(self.R, self.inpon)
lhs = tf.matmul(n_h, self.A) + lhs
rhs = tf.matmul(n_t, self.A) + rhs
# rell = tf.matmul(self.n_x_y, self.B) + rell
lhsn = tf.matmul(n_hn, self.A) + lhsn
rhsn = tf.matmul(n_tn, self.A) + rhsn
# relln = tf.matmul(self.n_x_yn, self.B) + relln
if self.fnsim == dot_similarity:
simi = tf.diag_part(
self.fnsim(self.leftop(lhs, rell),
tf.transpose(self.rightop(rhs, rell)),
broadcast=False))
simin = tf.diag_part(
self.fnsim(self.leftop(lhsn, rell),
tf.transpose(self.rightop(rhsn, rell)),
broadcast=False))
else:
simi = self.fnsim(self.leftop(lhs, rell),
self.rightop(rhs, rell),
broadcast=False)
simin = self.fnsim(self.leftop(lhsn, relln),
self.rightop(rhsn, relln),
broadcast=False)
kg_loss = max_margin(simi, simin)
# self.reg1 = tf.maximum(0., tf.reduce_sum(tf.sqrt(tf.reduce_sum(tf.matmul(n_h, self.A)**2, axis=1)) - 1))
# self.reg2 = tf.maximum(0., tf.reduce_sum(tf.sqrt(tf.reduce_sum(tf.matmul(n_t, self.A)**2, axis=1)) - 1))
self.loss = kg_loss
self.global_step = tf.Variable(0, trainable=False)
starter_learning_rate = self.init_lr
learning_rate = tf.constant(starter_learning_rate)
self.optimizer = tf.train.AdagradOptimizer(learning_rate).minimize(
self.loss)