def compute_message_sent_by_one_layer(hidden, graph, n_dims, reuse=None):
''' hidden: bs x n_nodes x n_dims
'''
with tf.variable_scope('message', reuse=reuse):
weight_rel_set = tf.get_variable(
'weight_rel_set',
shape=[graph.n_relations, n_dims, n_dims],
initializer=tf.constant_initializer(
np.tile(np.identity(n_dims),
(graph.n_relations, 1,
1)))) # n_relations x n_dims x n_dims
bias_rel_set = tf.get_variable(
'bias_rel_set',
shape=[graph.n_relations, n_dims],
initializer=tf.zeros_initializer()) # n_relations x n_dims
weight_comb = tf.sparse_segment_sum(
weight_rel_set, graph.unfolded_edges('rel_id_list'),
graph.unfolded_edges('edge_id_list')) # n_edges x n_dims x n_dims
bias_comb = tf.sparse_segment_sum(
bias_rel_set, graph.unfolded_edges('rel_id_list'),
graph.unfolded_edges('edge_id_list')) # n_edges x n_dims
hidden_emb = tf.transpose(hidden, perm=[1, 0,
2]) # n_nodes x bs x n_dims
hidden_v1 = tf.gather(
hidden_emb, graph.folded_edges('v1_list')) # n_edges x bs x n_dims
message = tf.tanh(
tf.matmul(hidden_v1, weight_comb) +
tf.expand_dims(bias_comb, axis=1)) # n_edges x bs x n_dims
message = tf.transpose(message, perm=[1, 0,
2]) # bs x n_edges x n_dims
return message
def main_model(u0, u1, ad0, ad1, label, ids0, ids1, gear_inputs): graph0 = tf.sparse_segment_sum(gear_inputs[0], tf.range(tf.shape(gear_inputs[0])[0]), ids0, num_segments=tf.shape(label)[0]) graph1 = tf.sparse_segment_sum(gear_inputs[1], tf.range(tf.shape(gear_inputs[1])[0]), ids1, num_segments=tf.shape(label)[0]) input_layer = tf.concat([u0, u1, ad0, ad1, graph0, graph1], -1) l1 = fc(input_layer, main_input_length, 100, "mainfc1") l2 = fc(l1, 100, 10, "mainfc2") l3 = fc(l2, 10, 1, "mainfc3") return tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(logits=l3, labels=label))
def main_model(u0, u1, ad0, ad1, label, ids0, ids1, gear_inputs):
graph0 = tf.sparse_segment_sum(gear_inputs[0],
tf.range(tf.shape(gear_inputs[0])[0]),
ids0,
num_segments=tf.shape(label)[0])
graph1 = tf.sparse_segment_sum(gear_inputs[1],
tf.range(tf.shape(gear_inputs[1])[0]),
ids1,
num_segments=tf.shape(label)[0])
input_layer = tf.concat([u0, u1, ad0, ad1, graph0, graph1], -1)
l1 = fc(input_layer, main_input_length, 100, "mainfc1")
l2 = fc(l1, 100, 10, "mainfc2")
l3 = fc(l2, 10, 1, "mainfc3")
return tf.reduce_sum(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l3, labels=label))
def compute_edge_attention_by_one_layer(hidden, graph, n_dims, reuse=None):
''' hidden: bs x n_nodes x n_dims
'''
with tf.variable_scope('edge_attention', reuse=reuse):
weight_rel_set = tf.get_variable(
'weight_rel_set',
shape=[graph.n_relations, n_dims, n_dims],
initializer=tf.constant_initializer(
np.tile(np.identity(n_dims),
(graph.n_relations, 1,
1)))) # n_relations x n_dims x n_dims
weight_comb = tf.sparse_segment_sum(
weight_rel_set, graph.unfolded_edges('rel_id_list'),
graph.unfolded_edges('edge_id_list')) # n_edges x n_dims x n_dims
hidden_emb = tf.transpose(hidden, perm=[1, 0,
2]) # n_nodes x bs x n_dims
hidden_v1 = tf.gather(
hidden_emb, graph.folded_edges('v1_list')) # n_edges x bs x n_dims
hidden_v2 = tf.gather(
hidden_emb, graph.folded_edges('v2_list')) # n_edges x bs x n_dims
edge_attention = tf.nn.softplus(
tf.reduce_sum(tf.matmul(hidden_v1, weight_comb) * hidden_v2,
axis=2)) # n_edges x bs
edge_attention_sum = tf.segment_sum(
edge_attention, graph.folded_edges('v1_list')) # n_nodes x bs
edge_attention_sum = tf.gather(
edge_attention_sum, graph.folded_edges('v1_list')) # n_edges x bs
edge_attention_nor = tf.transpose(edge_attention /
edge_attention_sum) # bs x n_edges
return edge_attention_nor
def embedding_lookup_sparse_sumexp(params, sp_ids, name=None):
segment_ids = sp_ids.indices[:, 0]
if segment_ids.dtype != tf.int32:
segment_ids = tf.cast(segment_ids, tf.int32)
ids = sp_ids.values
ids, idx = tf.unique(ids)
embeddings = tf.nn.embedding_lookup(params, ids)
embeddings = tf.exp(embeddings)
embeddings = tf.sparse_segment_sum(embeddings, idx, segment_ids, name=name)
return embeddings
def test_SparseSegmentSum(self):
t = tf.sparse_segment_sum(self.random(4, 3, 2), [0, 2, 3], [0, 1, 1])
self.check(t)
def AddTraining(self,
task_context,
batch_size,
learning_rate=0.1,
decay_steps=4000,
momentum=None,
corpus_name='documents'):
with tf.name_scope('training'):
n = self.training
n['accumulated_alive_steps'] = self._AddVariable(
[batch_size], tf.int32, 'accumulated_alive_steps',
tf.zeros_initializer())
n.update(self._AddBeamReader(task_context, batch_size, corpus_name))
# This adds a required 'step' node too:
learning_rate = tf.constant(learning_rate, dtype=tf.float32)
n['learning_rate'] = self._AddLearningRate(learning_rate, decay_steps)
# Call BuildNetwork *only* to set up the params outside of the main loop.
self._BuildNetwork(list(n['features']))
n.update(self._BuildSequence(batch_size, self._max_steps, n['features'],
n['state']))
flat_concat_scores = tf.reshape(n['concat_scores'], [-1])
(indices_and_paths, beams_and_slots, n['gold_slot'], n[
'beam_path_scores']) = gen_parser_ops.beam_parser_output(n[
'state'])
n['indices'] = tf.reshape(tf.gather(indices_and_paths, [0]), [-1])
n['path_ids'] = tf.reshape(tf.gather(indices_and_paths, [1]), [-1])
n['all_path_scores'] = tf.sparse_segment_sum(
flat_concat_scores, n['indices'], n['path_ids'])
n['beam_ids'] = tf.reshape(tf.gather(beams_and_slots, [0]), [-1])
n.update(AddCrossEntropy(batch_size, n))
if self._only_train:
trainable_params = {k: v for k, v in self.params.iteritems()
if k in self._only_train}
else:
trainable_params = self.params
for p in trainable_params:
tf.logging.info('trainable_param: %s', p)
regularized_params = [
tf.nn.l2_loss(p) for k, p in trainable_params.iteritems()
if k.startswith('weights') or k.startswith('bias')]
l2_loss = 1e-4 * tf.add_n(regularized_params) if regularized_params else 0
n['cost'] = tf.add(n['cross_entropy'], l2_loss, name='cost')
n['gradients'] = tf.gradients(n['cost'], trainable_params.values())
with tf.control_dependencies([n['alive_steps']]):
update_accumulators = tf.group(
tf.assign_add(n['accumulated_alive_steps'], n['alive_steps']))
def ResetAccumulators():
return tf.assign(
n['accumulated_alive_steps'], tf.zeros([batch_size], tf.int32))
n['reset_accumulators_func'] = ResetAccumulators
optimizer = tf.train.MomentumOptimizer(n['learning_rate'],
momentum,
use_locking=self._use_locking)
train_op = optimizer.minimize(n['cost'],
var_list=trainable_params.values())
for param in trainable_params.values():
slot = optimizer.get_slot(param, 'momentum')
self.inits[slot.name] = state_ops.init_variable(slot,
tf.zeros_initializer())
self.variables[slot.name] = slot
def NumericalChecks():
return tf.group(*[
tf.check_numerics(param, message='Parameter is not finite.')
for param in trainable_params.values()
if param.dtype.base_dtype in [tf.float32, tf.float64]])
check_op = cf.cond(tf.equal(tf.mod(self.GetStep(), self._check_every), 0),
NumericalChecks, tf.no_op)
avg_update_op = tf.group(*self._averaging.values())
train_ops = [train_op]
if self._check_parameters:
train_ops.append(check_op)
if self._use_averaging:
train_ops.append(avg_update_op)
with tf.control_dependencies([update_accumulators]):
n['train_op'] = tf.group(*train_ops, name='train_op')
n['alive_steps'] = tf.identity(n['alive_steps'], name='alive_steps')
return n
import tensorflow as tf
import numpy as np
input_a = np.array([[1, 1, 2], [2, 3, 4], [3, 1, 1], [2, 4, 6]])
a_seg_sum = tf.segment_sum(data=input_a, segment_ids=[0, 1, 1, 1])
a_seg_prod = tf.segment_prod(data=input_a, segment_ids=[0, 0, 1, 1])
a_seg_max = tf.segment_max(data=input_a, segment_ids=[0, 0, 0, 1])
a_seg_min = tf.segment_min(data=input_a, segment_ids=[1, 1, 1, 1])
a_seg_mean = tf.segment_mean(data=input_a, segment_ids=[0, 0, 0, 1])
a_seg_sum_num = tf.unsorted_segment_sum(data=input_a,
segment_ids=[0, 1, 1, 0],
num_segments=2)
a_sparse_seg_sum = tf.sparse_segment_sum(data=input_a,
indices=[0, 1, 2],
segment_ids=[0, 0, 1])
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
print(sess.run(a_seg_sum), '\n', sess.run(a_seg_prod), '\n',
sess.run(a_seg_max), '\n', sess.run(a_seg_min))
print(sess.run(a_seg_mean), '\n', sess.run(a_seg_sum_num), '\n',
sess.run(a_sparse_seg_sum))
import tensorflow as tf """tf.sparse_segment_sum(data, indices, segment_ids, name=None) 功能：tensor进行拆分后求和。和segment_sum类似，只是segment_ids的rank数可以小于‘data’第0维度数。 输入：indices:选择第0维度参与运算的编号。""" a = tf.constant([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]) z = tf.sparse_segment_sum(a, tf.constant([0, 1]), tf.constant([0, 0])) # 选择前两行，并且计算前两行的和 z2 = tf.sparse_segment_sum(a, tf.constant([0, 1]), tf.constant([0, 1])) # 选择前两行，但是只计算第0行的和，第一行保留原样 z3 = tf.sparse_segment_sum(a, tf.constant([0, 2]), tf.constant([0, 1])) # 选择第0行和第2行但是只计算第0行，第2行保留原样 z4 = tf.sparse_segment_sum(a, tf.constant([0, 1, 2]), tf.constant([0, 0, 1])) # 选择三行，但是只是计算0行和1行，第2行保留原样 sess = tf.Session() print(sess.run(z)) print(sess.run(z2)) print(sess.run(z3)) print(sess.run(z4)) sess.close() # z==>[[6 8 10 12]] # z2==>[[1 2 3 4] # [5 6 7 8]] # z3==>[[1 2 3 4] # [9 10 11 12]] # z4==>[[6 8 10 12] # [9 10 11 12]]
def AddTraining(self,
task_context,
batch_size,
learning_rate=0.1,
decay_steps=4000,
momentum=None,
corpus_name='documents'):
with tf.name_scope('training'):
n = self.training
n['accumulated_alive_steps'] = self._AddVariable(
[batch_size], tf.int32, 'accumulated_alive_steps',
tf.zeros_initializer())
n.update(self._AddBeamReader(task_context, batch_size,
corpus_name))
# This adds a required 'step' node too:
learning_rate = tf.constant(learning_rate, dtype=tf.float32)
n['learning_rate'] = self._AddLearningRate(learning_rate,
decay_steps)
# Call BuildNetwork *only* to set up the params outside of the main loop.
self._BuildNetwork(list(n['features']))
n.update(
self._BuildSequence(batch_size, self._max_steps, n['features'],
n['state']))
flat_concat_scores = tf.reshape(n['concat_scores'], [-1])
(indices_and_paths, beams_and_slots, n['gold_slot'],
n['beam_path_scores']) = gen_parser_ops.beam_parser_output(
n['state'])
n['indices'] = tf.reshape(tf.gather(indices_and_paths, [0]), [-1])
n['path_ids'] = tf.reshape(tf.gather(indices_and_paths, [1]), [-1])
n['all_path_scores'] = tf.sparse_segment_sum(
flat_concat_scores, n['indices'], n['path_ids'])
n['beam_ids'] = tf.reshape(tf.gather(beams_and_slots, [0]), [-1])
n.update(AddCrossEntropy(batch_size, n))
if self._only_train:
trainable_params = {
k: v
for k, v in self.params.iteritems()
if k in self._only_train
}
else:
trainable_params = self.params
for p in trainable_params:
tf.logging.info('trainable_param: %s', p)
regularized_params = [
tf.nn.l2_loss(p) for k, p in trainable_params.iteritems()
if k.startswith('weights') or k.startswith('bias')
]
l2_loss = 1e-4 * tf.add_n(
regularized_params) if regularized_params else 0
n['cost'] = tf.add(n['cross_entropy'], l2_loss, name='cost')
n['gradients'] = tf.gradients(n['cost'], trainable_params.values())
with tf.control_dependencies([n['alive_steps']]):
update_accumulators = tf.group(
tf.assign_add(n['accumulated_alive_steps'],
n['alive_steps']))
def ResetAccumulators():
return tf.assign(n['accumulated_alive_steps'],
tf.zeros([batch_size], tf.int32))
n['reset_accumulators_func'] = ResetAccumulators
optimizer = tf.train.MomentumOptimizer(
n['learning_rate'], momentum, use_locking=self._use_locking)
train_op = optimizer.minimize(n['cost'],
var_list=trainable_params.values())
for param in trainable_params.values():
slot = optimizer.get_slot(param, 'momentum')
self.inits[slot.name] = state_ops.init_variable(
slot, tf.zeros_initializer())
self.variables[slot.name] = slot
def NumericalChecks():
return tf.group(*[
tf.check_numerics(param,
message='Parameter is not finite.')
for param in trainable_params.values()
if param.dtype.base_dtype in [tf.float32, tf.float64]
])
check_op = cf.cond(
tf.equal(tf.mod(self.GetStep(), self._check_every), 0),
NumericalChecks, tf.no_op)
avg_update_op = tf.group(*self._averaging.values())
train_ops = [train_op]
if self._check_parameters:
train_ops.append(check_op)
if self._use_averaging:
train_ops.append(avg_update_op)
with tf.control_dependencies([update_accumulators]):
n['train_op'] = tf.group(*train_ops, name='train_op')
n['alive_steps'] = tf.identity(n['alive_steps'],
name='alive_steps')
return n
def test_SparseSegmentSum(self):
t = tf.sparse_segment_sum(self.random(4, 3, 2), [0, 2, 3], [0, 1, 1])
self.check(t)
# tf.segment_mean
x = np.random.rand(10, 4, 3)
segment_ids = [0, 0, 0, 1, 2, 2, 3, 4, 5, 5]
z_segment_mean = tf.segment_mean(x, segment_ids)
# tf.unsorted_segment_sum
x = np.random.rand(10, 4, 3)
segment_ids = [5, 0, 0, 2, 1, 2, 3, 4, 0, 5]
num_segments = np.max(segment_ids) + 1
z_unsorted_segment_sum = tf.unsorted_segment_sum(x, segment_ids, num_segments)
# tf.sparse_segment_sum
x = np.random.rand(10, 4, 3)
indices = [0, 2, 3, 4]
segment_ids = [0, 0, 1, 1]
z_sparse_segment_sum = tf.sparse_segment_sum(x, indices, segment_ids)
# tf.sparse_segment_mean
x = np.random.rand(10, 4, 3)
indices = [0, 2, 3, 4]
segment_ids = [0, 1, 1, 1]
z_sparse_segment_mean = tf.sparse_segment_mean(x, indices, segment_ids)
with tf.Session() as sess:
print "tf.segment_sum"
print sess.run(z_segment_sum)
print "tf.segment_prod"
print sess.run(z_segment_prod)
print "tf.segment_min"
其中id为int型数据，最大id不大于size
c = tf.constant([[1,2,3,4], [-1,-2,-3,-4], [5,6,7,8]])
tf.segment_sum(c, tf.constant([0, 0, 1]))
==>[[0 0 0 0]
[5 6 7 8]]
上面例子分为[0,1]两id,对相同id的data相应数据进行求和,
并放入结果的相应id中，
且segment_ids只升不降
tf.segment_prod(data, segment_ids, name=None) 根据segment_ids的分段计算各个片段的积
tf.segment_min(data, segment_ids, name=None) 根据segment_ids的分段计算各个片段的最小值
tf.segment_max(data, segment_ids, name=None) 根据segment_ids的分段计算各个片段的最大值
tf.segment_mean(data, segment_ids, name=None) 根据segment_ids的分段计算各个片段的平均值
tf.unsorted_segment_sum(data, segment_ids,
num_segments, name=None) 与tf.segment_sum函数类似，
不同在于segment_ids中id顺序可以是无序的
tf.sparse_segment_sum(data, indices,
segment_ids, name=None) 输入进行稀疏分割求和
c = tf.constant([[1,2,3,4], [-1,-2,-3,-4], [5,6,7,8]])
# Select two rows, one segment.
tf.sparse_segment_sum(c, tf.constant([0, 1]), tf.constant([0, 0]))
==> [[0 0 0 0]]
对原data的indices为[0,1]位置的进行分割，
并按照segment_ids的分组进行求和
七、序列比较与索引提取(Sequence Comparison and Indexing)
tf.argmin(input, dimension, name=None) 返回input最小值的索引index
tf.argmax(input, dimension, name=None) 返回input最大值的索引index
tf.listdiff(x, y, name=None) 返回x，y中不同值的索引
tf.where(input, name=None) 返回bool型tensor中为True的位置
# ‘input’ tensor is
#[[True, False]
#[True, False]]
import tensorflow as tf with tf.Session(): c = tf.constant([[1.,2.,3.,4.],[-1.,-2.,-3.,-5.],[5.,6.,7.,8.]]) print tf.sparse_segment_sum(c, tf.constant([0, 1]), tf.constant([0, 0])).eval()
