def testSegmentMinGradient(self):
data = tf.constant([1.0, 2.0, 3.0], dtype=tf.float32)
segment_ids = tf.constant([0, 0, 1], dtype=tf.int64)
segment_min = tf.segment_min(data, segment_ids)
with self.test_session():
error = tf.test.compute_gradient_error(data, [3], segment_min, [2])
self.assertLess(error, 1e-4)
def testSegmentMinGradient(self):
data = tf.constant([1.0, 2.0, 3.0], dtype=tf.float32)
segment_ids = tf.constant([0, 0, 1], dtype=tf.int64)
segment_min = tf.segment_min(data, segment_ids)
with self.test_session():
error = tf.test.compute_gradient_error(data, [3], segment_min, [2])
self.assertLess(error, 1e-4)
def matrix_segment():
"""
:return:
"""
isses = tf.InteractiveSession()
# 对角值
X = tf.constant([5., 1., 7., 2., 3., 4., 1., 3.], dtype=tf.float32)
s_id = [0, 0, 0, 1, 2, 2, 3, 3]
logger.info("X\n%s" % X)
logger.info("s_id\n%s" % s_id)
logger.info("tf.segment_sum(X,s_id)\n {0}".format(tf.segment_sum(X, s_id)))
logger.info("tf.segment_mean(X,s_id)\n {0}".format(
tf.segment_mean(X, s_id).eval()))
logger.info("tf.segment_max(X, s_id)\n {0}".format(
tf.segment_max(X, s_id).eval()))
logger.info("tf.segment_min(X, s_id)\n {0}".format(
tf.segment_min(X, s_id).eval()))
logger.info("tf.segment_prod(X, s_id)\n {0}".format(
tf.segment_prod(X, s_id).eval()))
logger.info("tf.unsorted_segment_sum(X, s_id)\n {0}".format(
tf.unsorted_segment_sum(X, s_id, 2)))
# c = tf.constant([0., 1.], dtype=tf.float32)
# logger.info("tf.sparse_segment_sum(X, s_id)\n {0}".format(tf.sparse_segment_sum(X, c, s_id)))
# logger.info("tf.sparse_segment_mean(X, s_id)\n {0}".format(tf.sparse_segment_mean(X, c, s_id)))
# logger.info("tf.sparse_segment_sqrt_n(X, s_id)\n {0}".format(tf.sparse_segment_sqrt_n(X, c, s_id)))
isses.close()
def do_segments(values):
"""Actually does segmentation.
Args:
values: 1D tensor of any type. Non-empty.
Returns:
run_centers: int32 tensor
run_lengths: int32 tensor
Raises:
ValueError: if mode is not recognized.
"""
length = tf.shape(values)[0]
values = tf.convert_to_tensor(values)
# The first run has id 0, so we don't increment the id.
# Otherwise, the id is incremented when the value changes.
run_start_bool = tf.concat(
[[False], tf.not_equal(values[1:], values[:-1])], axis=0)
# Cumulative sum the run starts to get the run ids.
segment_ids = tf.cumsum(tf.cast(run_start_bool, tf.int32))
if mode is SegmentsMode.STARTS:
run_centers = tf.segment_min(tf.range(length), segment_ids)
elif mode is SegmentsMode.CENTERS:
run_centers = tf.segment_mean(
tf.cast(tf.range(length), tf.float32), segment_ids)
run_centers = tf.cast(tf.floor(run_centers), tf.int32)
else:
raise ValueError("Unexpected mode: %s" % mode)
run_lengths = tf.segment_sum(tf.ones([length], tf.int32),
segment_ids)
return run_centers, run_lengths
def reduce_per_answer_loss(self, loss):
# Get any of the alternatives right
loss = tf.segment_min(loss, self.answer_partition)
# Get all of the answers right
loss = tf.segment_mean(loss, self.answer_question_partition)
return tf.reduce_mean(loss)
def get_feature_with_length(features, name):
"""Reads out embeddings and sequence lengths for a symbol modality
from the features.
Args:
features (dict): Dictionary with features.
name (string): Feature to extract (will read features[name] and maybe
features[name_raw]
Returns:
Pair of (embed, length) tensors, where `embed` is a (batch_size,
max_len, embed_size) float32 tensor with embeddings, and `length`
is a (batch_size,) int32 tensor with sequence lengths.
"""
# features[name] shape: (batch_size, max_len, 1, embed_size)
embed = common_layers.flatten4d3d(features[name])
# embed shape: (batch_size, max_len, embed_size)
if "%s_raw" % name in features:
raw = tf.squeeze(features["%s_raw" % name], axis=[2, 3])
not_padding = tf.not_equal(raw, text_encoder.PAD_ID)
else:
tf.logging.warn(
"Feature %s is not exposed by T2T in raw form which makes it difficult "
"to extract sequence lengths. Consider using the T2T fork at "
"https://github.com/fstahlberg/tensor2tensor. For now we back off to a "
"more ineffective way to get sequence lengths to maintain compatibility "
"with the T2T master fork.", name)
not_padding = tf.greater(tf.reduce_sum(tf.abs(embed), axis=2),
0.000001)
not_padding_with_guardian = tf.pad(not_padding, [[0, 0], [0, 1]])
indices = tf.where(tf.logical_not(not_padding_with_guardian))
length = tf.segment_min(indices[:, 1], indices[:, 0])
return embed, tf.cast(length, tf.int32)
def get_len(sen, eos):
'''
get the length of each sample
'''
indices = tf.where(tf.equal(sen, eos))
result = tf.segment_min(indices[:, 1], indices[:, 0])
return result
def testSegmentMinGradientWithTies(self):
inputs = tf.constant([1.0], dtype=tf.float32)
data = tf.concat_v2([inputs, inputs], 0)
segment_ids = tf.constant([0, 0], dtype=tf.int64)
segment_min = tf.segment_min(data, segment_ids)
with self.test_session():
error = tf.test.compute_gradient_error(inputs, [1], segment_min, [1])
self.assertLess(error, 1e-4)
def testSegmentMinGradientWithTies(self):
inputs = tf.constant([1.0], dtype=tf.float32)
data = tf.concat(0, [inputs, inputs])
segment_ids = tf.constant([0, 0], dtype=tf.int64)
segment_min = tf.segment_min(data, segment_ids)
with self.test_session():
error = tf.test.compute_gradient_error(inputs, [1], segment_min, [1])
self.assertLess(error, 1e-4)
def get_first_occurrence_indices(reference, symbol, optimize_for_tpu=False):
"""For each row in reference, get index after the first occurrence of symbol.
If symbol is not present on a row, return reference.shape[1] instead.
Args:
reference: [B, T] tensor of elements of the same type as symbol.
symbol: int or [] scalar tensor of the same dtype as symbol.
optimize_for_tpu: bool, whether to use a TPU-capable variant.
Returns:
A [B] reference of tf.int32 where x[i] is such that
reference[i, x[i]-1] == symbol, and reference[i, j] != symbol
for j<i-1. If symbol is not present on row i then x[i] = T.
"""
if optimize_for_tpu:
# Run code which can be compiled on TPU.
# Transpose refernce to [T, B]
reference = tf.transpose(reference, [1, 0])
range_tensor = tf.range(reference.shape.as_list()[0])
indexes = tf.stack([range_tensor] * reference.shape.as_list()[1], 1)
symbol = tf.stack([symbol] * reference.shape.as_list()[1], 0)
initial_indices = tf.constant(reference.shape.as_list()[0],
shape=[reference.shape.as_list()[1]],
dtype=tf.int32)
# We want a function which moves backwards.
def fn(current_index, elems):
ref, ind = elems
return tf.where(tf.equal(ref, symbol), ind + 1, current_index)
min_indexes = tf.scan(fn, (reference, indexes),
initializer=initial_indices,
parallel_iterations=1,
reverse=True)
return min_indexes[0]
batch_size, max_length = reference.get_shape().as_list()
symbol = tf.convert_to_tensor(symbol)
symbol.shape.assert_is_compatible_with([])
# Add symbol at the end of each row, to make sure tf.where works.
tensor = tf.concat(
[reference, tf.tile(symbol[None, None], [batch_size, 1])], axis=1)
index_all_occurrences = tf.where(tf.equal(tensor, symbol))
index_all_occurrences = tf.cast(index_all_occurrences, tf.int32)
# `index_all_occurrences` is a [N, 2] tensor with coordinates of all positions
# of `symbol` in `tensor`. So N will be >= batch size since there can be
# several `symbol` in one row of tensor. We need to take only the position
# of the first occurrence for each row. `segment_min` does that, taking the
# lowest column index for each row index.
index_first_occurrences = tf.segment_min(index_all_occurrences[:, 1],
index_all_occurrences[:, 0])
index_first_occurrences.set_shape([batch_size])
index_first_occurrences = tf.minimum(index_first_occurrences + 1,
max_length)
return index_first_occurrences
def _build_graph(self):
# Let
# N = batch size,
# K = embedding size,
# W = number of negative samples per a user-positive-item pair
# user embedding (N, K)
users = tf.nn.embedding_lookup(self.user_embeddings, self.user_positive_items_pairs[:, 0], name="users")
user_reg = tf.reduce_sum(tf.square(users), 1, name="user_reg")
# positive item embedding (N, K)
pos_items = tf.nn.embedding_lookup(self.item_embeddings, self.user_positive_items_pairs[:, 1], name="pos_items")
pos_reg = tf.reduce_sum(tf.square(pos_items), 1, name="pos_reg")
pos_bias = tf.squeeze(tf.nn.embedding_lookup(self.item_bias, self.user_positive_items_pairs[:, 1], name="pos_bias"))
# positive item to user distance (N)
pos_distances = tf.reduce_sum(tf.multiply(users, pos_items), 1 ) + pos_bias
# negative item embedding (N, K, W)
neg_items = tf.transpose(tf.nn.embedding_lookup(self.item_embeddings, self.negative_samples), (0, 2, 1), name="neg_items")
neg_reg = tf.reduce_sum(tf.square(neg_items), 1, name="neg_reg")
neg_bias = tf.squeeze(tf.nn.embedding_lookup(self.item_bias, self.negative_samples), name="neg_bias")
# distance to negative items (N x W)
distance_to_neg_items = tf.reduce_sum(tf.multiply(tf.expand_dims(users, -1), neg_items), 1) + neg_bias
impostors = tf.multiply(self.negative_flags, (tf.expand_dims(-pos_distances, -1) + distance_to_neg_items + self.margin))
indexes = tf.where(tf.greater(impostors, 0))
self.impostor_num = tf.shape(indexes)[0]
self.impostor_log = tf.nn.moments(impostors, axes=[0, 1])
x_min_y = tf.segment_min(indexes[:, 1], indexes[:, 0])
uni_x, _ = tf.unique(indexes[:,0])
uni_y = tf.nn.embedding_lookup(x_min_y, uni_x)
xy = tf.concat([tf.expand_dims(uni_x, -1), tf.expand_dims(uni_y, -1)], 1)
impostor_xy = tf.gather_nd(impostors, xy)
rank = tf.log((self.n_items - 1) / tf.cast(uni_y + 1, tf.float32))
self.eloss = tf.reduce_sum(tf.clip_by_value(rank * impostor_xy, 0, 10))
self.rloss = tf.reduce_sum(self.alpha * (tf.gather_nd(neg_reg, xy) + tf.nn.embedding_lookup(pos_reg, uni_x) + tf.nn.embedding_lookup(user_reg, uni_x)))
self.loss = (self.eloss + self.rloss) / tf.cast(tf.shape(self.user_positive_items_pairs)[0], tf.float32)
self.optimize = tf.train.AdamOptimizer(self.master_learning_rate).minimize(self.loss, var_list=[self.item_bias, self.item_embeddings, self.user_embeddings])
# (N_USER_IDS, 1, K)
user = tf.expand_dims(tf.nn.embedding_lookup(self.user_embeddings, self.score_user_ids), 1)
# (1, N_ITEM, K)
item = tf.expand_dims(self.item_embeddings, 0)
self.item_scores = tf.reduce_sum(tf.multiply(user, item), 2) + tf.squeeze(self.item_bias)
self.topk = tf.nn.top_k(self.item_scores, self.n_items)
def _comp_f(self):
"""
Encodes all queries (including supporting queries)
:return: encoded queries
"""
with tf.device("/cpu:0"):
max_length = tf.cast(tf.reduce_max(self._length), tf.int32)
context_t = tf.transpose(self._context)
context_t = tf.slice(context_t, [0, 0], tf.pack([max_length, -1]))
embedded = tf.nn.embedding_lookup(self.input_embedding, context_t)
embedded = tf.nn.dropout(embedded, self.keep_prob)
batch_size = tf.shape(self._context)[0]
batch_size_32 = tf.reshape(batch_size, [1])
batch_size_64 = tf.cast(batch_size, tf.int64)
with tf.device(self._device1):
#use other device for backward rnn
with tf.variable_scope("backward"):
min_end = tf.segment_min(self._ends, self._span_context)
init_state = tf.get_variable("init_state", [self._size], initializer=self._init)
init_state = tf.reshape(tf.tile(init_state, batch_size_32), [-1, self._size])
rev_embedded = tf.reverse_sequence(embedded, self._length, 0, 1)
# TIME-MAJOR: [T, B, S]
outs_bw = self._composition_function(rev_embedded, self._length - min_end, init_state)
# reshape to all possible queries for all sequences. Dim[0]=batch_size*(max_length+1).
# "+1" because we include the initial state
outs_bw = tf.reshape(tf.concat(0, [tf.expand_dims(init_state, 0), outs_bw]), [-1, self._size])
# gather respective queries via their lengths-start (because reversed sequence)
lengths_aligned = tf.gather(self._length, self._span_context)
out_bw = tf.gather(outs_bw, (lengths_aligned - self._ends) * batch_size_64 + self._span_context)
with tf.device(self._device2):
with tf.variable_scope("forward"):
#e_inputs = [tf.reshape(e, [-1, self._size]) for e in tf.split(1, self._max_length, embedded)]
max_start = tf.segment_max(self._starts, self._span_context)
init_state = tf.get_variable("init_state", [self._size], initializer=self._init)
init_state = tf.reshape(tf.tile(init_state, batch_size_32), [-1, self._size])
# TIME-MAJOR: [T, B, S]
outs_fw = self._composition_function(embedded, max_start, init_state)
# reshape to all possible queries for all sequences. Dim[0]=batch_size*(max_length+1).
# "+1" because we include the initial state
outs_fw = tf.reshape(tf.concat(0, [tf.expand_dims(init_state, 0), outs_fw]), [-1, self._size])
# gather respective queries via their positions (with offset of batch_size*ends)
out_fw = tf.gather(outs_fw, self._starts * batch_size_64 + self._span_context)
# form query from forward and backward compositions
query = tf.contrib.layers.fully_connected(tf.concat(1, [out_fw, out_bw]), self._size,
activation_fn=None, weights_initializer=None, biases_initializer=None)
query = tf.add_n([query, out_bw, out_fw])
return query
def get_eval_user_model(user_model, indices):
batch_idx = tf.reshape(
tf.slice(indices, begin=[0, 0], size=[tf.shape(indices)[0], 1]), [-1])
step_idx = tf.reshape(
tf.slice(indices, begin=[0, 1], size=[tf.shape(indices)[0], 1]), [-1])
uniq_batch_idx, ori_batch_idx = tf.unique(batch_idx)
min_step_idx = tf.segment_min(step_idx, batch_idx)
min_step_idx = tf.gather(min_step_idx, uniq_batch_idx)
used_indices = tf.concat([
tf.expand_dims(uniq_batch_idx, 1),
tf.expand_dims(min_step_idx, 1),
], 1)
used_model = tf.gather_nd(user_model, used_indices)
return used_model, uniq_batch_idx, ori_batch_idx, step_idx
def test_segment():
seg_ids = tf.constant([0, 1, 1, 2, 2])
x = tf.constant([[2, 5, 3, -5], [0, 3, -2, 5], [4, 3, 5, 3], [6, 1, 4, 0],
[6, 1, 4, 0]])
with tf.Session() as sess:
# 按seg_ids进行加法
print(tf.segment_sum(x, seg_ids).eval())
# 按seg_ids进行乘法
print(tf.segment_prod(x, seg_ids).eval())
# 按seg_ids进行min运算
print(tf.segment_min(x, seg_ids).eval())
# 按seg_ids进行max运算
print(tf.segment_max(x, seg_ids).eval())
# 按seg_ids进行mean运算
print(tf.segment_mean(x, seg_ids).eval())
def _form_triplet(tensors):
"""Returns triplet indices [ij, jk], where r_ij, r_jk < r_c"""
p_iind = tensors['ind_2'][:, 0]
n_atoms = tf.shape(tensors['ind_1'])[0]
n_pairs = tf.shape(tensors['ind_2'])[0]
p_aind = tf.cumsum(tf.ones(n_pairs, tf.int32))
p_rind = p_aind - tf.gather(tf.segment_min(p_aind, p_iind), p_iind)
t_dense = tf.scatter_nd(tf.stack([p_iind, p_rind], axis=1), p_aind,
[n_atoms, tf.reduce_max(p_rind) + 1])
t_dense = tf.gather(t_dense, p_iind)
t_index = tf.cast(tf.where(t_dense), tf.int32)
t_ijind = t_index[:, 0]
t_ikind = tf.gather_nd(t_dense, t_index) - 1
t_ind = tf.gather_nd(tf.stack([t_ijind, t_ikind], axis=1),
tf.where(tf.not_equal(t_ijind, t_ikind)))
return t_ind
def xqa_min_crossentropy_span_loss(candidate_scores, span_candidates, answer_span, answer_to_question):
"""
very common XQA loss function when predicting for entire spans
"""
# align total spans and scores with correct answer spans
span_candidates = tf.gather(span_candidates, answer_to_question)
candidate_scores = tf.gather(candidate_scores, answer_to_question)
# tile correct answers to num spans to find matching span
answer_span_tiled = tf.expand_dims(answer_span, 1)
span_labels = tf.cast(tf.reduce_all(tf.equal(answer_span_tiled, span_candidates), 2), tf.float32)
loss = tf.nn.softmax_cross_entropy_with_logits(logits=candidate_scores, labels=span_labels)
loss = tf.segment_min(loss, answer_to_question)
return [tf.reduce_mean(loss)]
def xqa_min_crossentropy_loss(start_scores, end_scores, answer_span, answer_to_question):
"""
very common XQA loss function
"""
start, end = [tf.squeeze(t, 1) for t in tf.split(answer_span, 2, 1)]
batch_size1 = tf.shape(start)[0]
batch_size2 = tf.unstack(tf.shape(start_scores))[0]
is_aligned = tf.equal(batch_size1, batch_size2)
start_scores = tf.cond(is_aligned, lambda: start_scores, lambda: tf.gather(start_scores, answer_to_question))
end_scores = tf.cond(is_aligned, lambda: end_scores, lambda: tf.gather(end_scores, answer_to_question))
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=start_scores,
labels=start) + \
tf.nn.sparse_softmax_cross_entropy_with_logits(logits=end_scores, labels=end)
loss = tf.segment_min(loss, answer_to_question)
return [tf.reduce_mean(loss)]
def one_bp_iteration_network(self, xe_c2v_pre_iter, xe_0, alpha, beta):
xe_c_sumv = self.staircase_quantizer(
tf.add(tf.matmul(self.H_x_to_xe0, xe_0),
alpha * tf.matmul(self.H_sumV_to_C, xe_c2v_pre_iter)), beta)
xe_sign = tf.to_float(tf.sign(xe_c_sumv))
xe_sum_log_img = tf.matmul(
self.H_sumC_to_V,
tf.multiply(tf.truediv((1 - xe_sign), [2.0]), [3.1415926]))
xe_sum_log_complex = tf.complex(
tf.zeros([self.num_all_edges, self.batch_size]), xe_sum_log_img)
xe_product = tf.real(tf.exp(xe_sum_log_complex))
xe_product_temp = tf.multiply(tf.sign(xe_product), -2e-7)
xe_pd_modified = tf.add(xe_product, xe_product_temp)
xe_v_minc = tf.segment_min(
tf.abs(tf.gather(xe_c_sumv, self.h_minC_to_V)),
self.h_min_segmentIDs)
xe_v_sumc = tf.multiply(xe_pd_modified, xe_v_minc)
return xe_v_sumc
def get_sentence_length(raw_x):
"""Extracts sequence lengths from the raw feature.
Example:
raw_x = [
[123, 3, 2, 0, 0],
[321, 1, 0, 0, 0]]
return:
[3, 2]
Args:
raw_x: A [batch_size, max_length] int32 tensor
Returns:
A [batch_size] int32 tensor with sequence lengths
"""
not_padding = tf.not_equal(raw_x, PAD_ID)
not_padding_with_guardian = tf.pad(not_padding, [[0, 0], [0, 1]])
indices = tf.where(tf.logical_not(not_padding_with_guardian))
length = tf.segment_min(indices[:, 1], indices[:, 0])
return length
def dense_to_sparse(tensor, eos_id, merge_repeated=True):
if merge_repeated:
added_values = tf.cast(tf.fill((tf.shape(tensor)[0], 1), eos_id), tensor.dtype)
# merge consecutive values
concat_tensor = tf.concat((tensor, added_values), axis=-1)
diff = tf.cast(concat_tensor[:, 1:] - concat_tensor[:, :-1], tf.bool)
# trim after first eos token
eos_indices = tf.where(tf.equal(concat_tensor, eos_id))
first_eos = tf.segment_min(eos_indices[:, 1], eos_indices[:, 0])
mask = tf.sequence_mask(first_eos, maxlen=tf.shape(tensor)[1])
indices = tf.where(diff & mask & tf.not_equal(tensor, -1))
values = tf.gather_nd(tensor, indices)
shape = tf.shape(tensor, out_type=tf.int64)
return tf.SparseTensor(indices, values, shape)
else:
return tf.contrib.layers.dense_to_sparse(tensor, eos_id)
def one_bp_iteration(self, xe_c2v_pre_iter, xe_0, llr_into_bp_net,
H_sumC_to_V, h_minC_to_V, h_min_segmentIDs,
H_sumV_to_C, H_xe_v_sumc_to_y):
quantizer_index, syndrome_weight = self.get_quantizer_index(
xe_c2v_pre_iter, H_xe_v_sumc_to_y, llr_into_bp_net)
xe_c_sumv_nonq = tf.add(xe_0, tf.matmul(H_sumV_to_C, xe_c2v_pre_iter))
xe_c_sumv = self.quantizer(
tf.add(xe_0, tf.matmul(H_sumV_to_C, xe_c2v_pre_iter)),
quantizer_index)
xe_sign = tf.to_float(tf.sign(xe_c_sumv))
xe_sum_log_img = tf.matmul(
H_sumC_to_V,
tf.multiply(tf.truediv((1 - xe_sign), [2.0]), [3.1415926]))
xe_sum_log_complex = tf.complex(
tf.zeros([self.num_all_edges, self.batch_size]), xe_sum_log_img)
xe_product = tf.real(tf.exp(xe_sum_log_complex))
xe_product_temp = tf.multiply(tf.sign(xe_product), -2e-7)
xe_pd_modified = tf.add(xe_product, xe_product_temp)
xe_v_minc = tf.segment_min(tf.abs(tf.gather(xe_c_sumv, h_minC_to_V)),
h_min_segmentIDs)
xe_v_sumc = tf.multiply(xe_pd_modified, xe_v_minc)
return xe_v_sumc, quantizer_index, syndrome_weight, xe_c_sumv_nonq, xe_c_sumv
def dense_to_sparse(tensor, eos_id):
'''
Convert tensor to a specific sparse format
Because when calculating tf.edit_distance, only sparse tensor is received
'''
added_values = tf.cast(tf.fill((tf.shape(tensor)[0], 1), eos_id),
tensor.dtype)
#Add eos to the entire tensor
concat_tensor = tf.concat((tensor, added_values), axis=-1)
#Find duplicate phonemes
diff = tf.cast(concat_tensor[:, 1:] - concat_tensor[:, :-1], tf.bool)
eos_indices = tf.where(tf.equal(concat_tensor, eos_id))
#Find the position of the first eos in each decoded sequence
first_eos = tf.segment_min(eos_indices[:, 1], eos_indices[:, 0])
#
mask = tf.sequence_mask(first_eos, maxlen=tf.shape(tensor)[1])
indices = tf.where(diff & mask & tf.not_equal(tensor, -1))
values = tf.gather_nd(tensor, indices)
shape = tf.shape(tensor, out_type=tf.int64)
return tf.SparseTensor(indices, values, shape)
def one_bp_iteration(self, xe_v2c_pre_iter, H_sumC_to_V, H_sumV_to_C,
h_minC_to_V, h_min_segmentIDs, xe_0):
xe_sign = tf.to_float(tf.sign(xe_v2c_pre_iter))
xe_sum_log_img = tf.matmul(
H_sumC_to_V,
tf.multiply(tf.truediv((1 - xe_sign), [2.0]), [3.1415926]))
xe_sum_log_complex = tf.complex(
tf.zeros([self.num_all_edges, self.batch_size]), xe_sum_log_img)
xe_product = tf.real(tf.exp(xe_sum_log_complex))
xe_product_temp = tf.multiply(tf.sign(xe_product), -2e-7)
xe_pd_modified = tf.add(xe_product, xe_product_temp)
xe_v_minc = tf.zeros([self.num_all_edges, self.batch_size],
dtype=tf.float32)
xe_v_minc = tf.segment_min(
tf.abs(tf.gather(xe_v2c_pre_iter, h_minC_to_V)), h_min_segmentIDs)
xe_v_sumc = tf.multiply(
xe_pd_modified,
tf.maximum((self.alpha * xe_v_minc + self.beta),
tf.zeros([self.num_all_edges, self.batch_size],
dtype=tf.float32)))
xe_c_sumv = tf.add(xe_0, tf.matmul(H_sumV_to_C, xe_v_sumc))
return xe_v_sumc, xe_c_sumv
def _calculate_marginal_x_by_y_axis(self, indices):
xs = indices[..., 1]
ys = indices[..., 0]
x_mins = tf.segment_min(xs, ys)
x_maxs = tf.segment_max(xs, ys)
y_pos = tf.range(0, tf.shape(x_mins)[0])
y_pos = tf.cast(y_pos, tf.int64)
left_marginal = tf.gather_nd(tf.stack([y_pos, x_mins], axis=-1),
tf.where(tf.not_equal(x_mins, x_maxs)))
right_marginal = tf.gather_nd(tf.stack([y_pos, x_maxs], axis=-1),
tf.where(tf.not_equal(x_mins, x_maxs)))
# 노이즈을 줄이기 위해 앞뒤 15% drop
valid_counts = tf.cast(tf.shape(left_marginal)[0], tf.float32)
drop_counts = tf.clip_by_value(tf.cast(valid_counts * 0.15, tf.int32),
1, 2**31)
left_marginal = tf.cast(left_marginal[drop_counts:-drop_counts],
tf.float32)
right_marginal = tf.cast(right_marginal[drop_counts:-drop_counts],
tf.float32)
return left_marginal, right_marginal
def _get_constrained_steps(self, indices, mode):
batch_idx = tf.reshape(
tf.slice(indices, begin=[0, 0], size=[tf.shape(indices)[0], 1]),
[-1])
uniq_batch_idx, ori_batch_idx = tf.unique(batch_idx)
step_idx = tf.reshape(
tf.slice(indices, begin=[0, 1], size=[tf.shape(indices)[0], 1]),
[-1])
start_limit = None
end_limit = None
if mode == 'train':
split_limit = tf.gather(tf.segment_max(step_idx, batch_idx),
uniq_batch_idx)
end_limit = tf.gather(split_limit, ori_batch_idx)
start_limit = end_limit - self._train_steps + 1
elif mode == 'eval':
split_limit = tf.gather(tf.segment_min(step_idx, batch_idx),
uniq_batch_idx)
start_limit = tf.gather(split_limit, ori_batch_idx)
end_limit = start_limit + self._eval_steps - 1
return tf.boolean_mask(
indices,
tf.logical_and(step_idx >= start_limit, step_idx <= end_limit))
def test_SegmentMin(self):
t = tf.segment_min(self.random(4, 2, 3), np.array([0, 1, 1, 2]))
self.check(t)
import os
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
import tensorflow as tf
sess = tf.InteractiveSession()
seg_ids = tf.constant([0,1,1,2,2]) # Group indexes : 0|1,2|3,4
tens1 = tf.constant([[2, 5, 3, -5],
[0, 3,-2, 5],
[4, 3, 5, 3],
[6, 1, 4, 0],
[6, 1, 4, 0]]) # A sample constant m
print('\nseg_ids->', seg_ids.eval())
print('tens1->', tens1.eval())
print("\ntf.segment_sum(tens1, seg_ids).eval() ") # Sum segmen
print(tf.segment_sum(tens1, seg_ids).eval() ) # Sum segmen
print("\ntf.segment_prod(tens1, seg_ids).eval() ") # Product segmen
print(tf.segment_prod(tens1, seg_ids).eval() ) # Product segmen
print(tf.segment_min(tens1, seg_ids).eval() ) # minimun value goes to
print(tf.segment_max(tens1, seg_ids).eval() ) # maximum value goes to
print(tf.segment_mean(tens1, seg_ids).eval() ) # mean value goes to group
import tensorflow as tf
import numpy as np
input_a = np.array([[1, 1, 2], [2, 3, 4], [3, 1, 1], [2, 4, 6]])
a_seg_sum = tf.segment_sum(data=input_a, segment_ids=[0, 1, 1, 1])
a_seg_prod = tf.segment_prod(data=input_a, segment_ids=[0, 0, 1, 1])
a_seg_max = tf.segment_max(data=input_a, segment_ids=[0, 0, 0, 1])
a_seg_min = tf.segment_min(data=input_a, segment_ids=[1, 1, 1, 1])
a_seg_mean = tf.segment_mean(data=input_a, segment_ids=[0, 0, 0, 1])
a_seg_sum_num = tf.unsorted_segment_sum(data=input_a,
segment_ids=[0, 1, 1, 0],
num_segments=2)
a_sparse_seg_sum = tf.sparse_segment_sum(data=input_a,
indices=[0, 1, 2],
segment_ids=[0, 0, 1])
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
print(sess.run(a_seg_sum), '\n', sess.run(a_seg_prod), '\n',
sess.run(a_seg_max), '\n', sess.run(a_seg_min))
print(sess.run(a_seg_mean), '\n', sess.run(a_seg_sum_num), '\n',
sess.run(a_sparse_seg_sum))
#Segmentation Examples
import tensorflow as tf
sess = tf.InteractiveSession()
seg_ids = tf.constant([0,1,1,2,2]); # Group indexes : 0|1,2|3,4
tens1 = tf.constant([[2, 5, 3, -5],
[0, 3,-2, 5],
[4, 3, 5, 3],
[6, 1, 4, 0],
[6, 1, 4, 0]]) # A sample constant matrix
tf.segment_sum(tens1, seg_ids).eval() # Sum segmentation
tf.segment_prod(tens1, seg_ids).eval() # Product segmantation
tf.segment_min(tens1, seg_ids).eval() # minimun value goes to group
tf.segment_max(tens1, seg_ids).eval() # maximum value goes to group
tf.segment_mean(tens1, seg_ids).eval() # mean value goes to group
tf.cumsum([a, b, c], reverse=True) # => [a + b + c, b + c, c] tf.cumsum([a, b, c], exclusive=True, reverse=True) # => [b + c, c, 0] # Computes the sum/mean/max/min/prod along segments of a tensor tf.segment_sum(data, segment_ids, name=None) # Eg: m = tf.constant([5,1,7,2,3,4,1,3]) s_id = [0,0,0,1,2,2,3,3] s.run(tf.segment_sum(m, segment_ids=s_id)) """ >array([13, 2, 7, 4], dtype=int32) """ tf.segment_mean(data, segment_ids, name=None) tf.segment_max(data, segment_ids, name=None) tf.segment_min(data, segment_ids, name=None) tf.segment_prod(data, segment_ids, name=None) # 其它 tf.unsorted_segment_sum tf.sparse_segment_sum tf.sparse_segment_mean tf.sparse_segment_sqrt_n # 比较两个 list 或者 string 的不同，并返回不同的值和索引 tf.setdiff1d(x, y, index_dtype=tf.int32, name=None) # 返回 x 中的唯一值所组成的tensor 和原 tensor 中元素在现 tensor 中的索引 tf.unique(x, out_idx=None, name=None) # x if condition else y, condition 为 bool 类型的，可用tf.equal()等来表示
def worker(cluster, n_items, embed_dim, worker_n, epoch, alpha,
master_learning_rate, sampler, evaluator, num_batch, num_user):
per_item_embedding = int(
np.ceil(embed_dim * 1.0 / len(cluster.job_tasks('ps'))))
per_item_bias = int(np.ceil(n_items * 1.0 / len(cluster.job_tasks('ps'))))
item_embedding_chunk = list(chunks(range(embed_dim), per_item_embedding))
item_bias_chunk = list(chunks(range(n_items), per_item_bias))
item_embedding_list = []
item_bias_list = []
for ps_n in range(len(cluster.job_tasks('ps'))):
with tf.device("/job:ps/task:%s/cpu:0" % ps_n):
item_embedding_list.append(
tf.Variable(tf.random_normal([
n_items, item_embedding_chunk[ps_n][-1] -
item_embedding_chunk[ps_n][0] + 1
],
stddev=1 / (embed_dim**0.5),
dtype=tf.float32),
name="ps_item_embeddings_%s" % ps_n))
item_bias_list.append(
tf.Variable(tf.random_normal([
item_bias_chunk[ps_n][-1] - item_bias_chunk[ps_n][0] + 1, 1
],
stddev=1 / (embed_dim**0.5),
dtype=tf.float32),
name="ps_item_bias_%s" % ps_n))
with tf.device("/job:ps/task:0/cpu:0"):
task_queue = tf.FIFOQueue(
len(cluster.job_tasks('ps') + cluster.job_tasks('worker')),
[tf.bool],
shared_name='ps_task_queue')
with tf.device("/job:worker/task:%s" % worker_n):
item_embedding_copy_list = [
tf.identity(item) for item in item_embedding_list
]
item_bias_copy_list = [tf.identity(item) for item in item_bias_list]
item_embeddings_copy = tf.concat(item_embedding_copy_list, 1)
item_bias_copy = tf.concat(item_bias_copy_list, 0)
user_positive_items_pairs = tf.placeholder(tf.int32, [None, 2])
negative_samples = tf.placeholder(tf.int32, [None, None])
negative_flags = tf.placeholder(tf.float32, [None, None])
score_user_ids = tf.placeholder(tf.int32, [None])
user_embeddings = tf.Variable(
tf.random_normal([num_user, embed_dim],
stddev=1 / (embed_dim**0.5),
dtype=tf.float32),
name="worker_%s_user_embeddings" % worker_n)
# N = batch size,
# K = embedding size,
# W = number of negative samples per a user-positive-item pair
# user embedding (N, K)
users = tf.nn.embedding_lookup(user_embeddings,
user_positive_items_pairs[:, 0],
name="worker_%s_users" % worker_n)
user_reg = tf.reduce_sum(tf.square(users),
1,
name="worker_%s_user_reg" % worker_n)
# positive item embedding (N, K)
pos_items = tf.nn.embedding_lookup(item_embeddings_copy,
user_positive_items_pairs[:, 1])
pos_reg = tf.reduce_sum(tf.square(pos_items), 1)
pos_bias = tf.squeeze(
tf.nn.embedding_lookup(item_bias_copy,
user_positive_items_pairs[:, 1]))
# positive item to user distance (N)
pos_distances = tf.reduce_sum(tf.multiply(users, pos_items),
1) + pos_bias
# negative item embedding (N, K, W)
neg_items = tf.transpose(
tf.nn.embedding_lookup(item_embeddings_copy, negative_samples),
(0, 2, 1))
neg_reg = tf.reduce_sum(tf.square(neg_items), 1)
neg_bias = tf.squeeze(
tf.nn.embedding_lookup(item_bias_copy, negative_samples))
# distance to negative items (N x W)
distance_to_neg_items = tf.reduce_sum(
tf.multiply(tf.expand_dims(users, -1), neg_items), 1) + neg_bias
impostors = tf.multiply(
negative_flags,
(tf.expand_dims(-pos_distances, -1) + distance_to_neg_items + 1))
indexes = tf.where(tf.greater(impostors, 0))
impostor_num = tf.shape(indexes)[0]
impostor_log = tf.nn.moments(impostors, axes=[0, 1])
x_min_y = tf.segment_min(indexes[:, 1], indexes[:, 0])
uni_x, _ = tf.unique(indexes[:, 0])
uni_y = tf.nn.embedding_lookup(x_min_y, uni_x)
xy = tf.concat([tf.expand_dims(uni_x, -1),
tf.expand_dims(uni_y, -1)], 1)
impostor_xy = tf.gather_nd(impostors, xy)
rank = tf.log((n_items - 1) / tf.cast(uni_y + 1, tf.float32))
eloss = tf.reduce_sum(tf.clip_by_value(rank * impostor_xy, 0, 10))
rloss = tf.reduce_sum(
alpha * (tf.gather_nd(neg_reg, xy) + tf.nn.embedding_lookup(
pos_reg, uni_x) + tf.nn.embedding_lookup(user_reg, uni_x)))
loss = (eloss + rloss) / tf.cast(
tf.shape(user_positive_items_pairs)[0], tf.float32)
optimizer = tf.train.AdamOptimizer(master_learning_rate)
grads_and_vars = optimizer.compute_gradients(loss)
dense_grads_and_vars = [(tf.convert_to_tensor(grad), var)
for grad, var in grads_and_vars]
opt = optimizer.apply_gradients(dense_grads_and_vars)
item_scores = tf.reduce_sum(
tf.multiply(
tf.expand_dims(
tf.nn.embedding_lookup(user_embeddings, score_user_ids),
1), tf.expand_dims(item_embeddings_copy, 0)),
2) + tf.squeeze(item_bias_copy)
topk = tf.nn.top_k(item_scores, n_items)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
print("Worker %d: start server" % worker_n)
server = tf.train.Server(cluster,
job_name="worker",
task_index=worker_n,
config=config,
protocol="grpc+verbs")
print("Worker %d: start session" % worker_n)
sess = tf.Session(target=server.target)
print("Worker %d: initializing variables; free memory: %s" %
(worker_n, free_memory()))
sess.run(tf.global_variables_initializer())
sess.run(task_queue.enqueue(True))
while sess.run(task_queue.size()) != len(
cluster.job_tasks('ps') + cluster.job_tasks('worker')):
print("Worker %s: waiting..." % worker_n)
sleep(1)
print("Worker %d: variables initialized; free memory: %s" %
(worker_n, free_memory()))
_epoch = 0
while _epoch < epoch:
_losses, _users, _tops = [], evaluator.users(), []
for _ in tqdm(range(num_batch), desc="Optimizing...", file=sys.stdout):
user_pos, neg, flags = sampler.next_batch()
#_, loss= sess.run((opt, loss), {user_positive_items_pairs: user_pos, negative_samples: neg, negative_flags: flags})
#options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
#run_metadata = tf.RunMetadata()
_, _impostor_num, _impostor_log, _loss, _eloss, _rloss = sess.run(
[opt, impostor_num, impostor_log, loss, eloss, rloss],
{
user_positive_items_pairs: user_pos,
negative_samples: neg,
negative_flags: flags
},
# options=options, run_metadata=run_metadata
)
print("Worker %d: training" % worker_n, _impostor_num,
_impostor_log, _loss, _eloss, _rloss)
#fetched_timeline = timeline.Timeline(run_metadata.step_stats)
#chrome_trace = fetched_timeline.generate_chrome_trace_format()
#with open(os.path.join(output_path, 'timeline_%s.json' % worker_n), 'w') as f:
# f.write(chrome_trace)
#for device in run_metadata.step_stats.dev_stats:
# print(device.device)
# for node in device.node_stats:
# print(" ", node.node_name)
_losses.append(_loss)
print("Worker %d: training" % worker_n, free_memory())
for chunk in chunks(_users, 100):
_, _top = sess.run(topk, {score_user_ids: chunk})
_tops.extend(_top)
print("Worker %s: epoch: %s" % (worker_n, _epoch), free_memory(),
np.mean(_losses), 50, evaluator.eval(zip(_users, _tops), 50),
evaluator.eval(zip(_users, _tops), 50, 'recall'))
_epoch += 1
sess.run(task_queue.dequeue())
sampler.close()
print("Worker %d: done" % worker_n)
def cal_out():
r, ind = tf.nn.top_k(self.input, self.n_in)
r_ind = tf.reverse(ind, [0])
nx = tf.gather(self.input, r_ind)
nW = tf.transpose(tf.gather(self.W, r_ind))
nxW = tf.multiply(nx, nW)
def body_z(i, z):
z = tf.slice(
tf.concat([
tf.cast(
tf.reduce_sum(
tf.slice(nxW, [0, 0], [self.n_out, i + 1]), 1,
True), tf.float32), z
], 1), [0, 0], [self.n_out, self.n_in])
return [i + 1, z]
def body_W(i, z):
z = tf.slice(
tf.concat([
tf.cast(
tf.reduce_sum(
tf.slice(nW, [0, 0], [self.n_out, i + 1]), 1,
True), tf.float32), z
], 1), [0, 0], [self.n_out, self.n_in])
return [i + 1, z]
def condition(i, z):
return i < self.n_in
r1, n_sum_z = tf.while_loop(condition, body_z,
[self.i, self.sum_z])
r2, n_sum_W = tf.while_loop(condition, body_W,
[self.i, self.sum_W])
f_sum_z = tf.reverse(n_sum_z, [1])
f_sum_W = tf.reverse(n_sum_W, [1])
out_all = tf.divide(f_sum_z, tf.subtract(f_sum_W, 1))
out_all_2 = tf.concat([
out_all,
tf.transpose([
tf.tile([tf.divide(self.c_one, self.c_zero)], [self.n_out])
])
], 1)
out_ok = tf.where(
tf.logical_and(
tf.less(
tf.cast(
tf.tile([tf.concat([nx, [1]], 0)],
[self.n_out, 1]), tf.float32), out_all_2),
tf.greater_equal(
tf.cast(
tf.tile([
tf.slice(
tf.concat([
nx,
[
tf.divide(self.c_one, self.c_zero),
tf.divide(self.c_one, self.c_zero)
]
], 0), [1], [self.n_in + 1])
], [self.n_out, 1]), tf.float32), out_all_2)))
out_idx = tf.transpose(
tf.concat(
[[tf.range(0, self.n_out)],
[
tf.cast(tf.segment_min(out_ok[:, 1], out_ok[:, 0]),
tf.int32)
]], 0))
out = tf.gather_nd(out_all_2, out_idx)
output = tf.where(out > 1e5, tf.multiply(tf.ones_like(out), 1e5),
out)
return output
def get_finised_pos(token_seq, finished_index, max_length): tmp_indices = tf.where(tf.equal(token_seq, int(finished_index))) finished_pos = tf.segment_min(tmp_indices[:, 1], tmp_indices[:, 0]) sequence_mask = tf.sequence_mask(finished_pos+1, maxlen=max_length) return tf.cast(sequence_mask, tf.int32)
import tensorflow as tf sess = tf.InteractiveSession() seg_ids = tf.constant([0, 1, 1, 2, 2]) #下标 tens1 = tf.constant([[2, 5, 3, -5], [0, 3, -2, 5], [6, 1, 4, 0], [6, 1, 4, 0]]) print(tf.segment_sum(tens1, seg_ids).eval()) print(tf.segment_prod(tens1, seg_ids).eval()) print(tf.segment_min(tens1, seg_ids).eval()) print(tf.segment_max(tens1, seg_ids).eval()) print(tf.segment_mean(tens1, seg_ids).eval())
def _comp_f(self):
"""
Encodes all queries (including supporting queries)
:return: encoded queries
"""
with tf.device("/cpu:0"):
max_length = tf.cast(tf.reduce_max(self._length), tf.int32)
context_t = tf.transpose(self._context)
context_t = tf.slice(context_t, [0, 0], tf.pack([max_length, -1]))
embedded = tf.nn.embedding_lookup(self.input_embedding, context_t)
embedded = tf.nn.dropout(embedded, self.keep_prob)
batch_size = tf.shape(self._context)[0]
batch_size_32 = tf.reshape(batch_size, [1])
batch_size_64 = tf.cast(batch_size, tf.int64)
with tf.device(self._device1):
#use other device for backward rnn
with tf.variable_scope("backward"):
min_end = tf.segment_min(self._ends, self._span_context)
init_state = tf.get_variable("init_state", [self._size],
initializer=self._init)
init_state = tf.reshape(tf.tile(init_state, batch_size_32),
[-1, self._size])
rev_embedded = tf.reverse_sequence(embedded, self._length, 0,
1)
# TIME-MAJOR: [T, B, S]
outs_bw = self._composition_function(rev_embedded,
self._length - min_end,
init_state)
# reshape to all possible queries for all sequences. Dim[0]=batch_size*(max_length+1).
# "+1" because we include the initial state
outs_bw = tf.reshape(
tf.concat(0, [tf.expand_dims(init_state, 0), outs_bw]),
[-1, self._size])
# gather respective queries via their lengths-start (because reversed sequence)
lengths_aligned = tf.gather(self._length, self._span_context)
out_bw = tf.gather(
outs_bw, (lengths_aligned - self._ends) * batch_size_64 +
self._span_context)
with tf.device(self._device2):
with tf.variable_scope("forward"):
#e_inputs = [tf.reshape(e, [-1, self._size]) for e in tf.split(1, self._max_length, embedded)]
max_start = tf.segment_max(self._starts, self._span_context)
init_state = tf.get_variable("init_state", [self._size],
initializer=self._init)
init_state = tf.reshape(tf.tile(init_state, batch_size_32),
[-1, self._size])
# TIME-MAJOR: [T, B, S]
outs_fw = self._composition_function(embedded, max_start,
init_state)
# reshape to all possible queries for all sequences. Dim[0]=batch_size*(max_length+1).
# "+1" because we include the initial state
outs_fw = tf.reshape(
tf.concat(0, [tf.expand_dims(init_state, 0), outs_fw]),
[-1, self._size])
# gather respective queries via their positions (with offset of batch_size*ends)
out_fw = tf.gather(
outs_fw, self._starts * batch_size_64 + self._span_context)
# form query from forward and backward compositions
query = tf.contrib.layers.fully_connected(tf.concat(
1, [out_fw, out_bw]),
self._size,
activation_fn=None,
weights_initializer=None,
biases_initializer=None)
query = tf.add_n([query, out_bw, out_fw])
return query
def test_SegmentMin(self):
t = tf.segment_min(self.random(4, 2, 3), np.array([0, 1, 1, 2]))
self.check(t)
