def _get_coordinatewise_learning_rate(self, grad, var):
# Compute the learning rate using a moving average for the diagonal of BB^T
avg_first = self.get_slot(var, 'first_moment')
avg_second = self.get_slot(var, 'second_moment')
decay_tensor = tf.cast(self._decay_tensor, var.dtype)
batch_size = tf.cast(self._batch_size_tensor, var.dtype)
# Create an estimator for the moving average of gradient mean and variance
# via Welford's algorithm
if isinstance(grad, tf.Tensor):
delta = grad - avg_first
first_moment_update = avg_first.assign_add(
delta * tf.where(self._counter < 1,
tf.cast(1, var.dtype),
1. - decay_tensor))
with tf.control_dependencies([first_moment_update]):
second_moment_update = avg_second.assign_add(
tf.cast(self._counter < 1, var.dtype) *
-(1. - decay_tensor) * (
avg_second - decay_tensor * tf.square(delta)))
diag_preconditioner = control_flow_ops.with_dependencies(
[second_moment_update],
tf.clip_by_value(avg_second, 1e-12, 1e12))
elif isinstance(grad, tf.IndexedSlices):
delta = grad.values - tf.gather_nd(avg_first, grad.indices)
first_moment_update = tf.scatter_add(
avg_first,
grad.indices,
delta * tf.where(self._counter < 1,
tf.cast(1., var.dtype),
1. - decay_tensor))
with tf.control_dependencies([first_moment_update]):
avg_second = tf.scatter_add(
avg_second,
grad.indices,
tf.cast(self._counter < 1, var.dtype) *
-(1. - decay_tensor) * (
tf.gather_nd(avg_second, grad.indices) - decay_tensor *
tf.square(delta)))
avg_second = tf.gather_nd(avg_second, grad.indices)
# TODO(b/70783772): Needs dtype specific clipping.
diag_preconditioner = tf.clip_by_value(avg_second, 1e-12, 1e12)
else:
raise tf.errors.InvalidArgumentError(
None, None, 'grad must of type Tensor or IndexedSlice')
diag_preconditioner *= batch_size
if self._use_single_learning_rate:
diag_preconditioner = tf.reduce_mean(diag_preconditioner)
# From Theorem 2 Corollary 1 of Mandt et al. 2017
return 2. * batch_size / (
tf.cast(self._total_num_examples, var.dtype.base_dtype) *
diag_preconditioner)
def forward_prop(*parms):
(i0, sz, cur, act) = parms
with tf.device('/gpu:0'):
gact = act
gcur = cur
next_idx = i0 + gcur
node_out = tf.reshape(forward_node(next_idx, act, offset), [1, FLAGS.wvs, 1], name='node_out')
tf.scatter_add(gact, tf.pack([gcur]), node_out, name='act_update')
act = gact
return [i0, sz, cur + iONE, act]
def testWrongShape(self):
# Indices and values mismatch.
var = tf.Variable(tf.zeros(shape=[1024, 64, 64], dtype=tf.float32))
indices = tf.placeholder(tf.int32, shape=[32])
values = tf.placeholder(tf.float32, shape=[33, 64, 64])
with self.assertRaises(ValueError):
tf.scatter_add(var, indices, values)
# Var and values mismatch.
values = tf.placeholder(tf.float32, shape=[32, 64, 63])
with self.assertRaises(ValueError):
tf.scatter_add(var, indices, values)
def add_pointer(context_c, context_m, scores_c, scores_m,
context_distribution, state_t):
p_gen = tf.nn.sigmoid(
tf.matmul(context_c, W_hc) + tf.matmul(context_m, W_hm) +
tf.matmul(state_t[0], W_sc) + tf.matmul(state_t[1], W_sm))
context_distribution = tf.multiply(
tf.concat(1, [
context_distribution,
tf.zeros(shape=(self.config.batch_size,
self.config.max_enc_length))
]), p_gen)
scores = tf.multiply(scores_c + scores_m, 0.5 * (1 - p_gen))
output = []
init = np.array([
0 for i in range(self.config.vocab_size +
self.config.max_enc_length)
],
dtype=np.float32)
for i in range(self.config.batch_size):
'''
indices = self.input_placeholder[i,:]
values = scores[i,:]
indices, idx = tf.unique(indices)
values = tf.segment_sum(values, idx)
output.append(tf.sparse_to_dense(sparse_indices = indices, output_shape=(self.config.vocab_size + self.config.max_enc_length,), sparse_values = values, validate_indices = False))
'''
curr_dist = tf.Variable(init)
#curr_dist = tf.zeros((self.config.vocab_size + self.config.max_enc_length,), tf.float32)
curr_dist = tf.scatter_add(curr_dist,
self.input_placeholder[i, :],
scores[i, :])
output.append(curr_dist)
return context_distribution + tf.stack(output, axis=0)
def outer_product(*inputs):
"""Computes outer product.
Args:
inputs: a list of 1-D `Tensor` (vector)
"""
inputs = list(inputs)
order = len(inputs)
for idx, input_ in enumerate(inputs):
if len(input_.get_shape()) == 1:
inputs[idx] = tf.reshape(input_,
[-1, 1] if idx % 2 == 0 else [1, -1])
if order == 2:
output = tf.mul(inputs[0], inputs[1])
elif order == 3:
size = []
idx = 1
for i in xrange(order):
size.append(inputs[i].get_shape()[0])
output = tf.zeros(size)
u, v, w = inputs[0], inputs[1], inputs[2]
uv = tf.mul(inputs[0], inputs[1])
for i in xrange(self.size[-1]):
output = tf.scatter_add(output, [0, 0, i], uv)
return output
def make_update_op(self, upd_idxs, upd_keys, upd_vals, batch_size,
use_recent_idx, intended_output):
"""Function that creates all the update ops."""
base_update_op = super(LSHMemory,
self).make_update_op(upd_idxs, upd_keys,
upd_vals, batch_size,
use_recent_idx,
intended_output)
# compute hash slots to be updated
hash_slot_idxs = self.get_hash_slots(upd_keys)
# make updates
update_ops = []
with tf.control_dependencies([base_update_op]):
for i, slot_idxs in enumerate(hash_slot_idxs):
# for each slot, choose which entry to replace
entry_idx = tf.random_uniform([batch_size],
maxval=self.num_per_hash_slot,
dtype=tf.int32)
entry_mul = 1 - tf.one_hot(
entry_idx, self.num_per_hash_slot, dtype=tf.int32)
entry_add = (tf.expand_dims(upd_idxs, 1) * tf.one_hot(
entry_idx, self.num_per_hash_slot, dtype=tf.int32))
mul_op = tf.scatter_mul(self.hash_slots[i], slot_idxs,
entry_mul)
with tf.control_dependencies([mul_op]):
add_op = tf.scatter_add(self.hash_slots[i], slot_idxs,
entry_add)
update_ops.append(add_op)
return tf.group(*update_ops)
def body(tf_content_encodings, level, copy_score, output, i):
# find value for write
# which is "copy" tensor
##################################
current_content = tf_content_encodings[i] # single content sample in batch
current_level = level[i] # single level sample in batch
current_copy_score_vector = copy_score[i]
# find the copy indices to generate the s_t^copy vector
c, _ = tf.setdiff1d(current_content, current_level) # test should be content and target should be level
output_level_index, true_content_index = tf.setdiff1d(current_content, c)
# true_content_index is the index to extract copy score...
# output_level_index is the index to place the copy score
# Use the gather and the scatter_add mechanism to obtain the final s_t^copy vector
out_value_list = tf.gather(current_copy_score_vector, true_content_index)
final_output = tf.scatter_add(
# This reference must not be added to the optimizer's backpropagation ops
# so turn the trainable property off.
# Also, since the initial_value is a lambda, the dtype has to be explicitly mentioned
tf.Variable(lambda: tf.zeros(shape=(content_label_vocab_size)), trainable=False, dtype=tf.float32),
output_level_index, out_value_list)
#################################
w_v = final_output
output = output.write(i,w_v)
return tf_content_encodings, level,copy_score, output, i + 1
def make_update_op(self, upd_idxs, upd_keys, upd_vals,
batch_size, use_recent_idx, intended_output):
"""Function that creates all the update ops."""
base_update_op = super(LSHMemory, self).make_update_op(
upd_idxs, upd_keys, upd_vals,
batch_size, use_recent_idx, intended_output)
# compute hash slots to be updated
hash_slot_idxs = self.get_hash_slots(upd_keys)
# make updates
update_ops = []
with tf.control_dependencies([base_update_op]):
for i, slot_idxs in enumerate(hash_slot_idxs):
# for each slot, choose which entry to replace
entry_idx = tf.random_uniform([batch_size],
maxval=self.num_per_hash_slot,
dtype=tf.int32)
entry_mul = 1 - tf.one_hot(entry_idx, self.num_per_hash_slot,
dtype=tf.int32)
entry_add = (tf.expand_dims(upd_idxs, 1) *
tf.one_hot(entry_idx, self.num_per_hash_slot,
dtype=tf.int32))
mul_op = tf.scatter_mul(self.hash_slots[i], slot_idxs, entry_mul)
with tf.control_dependencies([mul_op]):
add_op = tf.scatter_add(self.hash_slots[i], slot_idxs, entry_add)
update_ops.append(add_op)
return tf.group(*update_ops)
def outer_product(*inputs):
"""Computes outer product.
Args:
inputs: a list of 1-D `Tensor` (vector)
"""
inputs = list(inputs)
order = len(inputs)
for idx, input_ in enumerate(inputs):
if len(input_.get_shape()) == 1:
inputs[idx] = tf.reshape(input_, [-1, 1] if idx % 2 == 0 else [1, -1])
if order == 2:
output = tf.mul(inputs[0], inputs[1])
elif order == 3:
size = []
idx = 1
for i in xrange(order):
size.append(inputs[i].get_shape()[0])
output = tf.zeros(size)
u, v, w = inputs[0], inputs[1], inputs[2]
uv = tf.mul(inputs[0], inputs[1])
for i in xrange(self.size[-1]):
output = tf.scatter_add(output, [0,0,i], uv)
return output
def forces_ext_brute_idx_multi(links, id):
""" Generate extra tensors for link external force calculation.
it has a placeholder links.f_ext_idx_[id] for indexing,
and defines a tensor links.force_ext_app[id], both unique to this each instance,
"""
idx = links.force_ext_idx_[id]
x = tf.gather(links.points, idx)
# all possible seg pairs
th0 = tf.gather(links.thickness, idx)
th_mat = th0 + th0[:, newaxis]
A = links.amplitude
links.r = x - x[:, newaxis]
rlen = vec_len(links.r)
# !!! must exclude pairs on same edge, otherwise edge won't contract
fmat = A*links.r*((rlen/th_mat)**(POW-2)/th_mat*tf.exp(-(rlen/th_mat)**POW)\
*links.self_mask_idx_multi[id])[:,:,newaxis]
# including selfrepulsion again
#fmat = A*links.r*((rlen/th_mat)**(POW-2)/th_mat*tf.exp(-(rlen/th_mat)**POW))[:,:,newaxis]
links.Force_LL = tf.reduce_sum(fmat, 0)
links.force_ext_apply[id] = tf.scatter_add(links.net.f_link, idx,
links.Force_LL)
#### Cos theta
dx = tf.gather(links.dp, idx)
dx1 = dx / tf.norm(dx, axis=-1)[:, newaxis]
dx2 = tf.matmul(dx1, tf.transpose(dx1))
links.cos[id] = dx2 # tf.norm(fmat, axis = -1)
def test_state_grads(sess):
v = tf.Variable([0., 0., 0.])
x = tf.ones((3,))
y0 = tf.assign(v, x)
y1 = tf.assign_add(v, x)
grad0 = tf.gradients(y0, [v, x])
grad1 = tf.gradients(y1, [v, x])
grad_vals = sess.run((grad0, grad1))
assert np.allclose(grad_vals[0][0], 0)
assert np.allclose(grad_vals[0][1], 1)
assert np.allclose(grad_vals[1][0], 1)
assert np.allclose(grad_vals[1][1], 1)
v = tf.Variable([0., 0., 0.])
x = tf.ones((1,))
y0 = tf.scatter_update(v, [0], x)
y1 = tf.scatter_add(v, [0], x)
grad0 = tf.gradients(y0, [v._ref(), x])
grad1 = tf.gradients(y1, [v._ref(), x])
grad_vals = sess.run((grad0, grad1))
assert np.allclose(grad_vals[0][0], [0, 1, 1])
assert np.allclose(grad_vals[0][1], 1)
assert np.allclose(grad_vals[1][0], 1)
assert np.allclose(grad_vals[1][1], 1)
def bilinear_scatter(ref, indices, weights, quantity=1.0):
assert len(indices) == len(weights) == 4
indices = tf.concat(indices, axis=0)
weights = tf.concat([weight * quantity for weight in weights], axis=0)
ref = tf.scatter_add(ref, indices, weights)
return ref
def update_contextual_features(contextual_features, indices, updates,
flattened_idx_offset):
first_indices, second_indices = tf.split(1, 2, indices)
indices = tf.squeeze(first_indices + second_indices)
indices = indices + flattened_idx_offset
contextual_features = tf.scatter_add(contextual_features, indices,
updates, use_locking=None)
return contextual_features
def _streaming_num_predictions(mask, user_ids, num_users, k, name=None):
with tf.name_scope(name, _at_k_name("total_prediction_count", k)) as scope:
var = metric_variable([num_users], tf.float64, name=scope)
return var, tf.scatter_add(
var, user_ids, tf.cast(tf.reduce_sum(mask, axis=1), tf.float64))
def class_level_triplet_loss_tf(features, nrof_samples, label, alfa, nrof_classes, beta, gamma): # tensorflow version
""" Class_level_Triple_loss, triple loss implemented on the centers of the class intead of the individual sample
--mzh 30072017s
"""
dim_features = features.get_shape()[1].value
centers = tf.get_variable('centers', [nrof_classes, dim_features], dtype=tf.float32,
initializer=tf.constant_initializer(0), trainable=False)
nrof_elements_per_class = tf.get_variable('centers_cts', [nrof_classes], dtype=tf.float32,
initializer=tf.constant_initializer(0), trainable=False)
## normalisation as the embedding vectors in order to similarity distance
#features = tf.nn.l2_normalize(features, 1, 1e-10, name='feat_emb')
## calculate centers
centers_batch = tf.gather(centers, label)
diff = (1 - alfa) * (centers_batch - features)
diff_within = centers_batch - features
dist_within = tf.reduce_sum(diff_within**2/2, axis=1, keep_dims=True)
dist_within_center = tf.reduce_sum(dist_within, axis=0) ## sum all the elements in the dist_centers_sum, dist_within_center is a scale
## inter_center_loss calculation
label_unique,idx = tf.unique(label)
centers_batch_unique = tf.gather(centers,label_unique)#select the centers corresponding to the batch samples, otherwise the whole centers will cause the overflow of the centers_2D
nrof_centers_batch_unique = tf.shape(centers_batch_unique)[0]##very important, tf.shape() can be used to get the run-time dynamic tensor shape; however .get_shape() can only be used to get the shape of the static shape of the tensor
centers_1D = tf.reshape(centers_batch_unique, [1, nrof_centers_batch_unique * dim_features])
centers_2D = tf.tile(centers_1D, [nrof_samples, 1])
centers_3D = tf.reshape(centers_2D, [nrof_samples,nrof_centers_batch_unique, dim_features])
features_3D = tf.reshape(features, [nrof_samples, 1, dim_features])
dist_inter_centers = features_3D - centers_3D
dist_inter_centers_sum_dim = tf.reduce_sum(dist_inter_centers**2,2)/2 # calculate the L2 of the features, [nrof_samples, nrof_classes, feature_dimension]
dist_inter_centers_sum_all = tf.reduce_sum(dist_inter_centers_sum_dim)#sum all the elements in the dist_inter_centers_sum_dim
## total loss
dist_within_2D = tf.tile(dist_within, [1, nrof_centers_batch_unique])
dist_matrix = dist_within_2D + beta*tf.ones([nrof_samples, nrof_centers_batch_unique]) - gamma*dist_inter_centers_sum_dim
loss_matrix = tf.maximum(dist_matrix, tf.zeros([nrof_samples, nrof_centers_batch_unique], tf.float32))
loss_pre = tf.reduce_sum(loss_matrix) - nrof_samples*beta
#loss = tf.divide(loss_pre, nrof_samples)
loss = tf.divide(loss_pre, tf.multiply(tf.cast(nrof_samples, tf.float32),
tf.cast(nrof_centers_batch_unique, tf.float32) - tf.cast(1, tf.float32)))
#centers = tf.scatter_sub(centers, label, diff)
##update centers
zeros = tf.zeros_like(label_unique, tf.float32)
## calculation the repeat time of same label
nrof_elements_per_class_clean = tf.scatter_update(nrof_elements_per_class, label_unique, zeros)
ones = tf.ones_like(label, tf.float32)
## counting the number elments in each class, the class is in the order of the [0,1,2,3,....] as initialzation
nrof_elements_per_class_update = tf.scatter_add(nrof_elements_per_class_clean, label, ones)
## nrof_elements_per_class_list is the number of the elements in each class in the batch
nrof_elements_per_class_batch = tf.gather(nrof_elements_per_class_update, label)
centers_cts_batch_reshape = tf.reshape(nrof_elements_per_class_batch, [-1, 1])
diff_mean = tf.div(diff, centers_cts_batch_reshape)
centers = tf.scatter_sub(centers, label, diff_mean)
#return loss
return loss, centers, dist_within_center, dist_inter_centers_sum_all, nrof_centers_batch_unique
def test_builder_to_backend_programmatic(
self, use_cpu_only, backend, rankData_rankIndices, accumulate_mode
):
data_rank, indices_rank = rankData_rankIndices
data_shape = np.random.randint(low=2, high=5, size=data_rank)
indices_shape = np.random.randint(low=2, high=5, size=indices_rank)
updates_shape = list(indices_shape) + list(data_shape[1:])
data = np.random.rand(*data_shape).astype(np.float32)
updates = np.random.rand(*updates_shape).astype(np.float32)
indices = np.random.randint(0, data_shape[0], size=indices_shape).astype(
np.int32
)
def build(data, indices, updates):
return mb.scatter(
data=data, indices=indices, updates=updates, mode=accumulate_mode
)
with tf.Graph().as_default(), tf.Session() as sess:
tf_output = tf.Variable(data)
sess.run(tf.global_variables_initializer())
if accumulate_mode == "update":
sess.run(tf.scatter_update(tf_output, indices, updates))
if accumulate_mode == "add":
sess.run(tf.scatter_add(tf_output, indices, updates))
if accumulate_mode == "sub":
sess.run(tf.scatter_sub(tf_output, indices, updates))
if accumulate_mode == "mul":
sess.run(tf.scatter_mul(tf_output, indices, updates))
if accumulate_mode == "div":
sess.run(tf.scatter_div(tf_output, indices, updates))
if accumulate_mode == "max":
sess.run(tf.scatter_max(tf_output, indices, updates))
if accumulate_mode == "min":
sess.run(tf.scatter_min(tf_output, indices, updates))
expected_output = sess.run(tf_output)
input_placeholders = {
"data": mb.placeholder(shape=data.shape),
"indices": mb.placeholder(shape=indices.shape, dtype=types.int32),
"updates": mb.placeholder(shape=updates.shape),
}
input_values = {"data": data, "indices": indices, "updates": updates}
expected_output_types = tuple(data_shape[:]) + (types.fp32,)
run_compare_builder(
build,
input_placeholders,
input_values,
expected_output_types,
expected_output,
use_cpu_only=use_cpu_only,
frontend_only=False,
backend=backend,
)
def _sparse_moving_average(self, x_tm1, idxs, a_t_, name, beta=.9):
""""""
b_tm1 = self.get_accumulator(x_tm1, '%s' % name)
b_tm1_ = tf.gather(b_tm1, idxs)
shape = self.get_variable_shape(x_tm1)
tm1 = self.get_accumulator(x_tm1, '%s/tm1' % name, shape=[shape[0]]+[1]*(len(shape)-1))
tm1_ = tf.gather(tm1, idxs)
t = tf.scatter_add(tm1, idxs, tf.ones_like(tm1_))
t_ = tf.gather(t, idxs)
if beta < 1:
beta_t = tf.convert_to_tensor(beta, name='%s/decay' % name)
beta_t_ = beta_t * (1-beta_t**tm1_) / (1-beta_t**t_)
else:
beta_t_ = tm1_/t_
b_t = tf.scatter_update(b_tm1, idxs, beta_t_*b_tm1_)
b_t = tf.scatter_add(b_t, idxs, (1-beta_t_)*a_t_)
return b_t, t
def build_computation_graph(self):
# -------------------------------- E-step -------------------------------- #
emission_for_labels = tf.gather_nd(self.emission, self.labels) # UxC
''' See tf.gather_nd
indices = [[1], [0]]
params = [['a', 'b'], ['c', 'd']]
output = [['c', 'd'], ['a', 'b']]
'''
# Broadcasting the prior
numerator = tf.multiply(emission_for_labels,
tf.reshape(self.prior,
shape=[1, self.C])) # --> UxC
denominator = tf.matmul(emission_for_labels,
tf.reshape(self.prior, shape=[self.C,
1])) # --> Ux1
posterior_estimate = tf.divide(numerator, denominator) # --> UxC
# Compute the expected complete log likelihood
compute_likelihood = tf.assign_add(self.likelihood, [
tf.reduce_sum(tf.multiply(posterior_estimate, tf.log(numerator)))
])
# -------------------------------- M-step -------------------------------- #
# These are used at each minibatch
update_prior_num = tf.assign_add(
self.prior_numerator, tf.reduce_sum(posterior_estimate, axis=0))
update_prior_den = tf.assign_add(self.prior_denominator,
[tf.reduce_sum(posterior_estimate)])
labels = tf.squeeze(
self.labels) # removes dimensions of size 1 (current is ?x1)
update_emission_num = tf.scatter_add(self.emission_numerator, labels,
posterior_estimate)
update_emission_den = tf.assign_add(
self.emission_denominator,
[tf.reduce_sum(posterior_estimate, axis=0)])
# These are used at the end of an epoch to update the distributions
update_prior = tf.assign(
self.prior, tf.divide(self.prior_numerator,
self.prior_denominator))
update_emission = tf.assign(
self.emission,
tf.divide(self.emission_numerator, self.emission_denominator))
# ------------------------------- Inference ------------------------------ #
inference = tf.argmax(numerator, axis=1)
return compute_likelihood, update_prior_num, update_prior_den, update_emission_num, update_emission_den, \
update_prior, update_emission, inference
def deconv_local_st5_unfilter(images, output_shape, conv_layer_scope):
#augment = inference_augment_st_filter(images, KERNEL_SIZE)
augment = deconv_augment_s_filter(images)
images = augment
# conv_output
with tf.variable_scope(conv_layer_scope, reuse=True) as scope:
images_shape = images.get_shape().as_list()
inv_map = tf.range(
0, images_shape[0] * images_shape[1] * images_shape[2] *
output_shape[3])
inv_map = tf.reshape(inv_map, shape=output_shape)
inv_map = deconv_augment_s_filter(inv_map)
inv_map_1d = tf.reshape(inv_map, shape=[-1])
kernel = tf.get_variable('weights')
biases = tf.get_variable('biases')
relu_output = tf.nn.relu(images, name=scope.name)
bias = tf.nn.bias_add(relu_output, -biases)
#deconv = tf.reshape(tf.nn.bias_add(relu_output, -biases), augment.get_shape().as_list())
#deconv_output = tf.nn.conv2d(relu_output*25, tf.transpose(kernel, perm=[0, 1, 3, 2]), [1, 5, 1, 1], padding='SAME')
deconv_output = tf.nn.conv2d_transpose(bias,
kernel,
output_shape=[
output_shape[0],
output_shape[1] * 5,
output_shape[2],
output_shape[3]
],
strides=[1, 5, 1, 1],
padding='SAME')
deconv_output_1d = tf.reshape(deconv_output, [-1])
tmp_output = tf.Variable(tf.zeros(shape=[
output_shape[0] * output_shape[1] * output_shape[2] *
output_shape[3]
],
dtype=tf.float32),
name='tmp_output')
tmp_output = tf.assign(
tmp_output,
tf.zeros(shape=[
output_shape[0] * output_shape[1] * output_shape[2] *
output_shape[3]
],
dtype=tf.float32))
tmp_output2 = tf.scatter_add(tmp_output, inv_map_1d, deconv_output_1d)
tmp_output3 = tmp_output2 * (1.0 / 5.0)
tmp_output4 = tf.reshape(tmp_output3, shape=output_shape)
_print_tensor_size(tmp_output4)
return tmp_output4
def collect_gradients(gradients, variables):
ops = []
for grad, var in zip(gradients, variables):
if isinstance(grad, tf.Tensor):
ops.append(tf.assign_add(var, grad))
else:
ops.append(tf.scatter_add(var, grad.indices, grad.values))
return tf.group(*ops)
def _apply_sparse(self, grad, var):
return self._apply_sparse_shared(
grad.values,
var,
grad.indices,
lambda x, i, v: tf.scatter_add( # pylint: disable=g-long-lambda
x,
i,
v,
use_locking=self._use_locking))
def vertex_normals_pre_split_fixtopo(vertices,
faces,
ver_ref_face,
ver_ref_face_index,
ver_ref_face_num,
name=None):
"""
:param vertices: batch size, vertex-index, x/y/z[/w]
:param faces: face-index, vertex-in-face, tf_render.int32
:param ver_ref_face: vertex-index*flat
:param ver_ref_face_index: vertex-index*flat
:param ver_ref_face_num: vertex-index
:param name:
:return:
"""
"""Computes vertex normals for the given pre-split meshes.
This function is identical to `vertex_normals`, except that it assumes each vertex is used by just one face, which
allows a more efficient implementation.
"""
# This is identical to vertex_normals, but assumes each vertex appears in exactly one face, e.g. due to having been
# processed by split_vertices_by_face
# vertices is indexed by
# faces is indexed by
# result is indexed by *
with ops.name_scope(name, 'VertexNormalsPreSplit',
[vertices, faces]) as scope:
vertices_num = int(vertices.get_shape()[1])
vertices, faces = _prepare_vertices_and_faces(vertices, faces)
normals_by_face, _ = _get_face_normals(
vertices, faces) # indexed by face-index, batch_size, x/y/z
normals_by_face = tf.transpose(normals_by_face, perm=[1, 0, 2])
ver_ref_face_num_tile = tf.tile(tf.expand_dims(ver_ref_face_num, -1),
multiples=[1, 3])
list_normals_by_ver = []
for b in range(vertices.shape[0]):
normals_by_face_b = normals_by_face[b]
normals_by_vertex_flat_b = tf.gather(normals_by_face_b,
ver_ref_face)
nv = tf.scatter_add(
tf.Variable(tf.zeros(shape=[vertices_num, 3]),
trainable=False), ver_ref_face_index,
normals_by_vertex_flat_b)
nv = nv / (ver_ref_face_num_tile + 1e-6)
nv = nv / (tf.norm(nv, axis=-1, keep_dims=True) + 1e-12) # ditto
list_normals_by_ver.append(nv)
normals_by_vertex = tf.stack(list_normals_by_ver)
return normals_by_vertex
def _apply_sparse(self, grad, var):
"""
:param tf.IndexedSlices grad:
:param tf.Variable var:
:return: group of update operations
:rtype: tf.Operation
"""
beta2_power = tf.cast(self._beta2_power, var.dtype.base_dtype)
lr_t = tf.cast(self._lr_t, var.dtype.base_dtype)
beta1_t = tf.cast(self._beta1_t, var.dtype.base_dtype)
beta2_t = tf.cast(self._beta2_t, var.dtype.base_dtype)
epsilon_t = tf.cast(self._epsilon_t, var.dtype.base_dtype)
mu_t = tf.cast(self._mu_t, var.dtype.base_dtype)
mu_t_next = tf.cast(self._mu_t_next, var.dtype.base_dtype)
mu_prod_t_next = tf.cast(self._mu_prod_t_next, var.dtype.base_dtype)
mu_prod_t_next2 = tf.cast(self._mu_prod_t_next2, var.dtype.base_dtype)
m_prev = self.get_slot(var, "m")
v_prev = self.get_slot(var, "v")
# called m_t in paper
m = beta1_t * m_prev
m = tf.assign(m_prev, m, use_locking=self._use_locking)
m = tf.scatter_add(m, grad.indices, (1 - beta1_t) * grad.values, use_locking=self._use_locking)
m_update = m
m_ = m / (1 - mu_prod_t_next2) # bias correction (with momentum schedule (include the next t+1))
# called n_t in paper
v = beta2_t * v_prev
v = tf.assign(v_prev, v, use_locking=self._use_locking)
v = tf.scatter_add(v, grad.indices, (1 - beta2_t) * (grad.values * grad.values), use_locking=self._use_locking)
v_update = v
v_ = v / (1 - beta2_power)
m__ = tf.sparse_add(
mu_t_next * m_,
tf.IndexedSlices((1 - mu_t) * grad.values / (1 - mu_prod_t_next), grad.indices, grad.dense_shape))
step = lr_t * m__ / (tf.sqrt(v_) + epsilon_t)
var_update = tf.assign_sub(var, step, use_locking=self._use_locking)
return tf.group(var_update, m_update, v_update)
def _assign_add(self, ref, updates, indices=None):
if indices is not None:
if isinstance(ref, tf.Variable):
return tf.scatter_add(ref, indices, updates, use_locking=self._use_locking)
elif isinstance(ref, resource_variable_ops.ResourceVariable):
with tf.control_dependencies([resource_variable_ops.resource_scatter_add(ref.handle, indices, updates)]):
return ref.value()
else:
raise TypeError("did not expect type %r" % type(ref))
else:
return tf.assign_add(ref, updates, use_locking=self._use_locking)
def _assign_add(self, ref, updates, indices=None):
if indices is not None:
if isinstance(ref, tf.Variable):
return tf.scatter_add(ref, indices, updates, use_locking=self._use_locking)
elif isinstance(ref, resource_variable_ops.ResourceVariable):
with tf.control_dependencies([resource_variable_ops.resource_scatter_add(ref.handle, indices, updates)]):
return ref.value()
else:
raise TypeError("did not expect type %r" % type(ref))
else:
return tf.assign_add(ref, updates, use_locking=self._use_locking)
def collect_gradients(gradients, variables):
ops = []
for grad, var in zip(gradients, variables):
if isinstance(grad, tf.Tensor):
ops.append(tf.assign_add(var, grad))
elif isinstance(grad, tf.IndexedSlices):
ops.append(tf.scatter_add(var, grad.indices, grad.values))
else:
print("grad : ", grad, " with type : ", type(grad))
return tf.group(*ops)
def gradient_updates(grads_and_vars):
updates = []
for grad, var in grads_and_vars:
if isinstance(grad, tf.Tensor):
updates.append(tf.assign(var, grad))
else:
new_var = tf.assign(var, tf.zeros_like(var))
updates.append(tf.scatter_add(new_var, grad.indices, grad.values))
return tf.group(*updates)
def sparse_moving_average(self, variable, unique_indices, accumulant, name='Accumulator', decay=.9):
""""""
accumulant = tf.clip_by_value(accumulant, -self.clip, self.clip)
first_dim = variable.get_shape().as_list()[0]
accumulator = self.get_accumulator(name, variable)
indexed_accumulator = tf.gather(accumulator, unique_indices)
iteration = self.get_accumulator('{}/iteration'.format(name), variable, shape=[first_dim, 1])
indexed_iteration = tf.gather(iteration, unique_indices)
iteration = tf.scatter_add(iteration, unique_indices, tf.ones_like(indexed_iteration))
indexed_iteration = tf.gather(iteration, unique_indices)
if decay < 1:
current_indexed_decay = decay * (1-decay**(indexed_iteration-1)) / (1-decay**indexed_iteration)
else:
current_indexed_decay = (indexed_iteration-1) / indexed_iteration
accumulator = tf.scatter_update(accumulator, unique_indices, current_indexed_decay*indexed_accumulator)
accumulator = tf.scatter_add(accumulator, unique_indices, (1-current_indexed_decay)*accumulant)
return accumulator, iteration
def force_ext_ellipsoid_idx_multi(links,idx):
gij, ginv = links.get_metrics(idx)
r = tf.gather(links.points, idx)
dr2 = links.get_dr2(r,gij)
A = links.amplitude
dr = r - r[:,newaxis]
drh = dr / (tf.norm(dr, axis = -1, keepdims=True)+1e-15)
#links.fmat0 = A*drh*((links.dr2[id]**(links.net.POW/2-1))*tf.exp(-links.dr2[id]**links.net.POW))
links.fmat0 = A*drh*((dr2**(links.net.POW/2.-1))*tf.exp(-dr2**(links.net.POW/2.)))
links.Force_LL_Ell = tf.reduce_sum(links.fmat0,0)
return tf.scatter_add(links.net.f_link,idx, links.Force_LL_Ell)
def make_logits(edge_list):
with tf.name_scope(name, "learned_unigram_distribution", [edge_list]):
edge_list_flat = tf.reshape(edge_list, [-1])
update_empirical = tf.scatter_add(
empirical_vertex_distribution,
edge_list_flat,
tf.ones_like(edge_list_flat))
with tf.control_dependencies([update_empirical]):
logits = power * tf.log(tf.to_float(empirical_vertex_distribution))
return logits
def update_contextual_features(contextual_features, indices, updates,
flattened_idx_offset):
try:
first_indices, second_indices = tf.split(value=indices, num_or_size_splits=2, axis=1) #v1.0 +
except:
first_indices, second_indices = tf.split(1, 2, indices) # before TF v1.0
indices = tf.squeeze(first_indices + second_indices)
indices = indices + flattened_idx_offset
contextual_features = tf.scatter_add(contextual_features, indices,
updates, use_locking=None)
return contextual_features
def create_stats_val(self, y_labels, labels):
with tf.variable_scope('MiscNNHelper/create_stats_val'):
labels = tf.cast(labels, dtype=tf.int32)
# define tf variables to use scatter_nd and scatter_nd_add
# pwtmp = tf.Variable(tf.zeros(self.num_labels), trainable=False, dtype=tf.float32)
# pytmp = tf.Variable(tf.zeros(self.cb_size), trainable=False, dtype=tf.float32)
# pw_y_tmp = tf.Variable(tf.zeros([self.num_labels, self.cb_size]), trainable=False, dtype=tf.float32)
pwtmp = tf.get_default_graph().get_tensor_by_name('p_w:0')
pytmp = tf.get_default_graph().get_tensor_by_name('p_y:0')
pw_y_tmp = tf.get_default_graph().get_tensor_by_name('p_w_y:0')
# self.reset_p_w
# create P(w)
pwtmp = tf.assign(pwtmp, tf.zeros([self.num_labels
])) # reset Variable/floor
# pwtmp = self.reset_variable(pwtmp)
pwtmp = tf.scatter_add(pwtmp, labels, tf.ones(tf.shape(labels)))
# create P(y)
pytmp = tf.assign(pytmp,
tf.zeros([self.cb_size])) # reset Variable/floor
pytmp = tf.scatter_add(pytmp, y_labels,
tf.ones(tf.shape(y_labels)))
# create P(w|y)
pw_y_tmp = tf.assign(pw_y_tmp,
tf.zeros([self.num_labels, self.cb_size
])) # reset Variable/floor
pw_y_tmp = tf.scatter_nd_add(
pw_y_tmp,
tf.concat([
tf.cast(labels, dtype=tf.int64),
tf.expand_dims(y_labels, 1)
],
axis=1), tf.ones(tf.shape(y_labels)))
return pwtmp, pytmp, pw_y_tmp
def variance_estimate(self):
# Variables and tensors initialization
tf_shape = tf.stack([self.n_users, self.n_factors])
self.variance = tf.Variable(tf.zeros(tf_shape, dtype=self.dtype),
name='variance',
validate_shape=False,
trainable=False)
init_sum_of_square = tf.Variable(tf.zeros(shape=tf_shape,
dtype=self.dtype),
name='sum_of_square',
validate_shape=False,
trainable=False)
init_nu = tf.Variable(tf.zeros(shape=self.n_users, dtype=tf.int64),
name='n_items_per_user',
validate_shape=False,
trainable=False)
ones = tf.ones(dtype=tf.int64, shape=tf.shape(self.x)[0])
u_idx = self.x.indices[:, 1]
lim_users = tf.expand_dims(self.n_users, axis=0)
where = tf.less(u_idx, lim_users)
indexes = tf.reshape(tf.where(where), shape=[-1])
indexes = tf.nn.embedding_lookup(self.x.indices, indexes)[:, 1]
# computes the square for the batch (batch_size, n_factors)
# each row represent the square root for a user
user_v = tf.nn.embedding_lookup(self.params, indexes)
dot = tf.sparse_tensor_dense_matmul(self.x, self.params)
dot = user_v - (dot - user_v)
sq = tf.square(dot)
sum_of_square = tf.scatter_add(init_sum_of_square, indexes, sq)
nu = tf.scatter_add(init_nu, indexes, ones)
nu = tf.tile(tf.expand_dims(tf.to_float(nu), 1), [1, self.n_factors])
computed_variance = sum_of_square / nu
self.variance_op = tf.assign(self.variance,
computed_variance,
validate_shape=False)
self.init_variance_vars = tf.variables_initializer(
[self.variance, init_nu, init_sum_of_square])
def softmax(self, weights, level):
# Temporary variables for getting the image-feature and image weights
with tf.variable_scope('temp_vars', reuse=tf.AUTO_REUSE):
temp_image_weights = tf.get_variable(
shape=[self.Config.batch_size],
dtype=np.float32,
initializer=tf.constant_initializer(0.),
name='tmp_img_weights',
trainable=False)
temp_user_weights = tf.get_variable(
shape=[self.Config.user_number + 1],
dtype=np.float32,
initializer=tf.constant_initializer(0.),
name='tmp_user_weights',
trainable=False)
weights = tf.exp(weights)
summation = None
if level == 'box':
temp_image_weights = tf.assign(
temp_image_weights,
tf.zeros(shape=[self.Config.batch_size], dtype=np.float32))
temp_image_weights = tf.scatter_add(temp_image_weights,
indices=self.box_group,
updates=weights)
summation = tf.nn.embedding_lookup(temp_image_weights,
self.box_group)
elif level == 'image':
temp_user_weights = tf.assign(
temp_user_weights,
tf.zeros(shape=[self.Config.user_number + 1],
dtype=np.float32))
temp_user_weights = tf.scatter_add(temp_user_weights,
indices=self.batch_user,
updates=weights)
summation = tf.nn.embedding_lookup(temp_user_weights,
self.batch_user)
else:
exit('Error!')
return tf.divide(weights, summation)
def weightedSum(self, features, weights, level):
with tf.variable_scope('temp_vars', reuse=tf.AUTO_REUSE):
temp_image_feature = tf.get_variable(
shape=[self.Config.batch_size, self.Config.feature_dimension],
dtype=np.float32,
initializer=tf.constant_initializer(0.),
name='tmp_img_feature',
trainable=False)
temp_auxi_feature = tf.get_variable(
shape=[
self.Config.user_number + 1, self.Config.latent_dimension
],
dtype=np.float32,
initializer=tf.constant_initializer(0.),
name='tmp_user_feature',
trainable=False)
features = weightedMultiply(features, weights)
result = None
if level == 'box':
temp_image_feature = tf.assign(
temp_image_feature,
tf.zeros(shape=[
self.Config.batch_size, self.Config.feature_dimension
],
dtype=np.float32))
temp_image_feature = tf.scatter_add(temp_image_feature,
indices=self.box_group,
updates=features)
result = tf.nn.embedding_lookup(temp_image_feature,
self.batch_image)
elif level == 'image':
temp_auxi_feature = tf.scatter_add(temp_auxi_feature,
indices=self.batch_user,
updates=features)
result = tf.nn.embedding_lookup(temp_auxi_feature, self.batch_user)
else:
exit('Error!')
return result
def getCenters(num_classes, feature_size, labels, final_embed):
#INF
INF = 99999.0
# this is wrong
# centers = tf.zeros(shape=[num_classes, feature_size], dtype=tf.float32)
#test
centers = tf.Variable(tf.zeros([num_classes, feature_size]), dtype=tf.float32, name='centers')
centers_count = tf.Variable(tf.zeros([num_classes, 1]), dtype=tf.float32, name='centers_count')
# centers_count = centers_count + 1
labels = tf.reshape(labels, [-1])
# appear_times
unique_label, unique_idx, unique_count = tf.unique_with_counts(labels)
appear_times = tf.gather(unique_count, unique_idx)
appear_times = tf.reshape(appear_times, [-1, 1])
appear_times = tf.cast(appear_times, tf.float32)
centers = tf.scatter_add(centers, labels, final_embed)
centers_count = tf.scatter_add(centers_count, labels, appear_times)
centers_count = tf.clip_by_value(centers_count, clip_value_min=tf.constant(1.0, dtype=tf.float32), clip_value_max=tf.constant(INF, dtype=tf.float32))
# centers_count = tf.reciprocal(centers_count)
# centers = tf.get_variable( shape=[num_classes, feature_size], dtype=tf.float32, initializer=tf.constant_initializer(0), trainable=False)
print ("====== getCenters =====")
print ("labels:", labels)
print ("final_embed: ", final_embed)
print ("centers: ", centers)
print ("appear_times: ", appear_times)
print ("centers_count: ", centers_count)
print ("====== getCenters =====")
# centers = centers * centers_count
centers = centers / centers_count
return centers
def center_loss(features, label, alfa, nrof_classes):
"""Center loss based on the paper "A Discriminative Feature Learning Approach for Deep Face Recognition"
(http://ydwen.github.io/papers/WenECCV16.pdf)
This is not exactly the algorthim proposed in the paper, since the update/shift of the centers is not moving towards the
centers (i.e. sum(Xi)/Nj, Xi is the element of class j) of the classes but the sum of the elements (sum(Xi)) in the class
"""
#nrof_features = features.get_shape()[1]
#centers = tf.get_variable('centers', [nrof_classes, nrof_features], dtype=tf.float32,
# initializer=tf.constant_initializer(0), trainable=False)
#label = tf.reshape(label, [-1])
#centers_batch = tf.gather(centers, label)
#diff = (1 - alfa) * (centers_batch - features)
#diff = alfa * (centers_batch - features)
#centers = tf.scatter_sub(centers, label, diff)
# loss = tf.nn.l2_loss(features - centers_batch)
# return loss, centers, diff, centers_batch
"""Center loss based on the paper "A Discriminative Feature Learning Approach for Deep Face Recognition"
(http://ydwen.github.io/papers/WenECCV16.pdf)
-- mzh 15/02/2017
-- Correcting the center updating, center updates/shifts towards to the center of the correponding class with a weight:
-- centers = centers- (1-alpha)(centers-sum(Xi)/Nj), where Xi is the elements of the class j, Nj is the number of the elements of class Nj
-- code has been tested by the test script '../test/center_loss_test.py'
"""
nrof_features = features.get_shape()[1]
centers = tf.get_variable('centers', [nrof_classes, nrof_features], dtype=tf.float32, initializer=tf.constant_initializer(0), trainable=False)
centers_cts = tf.get_variable('centers_cts', [nrof_classes], dtype=tf.float32, initializer=tf.constant_initializer(0), trainable=False)
#centers_cts_init = tf.zeros_like(nrof_classes, tf.float32)
label = tf.reshape(label, [-1])
centers_batch = tf.gather(centers, label) #get the corresponding center of each element in features, the list of the centers is in the same order as the features
loss_n = tf.reduce_sum(tf.square(features - centers_batch)/2, 1)
loss = tf.nn.l2_loss(features - centers_batch)
diff = (1 - alfa) * (centers_batch - features)
## update the centers
label_unique, idx = tf.unique(label)
zeros = tf.zeros_like(label_unique, tf.float32)
## calculation the repeat time of same label
nrof_elements_per_class_clean = tf.scatter_update(centers_cts, label_unique, zeros)
ones = tf.ones_like(label, tf.float32)
## counting the number elments in each class, the class is in the order of the [0,1,2,3,....] as initialzation
nrof_elements_per_class_update = tf.scatter_add(nrof_elements_per_class_clean, label, ones)
## nrof_elements_per_class_list is the number of the elements in each class in the batch
nrof_elements_per_class_batch = tf.gather(nrof_elements_per_class_update, label)
nrof_elements_per_class_batch_reshape = tf.reshape(nrof_elements_per_class_batch, [-1, 1])## reshape the matrix as 1 coloum no matter the dimension of the row (-1)
diff_mean = tf.div(diff, nrof_elements_per_class_batch_reshape)
centers = tf.scatter_sub(centers, label, diff_mean)
#return loss, centers, label, centers_batch, diff, centers_cts, centers_cts_batch, diff_mean,center_cts_clear, nrof_elements_per_class_batch_reshape
return loss, loss_n, centers, nrof_elements_per_class_clean, nrof_elements_per_class_batch_reshape,diff_mean # facenet_expression_addcnns_simple_joint_v4_dynamic.py
def body(
ix,
X_VW2,
XU_W2,
XUV_2,
):
ijk = tf.gather(X.indices, ix)
val = tf.gather(X.values, ix)
Ui = tf.gather(self.U, ijk[0]) # R-dimensional
Vj = tf.gather(self.V, ijk[1])
Wk = tf.gather(self.W, ijk[2])
Xijk_Vj_Wk = val * tf.multiply(Vj, Wk)
Xijk_Ui_Wk = val * tf.multiply(Ui, Wk)
Xijk_Ui_Vj = val * tf.multiply(Ui, Vj)
r1 = tf.scatter_add(
X_VW, ijk[0], Xijk_Vj_Wk
) # add Xijk(Vj*Wk) to X_VW(i,:) (as vectors in \mathbb{R}^R)
r2 = tf.scatter_add(XU_W, ijk[1], Xijk_Ui_Wk)
r3 = tf.scatter_add(XUV_, ijk[2], Xijk_Ui_Vj)
new_ix = tf.add(ix, tf.constant(1))
return new_ix, r1, r2, r3
def _collect_gradients(gradients, variables):
""" Collects gradients.
Args:
gradients: A list of gradients.
variables: A list of variables for collecting the gradients.
Returns: A tf op.
"""
ops = []
for grad, var in zip(gradients, variables):
if isinstance(grad, tf.Tensor):
ops.append(tf.assign_add(var, grad))
else:
ops.append(tf.scatter_add(var, grad.indices, grad.values))
return tf.group(*ops, name="collect_gradients")
def body(sequence_len,
step,
feature_pl,
path_pl,
flattened_idx_offset,
contextual_features):
begin = tf.get_variable("begin1",[3],dtype=tf.int32,initializer=tf.constant_initializer(0))
begin = tf.scatter_update(begin,1,step,use_locking=None)
step_feature = tf.squeeze(tf.slice(feature_pl,begin,[-1,1,-1]))
input_idx = tf.slice(path_pl, begin, [-1,1,1])
input_idx = tf.reshape(input_idx,[-1])
input_idx_flattened = flattened_idx_offset + input_idx
max_seq_len = FLAGS.max_seq_len
begin2 = tf.get_variable("begin2",[3],dtype=tf.int32,initializer=tf.constant_initializer(0))
begin2 = tf.scatter_update(begin2,1,step,use_locking=None)
begin2 = tf.scatter_update(begin2,2,1,use_locking=None)
tf.get_variable_scope().reuse_variables()
contextual_features = tf.get_variable("contextual_features")
# [max_seq_len * max_seq_len, encoding_nn_output_size],
# dtype=tf.float32)
step_contextual_features = tf.gather(contextual_features,input_idx_flattened) # use flattened indices1
inputs = tf.concat(1,[step_contextual_features,step_feature])
updated_contextual_vectors = single_layer_neural_network1(inputs)
updated_contextual_vectors = tf.tanh(updated_contextual_vectors)
output_idx = tf.reshape(tf.slice(path_pl, begin2, [-1,1, 1]),[-1])
output_idx_flattened = flattened_idx_offset + output_idx
contextual_features = tf.scatter_add(contextual_features,
output_idx_flattened,
updated_contextual_vectors, use_locking=None)
with tf.control_dependencies([contextual_features]):
return (sequence_len,
step+1,
feature_pl,
path_pl,
flattened_idx_offset,
contextual_features)
def deconv_local_st5_unfilter(images, output_shape, conv_layer_scope):
# augment = inference_augment_st_filter(images, KERNEL_SIZE)
augment = deconv_augment_s_filter(images)
images = augment
# conv_output
with tf.variable_scope(conv_layer_scope, reuse=True) as scope:
images_shape = images.get_shape().as_list()
inv_map = tf.range(0, images_shape[0] * images_shape[1] * images_shape[2] * output_shape[3])
inv_map = tf.reshape(inv_map, shape=output_shape)
inv_map = deconv_augment_s_filter(inv_map)
inv_map_1d = tf.reshape(inv_map, shape=[-1])
kernel = tf.get_variable("weights")
biases = tf.get_variable("biases")
relu_output = tf.nn.relu(images, name=scope.name)
bias = tf.nn.bias_add(relu_output, -biases)
# deconv = tf.reshape(tf.nn.bias_add(relu_output, -biases), augment.get_shape().as_list())
# deconv_output = tf.nn.conv2d(relu_output*25, tf.transpose(kernel, perm=[0, 1, 3, 2]), [1, 5, 1, 1], padding='SAME')
deconv_output = tf.nn.conv2d_transpose(
bias,
kernel,
output_shape=[output_shape[0], output_shape[1] * 5, output_shape[2], output_shape[3]],
strides=[1, 5, 1, 1],
padding="SAME",
)
deconv_output_1d = tf.reshape(deconv_output, [-1])
tmp_output = tf.Variable(
tf.zeros(shape=[output_shape[0] * output_shape[1] * output_shape[2] * output_shape[3]], dtype=tf.float32),
name="tmp_output",
)
tmp_output = tf.assign(
tmp_output,
tf.zeros(shape=[output_shape[0] * output_shape[1] * output_shape[2] * output_shape[3]], dtype=tf.float32),
)
tmp_output2 = tf.scatter_add(tmp_output, inv_map_1d, deconv_output_1d)
tmp_output3 = tmp_output2 * (1.0 / 5.0)
tmp_output4 = tf.reshape(tmp_output3, shape=output_shape)
_print_tensor_size(tmp_output4)
return tmp_output4
def OuterProd(*inputs):
order = len(inputs)
if order == 2:
output = tf.mul(inputs[0], inputs[1])
elif order == 3:
size = []
idx = 1
for i in xrange(order):
size.append(inputs[i].get_shape()[0])
output = tf.zeros(size)
u, v, w = inputs[0], inputs[1], inputs[2]
uv = tf.mul(inputs[0], inputs[1])
for i in xrange(self.size[-1]):
output = tf.scatter_add(output, [0,0,i], uv)
return output
def _thin_stack_lookup_gradient(op, grad_stack1, grad_stack2, grad_buf_top, _):
stack, buffer, _, _, buffer_cursors, transitions = op.inputs
stack2_ptrs = op.outputs[3]
t = op.get_attr("timestep")
batch_size = buffer_cursors.get_shape().as_list()[0]
num_tokens = buffer.get_shape().as_list()[0] / batch_size
batch_range = math_ops.range(batch_size)
batch_range_i = tf.to_float(batch_range)
grad_stack_name = "grad_stack_%i_%s" % (t, str(uuid.uuid4())[:15])
grad_buffer_name = "grad_buffer_%i_%s" % (t, str(uuid.uuid4())[:15])
grad_stack = gen_state_ops._temporary_variable(stack.get_shape().as_list(), tf.float32, grad_stack_name)
grad_buffer = gen_state_ops._temporary_variable(buffer.get_shape().as_list(), tf.float32, grad_buffer_name)
grad_stack = tf.assign(grad_stack, tf.zeros_like(grad_stack))
grad_buffer = tf.assign(grad_buffer, tf.zeros_like(grad_buffer))
updates = []
# Write grad_stack1 into block (t - 1)
if t >= 1:
in_cursors = (t - 1) * batch_size + batch_range
grad_stack = tf.scatter_add(grad_stack, in_cursors, grad_stack1)
# Write grad_stack2 using stored lookup pointers
grad_stack = floaty_scatter_add(grad_stack, stack2_ptrs * batch_size + batch_range_i, grad_stack2)
# Use buffer_cursors to scatter grads into buffer.
buffer_ptrs = tf.minimum((float) (num_tokens * batch_size) - 1.0,
buffer_cursors * batch_size + batch_range_i)
grad_buffer = floaty_scatter_add(grad_buffer, buffer_ptrs, grad_buf_top)
with tf.control_dependencies([grad_stack, grad_buffer]):
grad_stack = gen_state_ops._destroy_temporary_variable(grad_stack, grad_stack_name)
grad_buffer = gen_state_ops._destroy_temporary_variable(grad_buffer, grad_buffer_name)
with tf.control_dependencies([grad_stack, grad_buffer]):
return grad_stack, grad_buffer, None, None, None, None
def scatter_add(tensor, indices, updates):
return tf.scatter_add(tensor, indices, updates)
def _mini_batch_training_op(self, inputs, cluster_idx_list,
cluster_centers, cluster_centers_var,
total_counts):
"""Creates an op for training for mini batch case.
Args:
inputs: list of input Tensors.
cluster_idx_list: A vector (or list of vectors). Each element in the
vector corresponds to an input row in 'inp' and specifies the cluster id
corresponding to the input.
cluster_centers: Tensor of cluster centers, possibly normalized.
cluster_centers_var: Tensor Ref of cluster centers.
total_counts: Tensor Ref of cluster counts.
Returns:
An op for doing an update of mini-batch k-means.
"""
update_ops = []
for inp, cluster_idx in zip(inputs, cluster_idx_list):
with ops.colocate_with(inp):
assert total_counts is not None
cluster_idx = tf.reshape(cluster_idx, [-1])
# Dedupe the unique ids of cluster_centers being updated so that updates
# can be locally aggregated.
unique_ids, unique_idx = tf.unique(cluster_idx)
num_unique_cluster_idx = tf.size(unique_ids)
# Fetch the old values of counts and cluster_centers.
with ops.colocate_with(total_counts):
old_counts = tf.gather(total_counts, unique_ids)
with ops.colocate_with(cluster_centers):
old_cluster_centers = tf.gather(cluster_centers, unique_ids)
# Locally aggregate the increment to counts.
count_updates = tf.unsorted_segment_sum(
tf.ones_like(unique_idx, dtype=total_counts.dtype),
unique_idx,
num_unique_cluster_idx)
# Locally compute the sum of inputs mapped to each id.
# For a cluster with old cluster value x, old count n, and with data
# d_1,...d_k newly assigned to it, we recompute the new value as
# x += (sum_i(d_i) - k * x) / (n + k).
# Compute sum_i(d_i), see comment above.
cluster_center_updates = tf.unsorted_segment_sum(
inp,
unique_idx,
num_unique_cluster_idx)
# Shape to enable broadcasting count_updates and learning_rate to inp.
# It extends the shape with 1's to match the rank of inp.
broadcast_shape = tf.concat(
0,
[tf.reshape(num_unique_cluster_idx, [1]),
tf.ones(tf.reshape(tf.rank(inp) - 1, [1]), dtype=tf.int32)])
# Subtract k * x, see comment above.
cluster_center_updates -= tf.cast(
tf.reshape(count_updates, broadcast_shape),
inp.dtype) * old_cluster_centers
learning_rate = tf.inv(tf.cast(old_counts + count_updates, inp.dtype))
learning_rate = tf.reshape(learning_rate, broadcast_shape)
# scale by 1 / (n + k), see comment above.
cluster_center_updates *= learning_rate
# Apply the updates.
update_counts = tf.scatter_add(
total_counts,
unique_ids,
count_updates)
update_cluster_centers = tf.scatter_add(
cluster_centers_var,
unique_ids,
cluster_center_updates)
update_ops.extend([update_counts, update_cluster_centers])
return tf.group(*update_ops)
def island_loss(features, label, alpha, nrof_classes, nrof_features, lamda1=10):
"""Center loss based on the paper "Island Loss for Learning Discriminative Features in Facial Expression Recognition"
(https://github.com/SeriaZheng/EmoNet/blob/master/loss_function/loss_paper/Island_loss.pdf)
"""
# 生成可以共享的变量centers
with tf.variable_scope('center', reuse=True):
centers = tf.get_variable('centers')
label = tf.reshape(label, [-1])
# 取出对应label下对应的center值，注意label里面的值可能会重复，因为一个标签下有可能会出现多个人
centers_batch = tf.gather(centers, label)
# 求特征点到中心的距离并乘以一定的系数，diff1为center loss
diff1 = centers_batch - features
# 获取一个batch中同一样本出现的次数，这里需要理解论文中的更新公式
unique_label, unique_idx, unique_count = tf.unique_with_counts(label)
appear_times = tf.gather(unique_count, unique_idx)
appear_times = tf.reshape(appear_times, [-1, 1])
diff1 = diff1 / tf.cast((1 + appear_times), tf.float32)
diff1 = alpha * diff1
# diff2为island loss的center更新项
diff2 = tf.get_variable('diff2', [nrof_classes, nrof_features], dtype=tf.float32,
initializer=tf.constant_initializer(0), trainable=False)
for i in range(nrof_classes):
for j in range(nrof_classes):
if i!=j:
diff2 = tf.scatter_add(diff2, i,
(tf.gather(centers, i) / tf.sqrt(
tf.reduce_sum(tf.square(tf.gather(centers, i)))) * tf.sqrt(
tf.reduce_sum(tf.square(tf.gather(centers, j)))))
- tf.multiply(
(tf.reduce_sum(
tf.multiply(tf.gather(centers, i), tf.gather(centers, j))) / tf.sqrt(
tf.reduce_sum(tf.square(tf.gather(centers, i)))) *
tf.pow(tf.sqrt(tf.reduce_sum(tf.square(tf.gather(centers, j)))), 3)),
tf.gather(centers, j)))
diff2 = diff2 * lamda1 / (nrof_classes - 1)
diff2 = alpha * diff2
# 求center loss，这里是将l2_loss里面的值进行平方相加，再除以2，并没有进行开方
loss1 = tf.nn.l2_loss(features - centers_batch)
# 求island loss
loss2 = tf.zeros(1)
for i in range(nrof_classes):
for j in range(nrof_classes):
if i!=j:
loss2 = tf.add(tf.add(tf.reduce_sum(tf.multiply(tf.gather(centers, i), tf.gather(centers, j))) / (
tf.sqrt(tf.reduce_sum(tf.square(tf.gather(centers, i)))) *
tf.sqrt(tf.reduce_sum(tf.square(tf.gather(centers, j))))), tf.ones(1)), loss2)
loss2 = lamda1 * loss2
loss = tf.add(loss1,loss2)
# 更新center，输出是将对应于label的centers减去对应的diff，如果同一个标签出现多次，那么就减去多次(diff1与centers维度不同)
centers = tf.scatter_sub(centers, label, diff1)
# diff2维度与centers相同可以直接减
centers = tf.subtract(centers, diff2)
return loss, centers
def scatter_add(variables):
shape = utils.get_tensor_size(variables)
values, indices = tf.nn.top_k(-1 * variables, tf.cast(k * shape / 100, tf.int32))
return tf.scatter_add(variables, indices, values)
# coding: utf-8 import tensorflow as tf import prettytensor as pt import numpy as np import cmtf.data.data_mnist as data_mnist data = tf.Variable([1, 2, 3, 4]) indices = [1, 2] updates = [2, 3] data = tf.scatter_add(data, indices, updates) with tf.Session() as sess: sess.run(tf.initialize_all_variables()) print(sess.run(data))
def scatter(self, dst, val, mode="update"):
"""
Updates the base data corresponding to ``dst``.
Parameters
----------
dst : `.TensorSignal`
Signal indicating the data to be modified in base array
val : ``tf.Tensor``
Update data (same shape as ``dst``, i.e. a dense array <= the size
of the base array)
mode : "update" or "inc"
Overwrite/add the data at ``dst`` with ``val``
"""
if val.dtype.is_floating and val.dtype.base_dtype != self.dtype:
raise BuildError("Tensor detected with wrong dtype (%s), should "
"be %s." % (val.dtype.base_dtype, self.dtype))
# align val shape with dst base shape
self.bases[dst.key].get_shape().assert_is_fully_defined()
val.get_shape().assert_is_fully_defined()
dst_shape = ((dst.shape[0],) +
tuple(self.bases[dst.key].get_shape().as_list()[1:]))
if val.get_shape() != dst_shape:
val = tf.reshape(val, dst.tf_shape)
logger.debug("scatter")
logger.debug("values %s", val)
logger.debug("dst %s", dst)
logger.debug("indices %s", dst.indices)
logger.debug("dst base %s", self.bases[dst.key])
logger.debug("reads_by_base %s",
self.reads_by_base[self.bases[dst.key]])
# make sure that any reads to the target signal happen before this
# write (note: this is only any reads that have happened since the
# last write, since each write changes the base array object)
with tf.control_dependencies(self.reads_by_base[self.bases[dst.key]]):
var = self.bases[dst.key]
if (dst.tf_slice is not None and
var.get_shape().is_compatible_with(val.get_shape()) and
dst.indices[0] == 0 and
dst.indices[-1] == var.get_shape()[0].value - 1 and
len(dst.indices) == var.get_shape()[0]):
if mode == "inc":
result = tf.assign_add(var, val, use_locking=False)
self.write_types["assign_add"] += 1
else:
result = tf.assign(var, val, use_locking=False)
self.write_types["assign"] += 1
elif mode == "inc":
result = tf.scatter_add(var, dst.tf_indices, val,
use_locking=False)
self.write_types["scatter_add"] += 1
else:
result = tf.scatter_update(var, dst.tf_indices, val,
use_locking=False)
self.write_types["scatter_update"] += 1
self.bases[dst.key] = result
# update reads_by_base. the general workflow is
# gather -> computation -> scatter
# so when we get a scatter, we assume that that value indicates that
# all the previous gathers are complete. so we block any writes to
# those bases on the scatter value, to be sure that the
# computation step is complete before the values can be overwritten
for b in self.gather_bases:
self.reads_by_base[b] += [self.bases[dst.key]]
self.gather_bases = []
logger.debug("new dst base %s", self.bases[dst.key])
