def test_batch_gather_nd(self, axis):
batch_size = 10
axis_dims = [50, 40, 30]
x = tf.random.uniform([
batch_size,
] + axis_dims, minval=0, maxval=1)
indices = tf.random.uniform(
(batch_size, ),
minval=0,
maxval=axis_dims[axis - 1 if axis > 0 else axis],
dtype=tf.int32)
x_gathered = utils.batch_gather_nd(x, indices, axis)
if axis == 1:
identity = tf.eye(batch_size, batch_size)[:, :, tf.newaxis,
tf.newaxis]
if axis == 2:
identity = tf.eye(batch_size, batch_size)[:, tf.newaxis, :,
tf.newaxis]
if axis == 3 or axis == -1:
identity = tf.eye(batch_size, batch_size)[:, tf.newaxis,
tf.newaxis, :]
x_gathered2 = tf.reduce_sum(identity *
tf.gather(x, indices, axis=axis),
axis=axis)
diff = tf.reduce_sum(tf.math.abs(x_gathered - x_gathered2))
self.assertAlmostEqual(self.evaluate(diff), 0., 9)
def vec2mtrx(opt, p):
with tf.name_scope("vec2mtrx"):
if opt.warpType == "homography":
p1, p2, p3, p4, p5, p6, p7, p8 = tf.unstack(p, axis=1)
A = tf.transpose(tf.stack([[p3, p2, p1], [p6, -p3 - p7, p5],
[p4, p8, p7]]),
perm=[2, 0, 1])
#transpose -> 전치
elif opt.warpType == "affine":
O = tf.zeros([opt.batchSize])
p1, p2, p3, p4, p5, p6 = tf.unstack(p, axis=1)
A = tf.transpose(tf.stack([[p1, p2, p3], [p4, p5, p6], [O, O, O]]),
perm=[2, 0, 1])
else:
assert (False)
# matrix exponential
pMtrx = tf.tile(tf.expand_dims(tf.eye(3), axis=0),
[opt.batchSize, 1, 1])
numer = tf.tile(tf.expand_dims(tf.eye(3), axis=0),
[opt.batchSize, 1, 1])
denom = 1.0
for i in range(1, opt.warpApprox):
numer = tf.matmul(numer, A)
denom *= i
pMtrx += numer / denom
return pMtrx
def _create_gaussian(self, gaussian_type):
mu = tf.random_normal([3])
if gaussian_type == tfp.distributions.MultivariateNormalDiag:
scale_diag = tf.random_normal([3])
dist = tfp.distributions.MultivariateNormalDiag(mu, scale_diag)
if gaussian_type == tfp.distributions.MultivariateNormalDiagPlusLowRank:
scale_diag = tf.random_normal([3])
perturb_factor = tf.random_normal([3, 2])
scale_perturb_diag = tf.random_normal([2])
dist = tfp.distributions.MultivariateNormalDiagPlusLowRank(
mu,
scale_diag,
scale_perturb_factor=perturb_factor,
scale_perturb_diag=scale_perturb_diag)
if gaussian_type == tfp.distributions.MultivariateNormalTriL:
cov = tf.random_uniform([3, 3], minval=0, maxval=1.0)
# Create a PSD matrix.
cov = 0.5 * (cov + tf.transpose(cov)) + 3 * tf.eye(3)
scale = tf.cholesky(cov)
dist = tfp.distributions.MultivariateNormalTriL(mu, scale)
if gaussian_type == tfp.distributions.MultivariateNormalFullCovariance:
cov = tf.random_uniform([3, 3], minval=0, maxval=1.0)
# Create a PSD matrix.
cov = 0.5 * (cov + tf.transpose(cov)) + 3 * tf.eye(3)
dist = tfp.distributions.MultivariateNormalFullCovariance(mu, cov)
return (dist, mu, dist.covariance())
def create_q_network(self, state_dim, action_dim):
""" Assemble Critic network to predict q-values
"""
a_layer1_size = 100
a_layer2_size = 50
s_layer1_size = 100
s_layer2_size = 50
combined_layer1_size = 100
combined_layer2_size = 100
state_input = tf.placeholder("float", [None, state_dim])
action_input = tf.placeholder("float", [None, action_dim])
a_W1 = self.variable([action_dim, a_layer1_size], action_dim)
a_b1 = self.variable([a_layer1_size], action_dim)
a_W2 = self.variable([a_layer1_size, a_layer2_size], a_layer1_size)
a_b2 = self.variable([a_layer2_size], a_layer1_size)
s_W1 = self.variable([state_dim + action_dim, s_layer1_size],
state_dim + action_dim)
s_b1 = self.variable([s_layer1_size], state_dim + action_dim)
# s_W2 = tf.Variable(tf.random_uniform([s_layer1_size, s_layer2_size], -3e-5, 3e-5))
# s_b2 = tf.Variable(tf.random_uniform([s_layer2_size], -3e-5))
W1_action = tf.Variable(
tf.eye(a_layer2_size, num_columns=combined_layer1_size))
W1_state = tf.Variable(tf.zeros([s_layer1_size, combined_layer1_size]))
b1 = tf.Variable(tf.zeros([combined_layer1_size]))
W2 = tf.Variable(
tf.eye(combined_layer1_size, num_columns=combined_layer2_size))
b2 = tf.Variable(tf.zeros([combined_layer2_size]))
W3 = tf.Variable(
tf.random_uniform([combined_layer2_size, 1], -3e-3, 3e-3))
b3 = tf.Variable(tf.random_uniform([1], -3e-3, 3e-3))
a_layer1 = tf.nn.relu(tf.matmul(action_input, a_W1) + a_b1)
a_layer2 = tf.nn.relu(tf.matmul(a_layer1, a_W2) + a_b2)
s_layer1 = tf.nn.relu(
tf.matmul(tf.concat([action_input, state_input], axis=1), s_W1) +
s_b1)
# s_layer2 = tf.nn.relu(tf.matmul(s_layer1, s_W2) + s_b2)
combined_layer1 = tf.nn.relu(
tf.matmul(a_layer2, W1_action) + tf.matmul(s_layer1, W1_state) +
b1)
combined_layer2 = tf.nn.relu(tf.matmul(combined_layer1, W2) + b2)
q_value_output = tf.identity(tf.matmul(combined_layer2, W3) + b3)
return state_input, action_input, q_value_output, [
a_W1, a_b1, a_W2, a_b2, s_W1, s_b1, W1_action, W1_state, b1, W2,
b2, W3, b3
]
def regularizer(self, kl_loss, z_mean, z_logvar, z_sampled, group_feats_G,
lie_alg_basis, batch_size):
del z_mean, z_logvar, z_sampled
mat_dim = group_feats_G.get_shape().as_list()[1]
gfeats_G = group_feats_G[:batch_size]
# gfeats_G_sum = group_feats_G[batch_size:batch_size + batch_size // 2]
# gfeats_G_mul = tf.matmul(gfeats_G[:batch_size // 2],
# gfeats_G[batch_size // 2:batch_size])
lie_alg_basis_square = lie_alg_basis * lie_alg_basis
# [1, lat_dim, mat_dim, mat_dim]
_, lat_dim, mat_dim, _ = lie_alg_basis.get_shape().as_list()
lie_alg_basis_col = tf.reshape(lie_alg_basis,
[lat_dim, 1, mat_dim, mat_dim])
lie_alg_basis_mul = tf.matmul(lie_alg_basis, lie_alg_basis_col)
lie_alg_basis_mask = 1. - tf.eye(
lat_dim, dtype=lie_alg_basis_mul.dtype)[:, :, tf.newaxis,
tf.newaxis]
lie_alg_basis_mul = lie_alg_basis_mul * lie_alg_basis_mask
# gmat_loss = tf.reduce_mean(
# tf.reduce_sum(tf.square(gfeats_G_mul - gfeats_G_sum), axis=[1, 2]))
hessian_loss = tf.reduce_mean(
tf.reduce_sum(tf.square(lie_alg_basis_mul), axis=[2, 3]))
lin_loss = tf.reduce_mean(
tf.reduce_sum(lie_alg_basis_square, axis=[2, 3]))
# loss = self.hy_gmat * gmat_loss + self.hy_hes * hessian_loss + self.hy_lin * lin_loss
loss = self.hy_hes * hessian_loss + self.hy_lin * lin_loss
return kl_loss + loss
def convolve(image, pixel_filter, channels=3, name=None):
"""Perform a 2D pixel convolution on the given image.
Arguments:
image: A 3D `float32` `Tensor` of shape `[height, width, channels]`,
where `channels` is the third argument to this function and the
first two dimensions are arbitrary.
pixel_filter: A 2D `Tensor`, representing pixel weightings for the
kernel. This will be used to create a 4D kernel---the extra two
dimensions are for channels (see `tf.nn.conv2d` documentation),
and the kernel will be constructed so that the channels are
independent: each channel only observes the data from neighboring
pixels of the same channel.
channels: An integer representing the number of channels in the
image (e.g., 3 for RGB).
Returns:
A 3D `float32` `Tensor` of the same shape as the input.
"""
with tf.name_scope(name, "convolve"):
tf.assert_type(image, tf.float32)
channel_filter = tf.eye(channels)
filter_ = tf.expand_dims(tf.expand_dims(pixel_filter, -1),
-1) * tf.expand_dims(
tf.expand_dims(channel_filter, 0), 0)
result_batch = tf.nn.conv2d(
tf.stack([image]), # batch
filter=filter_,
strides=[1, 1, 1, 1],
padding="SAME",
)
return result_batch[0] # unbatch
def __init__(self, input_size, output_size, dt, learning_rate=5e-2):
"""The class initializer.
Parameters:
-----------
input size: int, how many inputs does the system have.
output size: int, how many outputs does the system have.
learning rate: learnin rate for adam optimizer"""
self.output_size = output_size
self.input_size = input_size
self.dt_c = tf.constant(dt)
self.Av = tf.Variable(tf.random_normal([output_size, output_size]))
self.Bv = tf.Variable(tf.random_normal([output_size, input_size]))
self.y0 = tf.placeholder(tf.float32, [output_size, None])
self.y = tf.placeholder(tf.float32, [output_size, None])
self.u = tf.placeholder(tf.float32, [input_size, None])
self.sess = tf.Session()
eye = tf.eye(output_size)
temp0 = eye - self.Av * self.dt_c
temp1 = tf.linalg.inv(temp0)
temp2 = tf.keras.backend.dot(self.Bv * self.dt_c, self.u) + self.y0
y_ = tf.keras.backend.dot(temp1, temp2)
self.cost = tf.reduce_mean(tf.losses.mean_squared_error(self.y, y_))
self.optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(self.cost)
def derotation(src_img, trt_img, rotation, input_fov, output_fov, output_shape,
derotate_both):
"""Transform a pair of images to cancel out the rotation.
Args:
src_img: [BATCH, HEIGHT, WIDTH, CHANNEL] input source images.
trt_img: [BATCH, HEIGHT, WIDTH, CHANNEL] input target images.
rotation: [BATCH, 3, 3] relative rotations between src_img and trt_img.
input_fov: [BATCH] a 1-D tensor (float32) of input field of view in degrees.
output_fov: (float) output field of view in degrees.
output_shape: a 2-D list of output dimension [height, width].
derotate_both: Derotate both input images to an intermediate frame using
half of the relative rotation between them.
Returns:
transformed images [BATCH, height, width, CHANNELS].
"""
batch = src_img.shape.as_list()[0]
if derotate_both:
half_derotation = half_rotation(rotation)
transformed_src = transformation.rotate_image_in_3d(
src_img, tf.matrix_transpose(half_derotation), input_fov,
output_fov, output_shape)
transformed_trt = transformation.rotate_image_in_3d(
trt_img, half_derotation, input_fov, output_fov, output_shape)
else:
transformed_src = transformation.rotate_image_in_3d(
src_img, tf.eye(3, batch_shape=[batch]), input_fov, output_fov,
output_shape)
transformed_trt = transformation.rotate_image_in_3d(
trt_img, rotation, input_fov, output_fov, output_shape)
return (transformed_src, transformed_trt)
def get_bspline_kernel(x, channels, transpose=False, dtype=tf.float32, order=4):
"""Creates a 5x5x5 b-spline kernel.
Args:
num_channels: The number of channels of the image to filter.
dtype: The type of an element in the kernel.
Returns:
A tensor of shape `[5, 5, 5, num_channels, num_channels]`.
"""
mesh = x.mesh
in_dim = x.shape[-1]
num_channels = channels.size
if order == 8:
kernel = np.array(( 1., 8., 28., 56., 70., 56., 28., 8., 1.), dtype=dtype.as_numpy_dtype())
elif order == 6:
kernel = np.array(( 1., 6., 15., 20., 15., 6., 1.), dtype=dtype.as_numpy_dtype())
elif order==2:
kernel = np.array(( 1., 2., 1.), dtype=dtype.as_numpy_dtype())
else:
kernel = np.array(( 1., 4., 6., 4., 1.), dtype=dtype.as_numpy_dtype())
size = len(kernel)
kernel = np.einsum('ij,k->ijk', np.outer(kernel, kernel), kernel)
kernel /= np.sum(kernel)
kernel = kernel[:, :, :, np.newaxis, np.newaxis]
kernel = tf.constant(kernel, dtype=dtype) * tf.eye(num_channels, dtype=dtype)
fd_dim = mtf.Dimension("fd", size)
fh_dim = mtf.Dimension("fh", size)
fw_dim = mtf.Dimension("fw", size)
if transpose:
return mtf.import_tf_tensor(mesh, kernel, shape=[fd_dim, fh_dim, fw_dim, channels, in_dim])
else:
return mtf.import_tf_tensor(mesh, kernel, shape=[fd_dim, fh_dim, fw_dim, in_dim, channels])
def infer_loss(self, x, y):
caps = self.ncaps - (self.n_iter - 1) * self.dcaps
with tf.variable_scope("ret_-2"):
ret_uw = tf.get_variable(
shape=[self.nhidden, caps],
initializer=tf.contrib.layers.xavier_initializer(),
name='weights')
ret_ub = tf.get_variable(shape=[caps],
initializer=tf.zeros_initializer(),
name='bias')
x_class = tf.matmul(tf.reshape(x[-1], [-1, self.nhidden]),
ret_uw) + ret_ub
y_class = tf.matmul(tf.reshape(y[-1], [-1, self.nhidden]),
ret_uw) + ret_ub
label = tf.tile(tf.eye(caps), [self.batch_size, 1])
user_infer_loss = tf.reduce_mean(
tf.reduce_sum(
tf.nn.softmax_cross_entropy_with_logits(labels=label,
logits=x_class)))
news_infer_loss = tf.reduce_mean(
tf.reduce_sum(
tf.nn.softmax_cross_entropy_with_logits(labels=label,
logits=y_class)))
loss = user_infer_loss + news_infer_loss
return loss, ret_uw
def _constant_kernel(self,
shape,
value=1.0,
diag=False,
flip=False,
regularizer=None,
trainable=None,
name=None):
name = 'fixed_w' if name is None else name + '/fixed_w'
with tf.device('/device:GPU:' + GPU_ID):
if not diag:
k = tf.get_variable(
name,
shape,
dtype=self.dtype,
initializer=tf.constant_initializer(value=value),
regularizer=regularizer,
trainable=trainable)
else:
w = tf.eye(shape[0], num_columns=shape[1])
if flip:
w = tf.reshape(w, (shape[0], shape[1], 1))
w = tf.image.flip_left_right(w)
w = tf.reshape(w, shape)
k = tf.get_variable(
name,
None,
dtype=self.
dtype, # constant initializer dont specific shape
initializer=w,
regularizer=regularizer,
trainable=trainable)
return k
def stateful_ds_creator():
whole_batch = tf.eye(batch_io_dim, dtype=tf.float32)
sub_batch = tf.slice(whole_batch,
[self.sub_batch_created_times * 2, 0],
[2, 4])
self.sub_batch_created_times += 1
return tf.data.Dataset.from_tensors(sub_batch).repeat().unbatch()
def initial_state(self, batch_size, trainable=False):
"""Creates the initial memory.
We should ensure each row of the memory is initialized to be unique,
so initialize the matrix to be the identity. We then pad or truncate
as necessary so that init_state is of size
(batch_size, self._mem_slots, self._mem_size).
Args:
batch_size: The size of the batch.
trainable: Whether the initial state is trainable. This is always True.
Returns:
init_state: A truncated or padded matrix of size
(batch_size, self._mem_slots, self._mem_size).
"""
init_state = tf.eye(self._mem_slots, batch_shape=[batch_size])
# Pad the matrix with zeros.
if self._mem_size > self._mem_slots:
difference = self._mem_size - self._mem_slots
pad = tf.zeros((batch_size, self._mem_slots, difference))
init_state = tf.concat([init_state, pad], -1)
# Truncation. Take the first `self._mem_size` components.
elif self._mem_size < self._mem_slots:
init_state = init_state[:, :, :self._mem_size]
return init_state
def _get_triplet_mask(labels):
"""Return a 3D mask where mask[a, p, n] is True iff the triplet (a, p, n) is valid.
A triplet (i, j, k) is valid if:
- i, j, k are distinct
- labels[i] == labels[j] and labels[i] != labels[k]
Args:
labels: tf.int32 `Tensor` with shape [batch_size]
"""
# Check that i, j and k are distinct
indices_equal = tf.cast(tf.eye(tf.shape(labels)[0]), tf.bool)
indices_not_equal = tf.logical_not(indices_equal)
i_not_equal_j = tf.expand_dims(indices_not_equal, 2)
i_not_equal_k = tf.expand_dims(indices_not_equal, 1)
j_not_equal_k = tf.expand_dims(indices_not_equal, 0)
distinct_indices = tf.logical_and(
tf.logical_and(i_not_equal_j, i_not_equal_k), j_not_equal_k)
# Check if labels[i] == labels[j] and labels[i] != labels[k]
label_equal = tf.equal(tf.expand_dims(labels, 0),
tf.expand_dims(labels, 1))
i_equal_j = tf.expand_dims(label_equal, 2)
i_equal_k = tf.expand_dims(label_equal, 1)
valid_labels = tf.logical_and(i_equal_j, tf.logical_not(i_equal_k))
# Combine the two masks
mask = tf.logical_and(distinct_indices, valid_labels)
return mask
def rodrigues(r):
"""
Rodrigues' rotation formula that turns axis-angle tensor into rotation
matrix in a batch-ed manner.
Parameter:
----------
r: Axis-angle rotation tensor of shape [batch_size, 1, 3].
Return:
-------
Rotation matrix of shape [batch_size, 3, 3].
"""
theta = tf.norm(r + tf.random_normal(r.shape, 0, 1e-8, dtype=tf.float64),
axis=(1, 2),
keepdims=True)
# avoid divide by zero
r_hat = r / theta
cos = tf.cos(theta)
z_stick = tf.zeros(theta.get_shape().as_list()[0], dtype=tf.float64)
m = tf.stack(
(z_stick, -r_hat[:, 0, 2], r_hat[:, 0, 1], r_hat[:, 0, 2], z_stick,
-r_hat[:, 0, 0], -r_hat[:, 0, 1], r_hat[:, 0, 0], z_stick),
axis=1)
m = tf.reshape(m, (-1, 3, 3))
i_cube = tf.expand_dims(tf.eye(3, dtype=tf.float64), axis=0) + tf.zeros(
(theta.get_shape().as_list()[0], 3, 3), dtype=tf.float64)
A = tf.transpose(r_hat, (0, 2, 1))
B = r_hat
dot = tf.matmul(A, B)
R = cos * i_cube + (1 - cos) * dot + tf.sin(theta) * m
return R
def __init__(self,
state_size,
action_size,
gamma_reg=0.001,
minibatch_size=5,
**kwargs):
"""
Parameters
----------
state_size, action_size : int
Size of the environment state space and action space
gamma_reg : float, optional
LS-IELM regularisation parameter
minibatch_size : int, optional
Size of minibatches for updating
**kwargs
Additional keyword arguments passed to `SingleLayerNetwork`
"""
super().__init__(state_size, action_size, **kwargs)
self.k = int(minibatch_size)
self.prep_state = self.act
self.H = tf.placeholder(shape=[self.k, self.N_hid], dtype=tf.float32)
self.T = tf.placeholder(shape=[self.k, action_size], dtype=tf.float32)
H_t = tf.transpose(self.H)
A_inv = tf.Variable(tf.random_uniform([self.N_hid, self.N_hid], 0, 1))
A0 = tf.add(tf.scalar_mul(1.0 / gamma_reg, tf.eye(self.N_hid)),
tf.matmul(H_t, self.H))
A0_inv = tf.matrix_inverse(A0)
W0 = tf.matmul(A0_inv, tf.matmul(H_t, self.T))
self.initModel = (self.W.assign(W0), A_inv.assign(A0_inv))
K1 = tf.add(tf.matmul(self.H, tf.matmul(A_inv, H_t)), tf.eye(self.k))
K_t = tf.subtract(
tf.eye(self.N_hid),
tf.matmul(A_inv,
tf.matmul(H_t, tf.matmul(tf.matrix_inverse(K1),
self.H))))
W_new = tf.add(
tf.matmul(K_t, self.W),
tf.matmul(tf.matmul(K_t, A_inv), tf.matmul(H_t, self.T)))
A_new = tf.matmul(K_t, A_inv)
self.updateModel = (self.W.assign(W_new), A_inv.assign(A_new))
self.first = True
self.var_init()
def test_raw_spectral_norm(self):
with tf.Graph().as_default():
ones_weight = 4 * tf.eye(8, 8)
sigma = spectral_normalization.spectral_norm(ones_weight)['sigma']
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
_, sigma_v = sess.run([ones_weight, sigma])
self.assertLess(abs(4.0 - sigma_v), _ACCEPTABLE_ERROR)
def f(x, A):
I = tf.eye(int(A.shape[0]), dtype=tf.float64)
xT = tf.transpose(x)
value1 = tf.matmul(xT, x) * A
value2 = (1 - tf.matmul(tf.matmul(xT, A), x)) * I
return tf.matmul((value1 + value2), x)
def _materialise_conv2d(w, b, input_height, input_width, padding, strides):
"""Converts a convolution to an equivalent linear layer.
Args:
w: 4D tensor of shape (kernel_height, kernel_width, input_channels,
output_channels) containing the convolution weights.
b: 1D tensor of shape (output_channels) containing the convolution biases,
or `None` if no biases.
input_height: height of the input tensor.
input_width: width of the input tensor.
padding: `"VALID"` or `"SAME"`, the convolution's padding algorithm.
strides: Integer list of `[vertical_stride, horizontal_stride]`.
Returns:
w: 2D tensor of shape (input_height * input_width * input_channels,
output_height * output_width * output_channels) containing weights.
b: 1D tensor of shape (output_height * output_width * output_channels)
containing biases, or `None` if no biases.
"""
kernel_height = w.shape[0].value
kernel_width = w.shape[1].value
input_channels = w.shape[2].value
output_channels = w.shape[3].value
# Temporarily move the input_channels dimension to output_channels.
w = tf.reshape(w,
shape=(kernel_height, kernel_width, 1,
input_channels * output_channels))
# Apply the convolution to elementary (i.e. one-hot) inputs.
diagonal_input = tf.reshape(
tf.eye(input_height * input_width, dtype=w.dtype),
shape=[input_height * input_width, input_height, input_width, 1])
conv = tf.nn.convolution(diagonal_input,
w,
padding=padding,
strides=strides)
output_height = conv.shape[1].value
output_width = conv.shape[2].value
# conv is of shape (input_height * input_width, output_height, output_width,
# input_channels * output_channels).
# Reshape it to (input_height * input_width * input_channels,
# output_height * output_width * output_channels).
w = tf.reshape(conv,
shape=([
input_height * input_width, output_height, output_width,
input_channels, output_channels
]))
w = tf.transpose(w, perm=[0, 3, 1, 2, 4])
w = tf.reshape(w,
shape=([
input_height * input_width * input_channels,
output_height * output_width * output_channels
]))
# Broadcast b over spatial dimensions.
b = tf.tile(b, [output_height * output_width]) if b is not None else None
return w, b
def generate_heatmap_target_sigmas_rotation(heatmap_size, landmarks, sigmas, rotation, scale=1.0, normalize=False, data_format='channels_first'):
"""
Generates heatmap images for the given parameters.
:param heatmap_size: The image size of a single heatmap.
:param landmarks: The list of landmarks. For each landmark, a heatmap on the given coordinate will be generated. If landmark.is_valid is False, then the heatmap will be empty.
:param sigmas: The sigmas for the individual heatmaps. May be either fixed, or trainable.
:param rotation: The rotation of the heatmap. May be either fixed, or trainable.
:param scale: The scale factor for each heatmap. Each pixel value will be multiplied by this value.
:param normalize: If true, each heatmap value will be multiplied by the normalization factor of the gaussian.
:param data_format: The data format of the resulting tensor of heatmap images.
:return: The tensor of heatmap images.
"""
landmarks_shape = landmarks.get_shape().as_list()
sigmas_shape = sigmas.get_shape().as_list()
batch_size = landmarks_shape[0]
num_landmarks = landmarks_shape[1]
dim = landmarks_shape[2] - 1
assert dim == 2, 'Currently only dim == 2 is supported.'
assert len(heatmap_size) == dim, 'Dimensions do not match.'
assert sigmas_shape[0] == num_landmarks, 'Number of sigmas does not match.'
rotation_matrix = tf.stack([tf.stack([tf.cos(rotation), -tf.sin(rotation)], axis=-1), tf.stack([tf.sin(rotation), tf.cos(rotation)], axis=-1)], axis=-1)
rotation_matrix_t = tf.stack([tf.stack([tf.cos(rotation), tf.sin(rotation)], axis=-1), tf.stack([-tf.sin(rotation), tf.cos(rotation)], axis=-1)], axis=-1)
det_covariances = tf.reduce_prod(sigmas, axis=-1)
sigmas_inv_eye = tf.eye(dim, dim, batch_shape=[num_landmarks]) * tf.expand_dims(1.0 / sigmas, -1)
inv_covariances = tf.matmul(tf.matmul(rotation_matrix, sigmas_inv_eye), rotation_matrix_t)
if data_format == 'channels_first':
heatmap_axis = 1
landmarks_reshaped = tf.reshape(landmarks[..., 1:], [batch_size, num_landmarks] + [1] * dim + [dim])
is_valid_reshaped = tf.reshape(landmarks[..., 0], [batch_size, num_landmarks] + [1] * dim)
det_covariances_reshaped = tf.reshape(det_covariances, [1, num_landmarks] + [1] * dim)
inv_covariances_reshaped = tf.reshape(inv_covariances, [1, num_landmarks] + [1] * dim + [dim, dim])
else:
heatmap_axis = dim + 1
landmarks_reshaped = tf.reshape(landmarks[..., 1:], [batch_size] + [1] * dim + [num_landmarks, dim])
is_valid_reshaped = tf.reshape(landmarks[..., 0], [batch_size] + [1] * dim + [num_landmarks])
det_covariances_reshaped = tf.reshape(det_covariances, [1] + [1] * dim + [num_landmarks])
inv_covariances_reshaped = tf.reshape(inv_covariances, [1] + [1] * dim + [num_landmarks, dim, dim])
aranges = [np.arange(s) for s in heatmap_size]
grid = tf.meshgrid(*aranges, indexing='ij')
grid_stacked = tf.stack(grid, axis=dim)
grid_stacked = tf.cast(grid_stacked, tf.float32)
grid_stacked = tf.stack([grid_stacked] * batch_size, axis=0)
grid_stacked = tf.stack([grid_stacked] * num_landmarks, axis=heatmap_axis)
if normalize:
scale /= tf.sqrt(tf.pow(2 * np.pi, dim) * det_covariances_reshaped)
x_minus_mu = grid_stacked - landmarks_reshaped
exp_factor = tf.reduce_sum(tf.reduce_sum(tf.expand_dims(x_minus_mu, -1) * inv_covariances_reshaped * tf.expand_dims(x_minus_mu, -2), axis=-1), axis=-1)
heatmap = scale * tf.exp(-0.5 * exp_factor)
heatmap_or_zeros = tf.where((is_valid_reshaped + tf.zeros_like(heatmap)) > 0, heatmap, tf.zeros_like(heatmap))
return heatmap_or_zeros
def f(v, A):
""" function for returning f(v) as described in the paper by Yi et al. (eq 1) """
n = int(A.shape[0])
I = tf.eye(n, dtype=tf.float64)
vt = tf.transpose(v)
return tf.matmul(
(tf.matmul(vt, v) * A + (1 - tf.matmul(tf.matmul(vt, A), v)) * I), v)
def body(self, features):
if self.hparams.mode != tf.estimator.ModeKeys.PREDICT:
# In training mode we need to embed both the queries and the code
# using the inputs and targets respectively.
with tf.variable_scope('string_embedding'):
string_embedding = self.encode(features, 'inputs')
with tf.variable_scope('code_embedding'):
code_embedding = self.encode(features, 'targets')
string_embedding_norm = tf.nn.l2_normalize(string_embedding,
axis=1)
code_embedding_norm = tf.nn.l2_normalize(code_embedding, axis=1)
# All-vs-All cosine distance matrix, reshaped as row-major.
cosine_dist = 1.0 - tf.matmul(
string_embedding_norm, code_embedding_norm, transpose_b=True)
cosine_dist_flat = tf.reshape(cosine_dist, [-1, 1])
# Positive samples on the diagonal, reshaped as row-major.
label_matrix = tf.eye(tf.shape(cosine_dist)[0], dtype=tf.int32)
label_matrix_flat = tf.reshape(label_matrix, [-1])
logits = tf.concat([1.0 - cosine_dist_flat, cosine_dist_flat],
axis=1)
labels = tf.one_hot(label_matrix_flat, 2)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels,
logits=logits)
return string_embedding_norm, {'training': loss}
# In predict mode we conditionally embed either the string query
# or the code based on the embed_code feature. In both cases the
# input will be in the inputs feature but the variable scope will
# be different
# Define predicates to be used with tf.cond
def embed_string():
with tf.variable_scope('string_embedding'):
string_embedding = self.encode(features, 'inputs')
return string_embedding
def embed_code():
with tf.variable_scope('code_embedding'):
code_embedding = self.encode(features, 'inputs')
return code_embedding
embed_code_feature = features.get('embed_code')
# embed_code_feature will be a tensor because inputs will be a batch
# of inputs. We need to reduce that down to a single value for use
# with tf.cond; so we simply take the max of all the elements.
# This implicitly assume all inputs have the same value.
is_embed_code = tf.reduce_max(embed_code_feature)
result = tf.cond(is_embed_code > 0, embed_code, embed_string)
result = tf.nn.l2_normalize(result)
return result
def f(x, A):
#Computes the function from Zhang and Yan article
I = tf.eye(n, dtype=tf.float64)
a = tf.matmul(tf.transpose(x), x) * A
b = I - I * tf.matmul(tf.matmul(tf.transpose(x), A), x)
return tf.matmul((a + b), x)
def _aug_smooth(self, images):
"""Smooths the image using a 5x5 uniform kernel."""
depth = int(images.shape[-1])
image_filter = tf.eye(num_rows=depth,
num_columns=depth,
dtype=tf.float32)
image_filter = tf.stack([image_filter] * 3, axis=0)
image_filter = tf.stack([image_filter] * 3, axis=0)
output = tf.nn.conv2d(images, image_filter / 9.0, padding='SAME')
return 255.0 * self._normalize(output)
def get_metrics(links, idx):
v_rot_idx = tf.gather(links.v_rot, idx)
# all possible seg pairs
V = v_rot_idx[newaxis] + v_rot_idx[:, newaxis]
eps = 1e-9
eye_eps = -eps * tf.eye(
links.points0.shape[-1], batch_shape=(1, 1), dtype=tf.float64)
ginv = tf.to_double(tf.matmul(V, V, transpose_b=True)) + eye_eps
gij = tf.to_float(tf.matrix_inverse(ginv))
return gij, ginv
def test_spectral_norm_creates_variables(self):
with tf.Graph().as_default():
ones_weight = 4 * tf.eye(8, 8)
pre_spec_norm_vars = tf.global_variables()
_ = spectral_normalization.spectral_norm(ones_weight)
post_spec_norm_vars = tf.global_variables()
self.assertEmpty(pre_spec_norm_vars)
self.assertLen(post_spec_norm_vars, 1)
self.assertEqual(post_spec_norm_vars[0].name.split('/')[-1],
'u0:0')
def get_decoder_mask(self_attn_inputs):
"""Returns causal mask to apply for self-attention layer.
Args:
self_attn_inputs: Inputs to self attention layer to determine mask shape
"""
len_s = tf.shape(self_attn_inputs)[1]
bs = tf.shape(self_attn_inputs)[:1]
mask = K.cumsum(tf.eye(len_s, batch_shape=bs), 1)
return mask
def testLossFunctionWithoutName(self):
"""Ensure loss functions get unique names if 'name' not specified."""
with tf.Graph().as_default():
logits = tf.eye(2)
lc = layer_collection.LayerCollection()
# Create a new loss function with default names.
lc.register_categorical_predictive_distribution(logits)
lc.register_categorical_predictive_distribution(logits)
self.assertEqual(2, len(lc.losses))
def f(x, A_tf):
"""
function denoted as f in the paper by YI et al.
x is tensor of size (n,1)
returns tensor of size (n,1)
"""
I = tf.eye(n, dtype=tf.float64)
xT = tf.transpose(x)
term1 = tf.matmul(xT, x) * A_tf
term2 = (1 - tf.matmul(tf.matmul(xT, A_tf), x)) * I
return tf.matmul((term1 + term2), x)
def solve_woodbury_sum_onefull_oneinv(A_full, Binv_diag, target):
n_B = tf.shape(Binv_diag)[1]
Binv_target = Binv_diag * target
# (A+B)inv * target = (Binv * target) - Binv * (Ainv + Binv)inv * (Binv * target)
Ainv = tf.linalg.inv(A_full)
Binv_full = tf.eye(n_B, dtype=T)[None, ...] * Binv_diag
Ainv_plus_Binv = Ainv + Binv_full
temp = tf.linalg.solve(Ainv_plus_Binv, Binv_target)
print(tf.shape(Ainv_plus_Binv), tf.shape(temp))
corr = tf.matmul(Binv_full, temp)
return Binv_target - corr