def _row_kernel(upsampled_region_size, upsample_factor, axis_offsets,
data_shape):
data_shape_float = tf.cast(data_shape, tf.float32)
row_constant = tf.cast(data_shape_float[1] * upsample_factor, tf.complex64)
row_constant = (-1j * 2 * np.pi / row_constant)
row_kernel_a = tf.range(0, upsampled_region_size, dtype=tf.float32)
row_kernel_a = tf.reshape(row_kernel_a, (1, -1))
row_kernel_a = tf.tile(row_kernel_a, (data_shape[0], 1))
row_kernel_a = tf.transpose(row_kernel_a)
row_kernel_a = row_kernel_a - axis_offsets[:, 0]
row_kernel_b = tf.range(0, data_shape_float[1], dtype=tf.float32)
row_kernel_b = fftshift1d(row_kernel_b)
row_kernel_b = tf.reshape(row_kernel_b, (1, -1))
row_kernel_b = tf.tile(row_kernel_b, (data_shape[0], 1))
row_kernel_b = row_kernel_b - tf.floor(data_shape_float[1] / 2.)
row_kernel_a = tf.expand_dims(row_kernel_a, 1)
row_kernel_b = tf.expand_dims(row_kernel_b, -1)
row_kernel = tf.transpose(row_kernel_a) * row_kernel_b
row_kernel = tf.transpose(row_kernel, perm=(0, 2, 1))
row_kernel = row_constant * tf.cast(row_kernel, tf.complex64)
row_kernel = tf.exp(row_kernel)
return row_kernel
def call(self, inputs):
kernel_shape = K.int_shape(self.kernel)
if self.renormalize:
w = K.reshape(self.kernel, (-1, kernel_shape[-1]))
sigma, u_bar = max_singular_val(
w,
self.u,
fully_differentiable=self.fully_diff_spectral,
ip=self.spectral_iterations)
else:
w = ktf.transpose(self.kernel, (0, 3, 1, 2))
w = K.reshape(w, [-1, kernel_shape[1] * kernel_shape[2]])
w = K.expand_dims(w, axis=-1)
sigma, u_bar = max_singular_val(
w,
self.u,
transpose=lambda x: ktf.transpose(x, [0, 2, 1]),
fully_differentiable=self.fully_diff_spectral,
ip=self.spectral_iterations)
sigma = K.reshape(sigma, [kernel_shape[0], 1, 1, kernel_shape[-1]])
self.add_update(K.update(self.u, u_bar))
kernel = self.kernel
self.kernel = self.kernel / sigma
outputs = super(SNConditionalDepthwiseConv2D, self).call(inputs)
self.kernel = kernel
return outputs
def _col_kernel(upsampled_region_size, upsample_factor, axis_offsets,
data_shape):
data_shape_float = tf.cast(data_shape, tf.float32)
col_constant = tf.cast(data_shape_float[2] * upsample_factor, tf.complex64)
col_constant = (-1j * 2 * np.pi / col_constant)
col_kernel_a = tf.range(0, data_shape_float[2], dtype=tf.float32)
col_kernel_a = fftshift1d(col_kernel_a)
col_kernel_a = tf.reshape(col_kernel_a, (-1, 1))
col_kernel_a -= tf.floor(data_shape_float[2] / 2.)
col_kernel_a = tf.reshape(col_kernel_a, (1, -1))
col_kernel_a = tf.tile(col_kernel_a, (data_shape[0], 1))
col_kernel_b = tf.range(0, upsampled_region_size, dtype=tf.float32)
col_kernel_b = tf.reshape(col_kernel_b, (1, -1))
col_kernel_b = tf.tile(col_kernel_b, (data_shape[0], 1))
col_kernel_b = tf.transpose(col_kernel_b)
col_kernel_b -= tf.transpose(axis_offsets[:, 1])
col_kernel_b = tf.transpose(col_kernel_b)
col_kernel_a = tf.expand_dims(col_kernel_a, 1)
col_kernel_b = tf.expand_dims(col_kernel_b, -1)
col_kernel = col_kernel_a * col_kernel_b
col_kernel = tf.transpose(col_kernel, perm=(0, 2, 1))
col_kernel = col_constant * tf.cast(col_kernel, tf.complex64)
col_kernel = tf.exp(col_kernel)
return col_kernel
def call(self, inputs):
print("xxxx",inputs)
expanded_tensor = ktf.expand_dims(inputs[0], -1)
multiples = [1, self.number_of_transforms, 1, 1, 1]
tiled_tensor = ktf.tile(expanded_tensor, multiples=multiples)
repeated_tensor = ktf.reshape(tiled_tensor, ktf.shape(inputs[0]) * np.array([self.number_of_transforms, 1, 1, 1]))
affine_transforms = inputs[1] / self.affine_mul
affine_transforms = ktf.reshape(affine_transforms, (-1, 8))
tranformed = tf_affine_transform(repeated_tensor, affine_transforms)
res = ktf.reshape(tranformed, [-1, self.number_of_transforms] + self.image_size)
res = ktf.transpose(res, [0, 2, 3, 1, 4])
#Use masks
if len(inputs) == 3:
mask = ktf.transpose(inputs[2], [0, 2, 3, 1])
mask = ktf.image.resize_images(mask, self.image_size[:2], method=ktf.image.ResizeMethod.NEAREST_NEIGHBOR)
res = res * ktf.expand_dims(mask, axis=-1)
if self.aggregation_fn == 'none':
res = ktf.reshape(res, [-1] + self.image_size[:2] + [self.image_size[2] * self.number_of_transforms])
elif self.aggregation_fn == 'max':
res = ktf.reduce_max(res, reduction_indices=[-2])
elif self.aggregation_fn == 'avg':
counts = ktf.reduce_sum(mask, reduction_indices=[-1])
counts = ktf.expand_dims(counts, axis=-1)
res = ktf.reduce_sum(res, reduction_indices=[-2])
res /= counts
res = ktf.where(ktf.is_nan(res), ktf.zeros_like(res), res)
return res
def get_image_emotion(image, net):
net = load_model(net, compile=True)
image = ktf.transpose(image, [0, 2, 3, 1])
image = ktf.image.resize_images(image, (299, 299), )
image = ktf.transpose(image, [0, 3, 1, 2])
score = net(image)
return score[:, 1]
def get_image_memorability(image, net='models/mem_internal.h5'):
memnet = load_model(net, custom_objects={'LRN': LRN}, compile=True)
image *= 255
image = ktf.transpose(image, [0, 2, 3, 1])
image = ktf.image.resize_images(image, (227, 227), )
image = ktf.transpose(image, [0, 3, 1, 2])
image = preprocess_symbolic_input(image, data_format='channels_first', mode='caffe')
score = memnet(image)
return score
def call(self, input_tensor):
"""
target: batch x ih x iw (queryL=ihxiw) x idf
source: batch x sourceL(seq_len) x idf
mask: batch x sourceL
-inf or 0 が入っている
"""
target, source, mask = input_tensor
idf = self.output_dim
ih = self.ih
iw = self.iw
queryL = self.queryL
sourceL = self.sourceL
# --> batch x queryL x idf
target = K.reshape(target, (-1, queryL, idf))
# --> batch x idf x sourceL
sourceT = ktf.transpose(source, perm=[0, 2, 1])
# Get attention
# (batch x queryL x idf)(batch x idf x sourceL)
# -->batch x queryL x sourceL
attn = ktf.matmul(target, sourceT)
addmask = K.switch(
self.use_mask,
lambda: K.repeat_elements(
K.reshape(mask, (-1, 1, sourceL)), rep=queryL, axis=1),
lambda: 0.0)
attn = attn + addmask
attn = K.softmax(attn)
# (batch x queryL x sourceL)(batch x sourceL x idf)
# --> batch x queryL x idf
weightedContext = ktf.matmul(attn, source)
weightedContext = K.reshape(weightedContext, (-1, ih, iw, idf))
attn = K.reshape(attn, (-1, ih, iw, sourceL)) #計算ではこの後未使用
return [weightedContext, attn]
def call(self, inputs):
kernel_shape = K.int_shape(self.kernel)
if not self.renormalize:
w = K.reshape(self.kernel,
(kernel_shape[0], kernel_shape[1] * kernel_shape[2] *
kernel_shape[3], kernel_shape[-1]))
sigma, u_bar = max_singular_val(
w,
self.u,
transpose=lambda x: ktf.transpose(x, [0, 2, 1]),
fully_differentiable=self.fully_diff_spectral,
ip=self.spectral_iterations)
sigma = K.reshape(sigma, (self.filters_emb, 1, 1, 1, 1))
else:
w = K.reshape(self.kernel, (-1, kernel_shape[-1]))
sigma, u_bar = max_singular_val(
w,
self.u,
fully_differentiable=self.fully_diff_spectral,
ip=self.spectral_iterations)
self.add_update(K.update(self.u, u_bar))
kernel = self.kernel
self.kernel = self.kernel / sigma
outputs = super(SNFactorizedConv11, self).call(inputs)
self.kernel = kernel
return outputs
def call(self, x):
x_shape = x.get_shape()
offsets = super(Pool2DOffset, self).call(x)
offsets = tf.transpose(offsets, [0, 3, 1, 2])
offsets = tf.reshape(offsets,
(-1, int(x_shape[1]), int(x_shape[2]), 2))
n_batches = tf.shape(offsets)[0]
x = tf.transpose(x, [0, 3, 1, 2])
x = tf.reshape(x, (-1, int(x_shape[1]), int(x_shape[2]), 1))
#offsets = tf.resampler(x, offsets)
x = batch_map_offsets(x, offsets)
x = tf.reshape(x,
(-1, int(x_shape[3]), int(x_shape[1]), int(x_shape[2])))
x = tf.transpose(x, [0, 2, 3, 1])
return x
def call(self, inputs):
w = self.kernel
kernel_shape = K.int_shape(self.kernel)
if self.renormalize:
w = K.reshape(w, [-1, kernel_shape[-1]])
sigma, u_bar = max_singular_val(
w,
self.u,
fully_differentiable=self.fully_diff_spectral,
ip=self.spectral_iterations)
else:
sigma, u_bar = max_singular_val(
w,
self.u,
transpose=lambda x: ktf.transpose(x, [0, 2, 1]),
fully_differentiable=self.fully_diff_spectral,
ip=self.spectral_iterations)
sigma = K.reshape(sigma, (self.number_of_classes, 1, 1))
self.add_update(K.update(self.u, u_bar))
kernel = self.kernel
self.kernel = self.kernel / sigma
outputs = super(SNCondtionalDense, self).call(inputs)
self.kernel = kernel
return outputs
def get_image_aesthetics(image, net='models/ava1.h5', ava1_mean=True):
ilgnet = load_model(net, custom_objects={'LRN': LRN}, compile=True)
if not ava1_mean:
IMAGENET_MEAN = K.constant(-np.array([90.05038554687053, 98.58527740451973, 107.50910979753826])[::-1])
else:
IMAGENET_MEAN = K.constant(-np.array([91.00178961467464, 99.46385127608664, 107.8456162486691])[::-1])
image *= 255
image = ktf.transpose(image, [0, 2, 3, 1])
image = ktf.image.resize_images(image, (227, 227), )
image = ktf.transpose(image, [0, 3, 1, 2])
image = preprocess_symbolic_input(image, data_format='channels_first', mode='caffe',
IMAGENET_MEAN=IMAGENET_MEAN)
score = ilgnet(image)
return score[:, 1]
def call(self, inputs):
mask = ktf.transpose(inputs[2], [0, 2, 3, 1])
# mask = ktf.image.resize_images(mask, self.image_size[:2], method=ktf.image.ResizeMethod.NEAREST_NEIGHBOR)
if self.number_of_transforms == 11:
# return inputs[0] * K.tile(ktf.expand_dims(1-mask[:,:,:,0], axis=3),(1,1,1,inputs[0].shape[-1]))
return inputs[0] * ktf.expand_dims(1 - mask[:, :, :, 0], axis=3)
else:
return inputs[0] * ktf.expand_dims(
K.clip(1 - K.sum(mask, 3), 0, 1), axis=3)
def upsampling_from_mask(inputs):
X_, A_, I_, M_ = inputs
S_ = tf.eye(tf.shape(M_)[0])
S_ = tf.boolean_mask(S_, M_)
S_t_ = tf.transpose(S_)
X_out_ = K.dot(S_t_, X_)
A_out_ = K.dot(K.transpose(K.dot(A_, S_)), S_)
I_out_ = K.dot(S_t_, K.cast(I_[:, None], tf.float32))[:, 0]
I_out_ = K.cast(I_out_, tf.int32)
return [X_out_, A_out_, I_out_]
def _postprocess_conv2d_output(x, data_format):
"""Transpose and cast the output from conv2d if needed.
# Arguments
x: A tensor.
data_format: string, `"channels_last"` or `"channels_first"`.
# Returns
A tensor.
"""
if data_format == 'channels_first':
x = tf.transpose(x, (0, 3, 1, 2))
if floatx() == 'float64':
x = tf.cast(x, 'float64')
return x
def _preprocess_conv2d_input(x, data_format):
"""Transpose and cast the input before the conv2d.
# Arguments
x: input tensor.
data_format: string, `"channels_last"` or `"channels_first"`.
# Returns
A tensor.
"""
if dtype(x) == 'float64':
x = tf.cast(x, 'float32')
if data_format == 'channels_first':
# TF uses the last dimension as channel dimension,
# instead of the 2nd one.
# TH input shape: (samples, input_depth, rows, cols)
# TF input shape: (samples, rows, cols, input_depth)
x = tf.transpose(x, (0, 2, 3, 1))
return x
def call(self, x):
x_shape = x.get_shape()
offsets = super(Conv2DOffset, self).call(x)
#offsets *= 10
channels = int(offsets.get_shape()[3].value)
n_batches = tf.shape(offsets)[0]
# Change offset's order from [x1, x2, ..., y1, y2, ...] to [x1, y1, x2, y2, ...]
# Codes below are written to make sure same results of MXNet implementation.
# You can remove them, and it won't influence the module's performance.
ind_shuffle = tf.concat(
[tf.range(0, channels, 2),
tf.range(1, channels + 1, 2)], axis=0)
#ind_shuffle = tf.expand_dims(ind_shuffle, axis=0)
#ind_shuffle = tf.expand_dims(ind_shuffle, axis=0)
#ind_shuffle = tf.tile(ind_shuffle, [input_w, input_h, 1])
offsets = tf.gather(offsets, ind_shuffle, axis=3)
# ------------------------------------------------------------------------
#x = tf.transpose(x, [0, 3, 1, 2])
#x = tf.reshape(x, (-1, int(x_shape[1]), int(x_shape[2])))
#offsets = tf.resampler(x, offsets)
offsets = batch_map_offsets(x, offsets)
#offsets = tf.reshape(x, (-1, int(x_shape[3]), int(x_shape[1]), int(x_shape[2])))
#offsets = tf.transpose(x, [0, 2, 3, 1])
offset_shape = offsets.get_shape()
num_channels = offset_shape[1].value
height = offset_shape[2].value
width = offset_shape[3].value
f_offset = [
tf.reshape(offsets[..., ind:ind + 3],
(-1, num_channels, height, width * 3))
for ind in range(0, 9, 3)
]
f_offset = tf.concat(f_offset, axis=-1)
f_offset = tf.reshape(f_offset,
(-1, num_channels, height * 3, width * 3))
f_offset = tf.transpose(f_offset, (0, 2, 3, 1))
return f_offset
def _get_vals_by_coords(input, coords, n_coords):
coords_shape = tf.shape(coords)
input_shape = input.get_shape()
input_w = input_shape[2].value
input_h = input_shape[1].value
channel_size = input_shape[3].value
batch_size = tf.shape(input)[0]
input = tf.transpose(input, (0, 3, 1, 2))
input = tf.reshape(input, (-1, channel_size, input_h * input_w))
indices = coords[..., 0] * input_w + coords[..., 1]
#indices = tf.expand_dims(indices, axis=1)
#indices = tf.tile(indices, [1, channel_size, 1, 1, 1])
#indices = tf.reshape(indices, (-1, channel_size, input_h * input_w * n_coords))
#indices = tf.transpose(indices, (0, 3, 1, 2))
indices = tf.reshape(indices, (-1, input_h * input_w * n_coords))
indices = tf.cast(indices, 'int32')
#indices = tf.reshape(indices, [-1])
#input = tf.reshape(input, [-1])
vals = tf.gather(input, indices[0], axis=-1)
#vals = tf.map_fn(lambda x: tf.gather(x[0], x[1], axis=-1), (input,indices), dtype=tf.float32)
vals = tf.reshape(vals, (-1, channel_size, input_h, input_w, n_coords))
return vals
def call(self, inputs, training=None):
input_shape = K.int_shape(inputs)
_, w, h, c = input_shape
bs = K.shape(inputs)[0]
#if c < self.group:
# raise ValueError('Input channels should be larger than group size' +
# '; Received input channels: ' + str(c) +
# '; Group size: ' + str(self.group)
# )
#x = K.reshape(inputs, (batch_size, h, w, self.group, c // self.group))
x_t = ktf.transpose(inputs, (3, 0, 1, 2))
#x_t = ktf.transpose(x, (3, 4, 0, 1, 2))
# BxCxHxW -> CxB*H*W
x_flat = ktf.reshape(x_t, (c, -1))
# Covariance
m = ktf.reduce_mean(x_flat, axis=1, keepdims=True)
m = K.in_train_phase(m, self.moving_mean)
f = x_flat - m
if self.decomposition == 'cholesky':
def get_inv_sqrt(ff):
sqrt = ktf.cholesky(ff)
inv_sqrt = ktf.matrix_triangular_solve(sqrt, ktf.eye(c))
return sqrt, inv_sqrt
elif self.decomposition in ['zca', 'zca-cor']:
def get_inv_sqrt(ff):
with ktf.device('/cpu:0'):
S, U, _ = ktf.svd(ff + ktf.eye(c) * self.epsilon,
full_matrices=True)
D = ktf.diag(ktf.pow(S, -0.5))
inv_sqrt = ktf.matmul(ktf.matmul(U, D), U, transpose_b=True)
D = ktf.diag(ktf.pow(S, 0.5))
sqrt = ktf.matmul(ktf.matmul(U, D), U, transpose_b=True)
return sqrt, inv_sqrt
elif self.decomposition in ['pca', 'pca-cor']:
def get_inv_sqrt(ff):
with ktf.device('/cpu:0'):
S, U, _ = ktf.svd(ff + ktf.eye(c) * self.epsilon,
full_matrices=True)
D = ktf.diag(ktf.pow(S, -0.5))
inv_sqrt = ktf.matmul(D, U, transpose_b=True)
D = ktf.diag(ktf.pow(S, 0.5))
sqrt = ktf.matmul(D, U, transpose_b=True)
return sqrt, inv_sqrt
else:
assert False
def train():
ff_apr = ktf.matmul(f, f, transpose_b=True) / (
ktf.cast(bs * w * h, ktf.float32) - 1.)
if self.decomposition in ['pca-cor', 'zca-cor']:
dinv = ktf.diag(ktf.sqrt(ktf.diag_part(ff_apr)))
ff_apr = ktf.matmul(ktf.matmul(dinv, ff_apr),
ktf.matrix_inverse(dinv),
transpose_b=True)
self.add_update([
K.moving_average_update(self.moving_mean, m, self.momentum),
K.moving_average_update(self.moving_cov, ff_apr, self.momentum)
], inputs)
ff_apr_shrinked = (
1 - self.epsilon) * ff_apr + ktf.eye(c) * self.epsilon
if self.renorm:
l, l_inv = get_inv_sqrt(ff_apr_shrinked)
ff_mov = (1 - self.epsilon
) * self.moving_cov + ktf.eye(c) * self.epsilon
_, l_mov_inverse = get_inv_sqrt(ff_mov)
l_ndiff = K.stop_gradient(l)
return ktf.matmul(ktf.matmul(l_mov_inverse, l_ndiff), l_inv)
return get_inv_sqrt(ff_apr_shrinked)[1]
def test():
ff_mov = (
1 - self.epsilon) * self.moving_cov + ktf.eye(c) * self.epsilon
return get_inv_sqrt(ff_mov)[1]
inv_sqrt = K.in_train_phase(train, test)
f_hat = ktf.matmul(inv_sqrt, f)
decorelated = K.reshape(f_hat, [c, bs, w, h])
decorelated = ktf.transpose(decorelated, [1, 2, 3, 0])
broadcast_shape = [1] * len(input_shape)
if self.axis is not None:
broadcast_shape[self.axis] = input_shape[self.axis]
if self.scale:
broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
decorelated = decorelated * broadcast_gamma
if self.center:
broadcast_beta = K.reshape(self.beta, broadcast_shape)
decorelated = decorelated + broadcast_beta
return decorelated
