def call(self, inputs, **kwargs):
boxes = K.stop_gradient(inputs[0])
fpn = K.stop_gradient(inputs[1])
time_distributed = K.ndim(boxes) == 4
if time_distributed:
boxes_shape = K.shape(boxes)
fpn_shape = K.shape(fpn)
new_boxes_shape = [-1] + [
boxes_shape[i] for i in range(2, K.ndim(boxes))
]
new_fpn_shape = [-1
] + [fpn_shape[i] for i in range(2, K.ndim(fpn))]
boxes = K.reshape(boxes, new_boxes_shape)
fpn = K.reshape(fpn, new_fpn_shape)
image_shape = K.cast(K.shape(fpn), K.floatx())
def _roi_align(args):
boxes = args[0]
fpn = args[1] # process the feature map
x1 = boxes[:, 0]
y1 = boxes[:, 1]
x2 = boxes[:, 2]
y2 = boxes[:, 3]
fpn_shape = K.cast(K.shape(fpn), dtype=K.floatx())
norm_boxes = K.stack([
(y1 / image_shape[1] * fpn_shape[0]) / (fpn_shape[0] - 1),
(x1 / image_shape[2] * fpn_shape[1]) / (fpn_shape[1] - 1),
(y2 / image_shape[1] * fpn_shape[0] - 1) / (fpn_shape[0] - 1),
(x2 / image_shape[2] * fpn_shape[1] - 1) / (fpn_shape[1] - 1)
],
axis=1)
rois = tf.image.crop_and_resize(
K.expand_dims(fpn, axis=0), norm_boxes,
tf.zeros((K.shape(norm_boxes)[0], ), dtype='int32'),
self.crop_size)
return rois
roi_batch = tf.map_fn(_roi_align,
elems=[boxes, fpn],
dtype=K.floatx(),
parallel_iterations=self.parallel_iterations)
if time_distributed:
roi_shape = tf.shape(roi_batch)
new_roi_shape = [boxes_shape[0], boxes_shape[1]] + \
[roi_shape[i] for i in range(1, K.ndim(roi_batch))]
roi_batch = tf.reshape(roi_batch, new_roi_shape)
return roi_batch
def call(self, inputs):
_, kernel_b = xnorize(self.kernel, self.H)
_, inputs_b = xnorize(inputs)
outputs = K.conv2d(inputs_b,
kernel_b,
strides=self.strides,
padding=self.padding,
data_format=self.data_format,
dilation_rate=self.dilation_rate)
# calculate Wa and xa
# kernel_a
mask = K.reshape(
self.kernel,
(-1,
self.filters)) # self.nb_row * self.nb_col * channels, filters
kernel_a = K.stop_gradient(K.mean(K.abs(mask), axis=0)) # filters
# inputs_a
if self.data_format == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
mask = K.mean(K.abs(inputs), axis=channel_axis, keepdims=True)
ones = K.ones(self.kernel_size + (1, 1))
inputs_a = K.conv2d(mask,
ones,
strides=self.strides,
padding=self.padding,
data_format=self.data_format,
dilation_rate=self.dilation_rate
) # nb_sample, 1, new_nb_row, new_nb_col
if self.data_format == 'channels_first':
outputs = outputs * K.stop_gradient(inputs_a) * K.expand_dims(
K.expand_dims(K.expand_dims(kernel_a, 0), -1), -1)
else:
outputs = outputs * K.stop_gradient(inputs_a) * K.expand_dims(
K.expand_dims(K.expand_dims(kernel_a, 0), 0), 0)
if self.use_bias:
outputs = K.bias_add(outputs,
self.bias,
data_format=self.data_format)
if self.activation is not None:
return self.activation(outputs)
return outputs
def train():
ff_aprs = utils.get_group_cov(f, self.group, self.m_per_group,
self.instance_norm, bs, w, h, c)
if self.instance_norm:
ff_aprs = tf.transpose(ff_aprs, (1, 0, 2, 3))
ff_aprs = (1 - self.epsilon) * ff_aprs + tf.expand_dims(
tf.expand_dims(tf.eye(self.m_per_group) * self.epsilon, 0),
0)
else:
ff_aprs = (1 - self.epsilon) * ff_aprs + tf.expand_dims(
tf.eye(self.m_per_group) * self.epsilon, 0)
whitten_matrix = get_inv_sqrt(ff_aprs, self.m_per_group)[1]
self.add_update([
K.moving_average_update(self.moving_mean, m, self.momentum),
K.moving_average_update(
self.moving_matrix,
whitten_matrix if '_wm' in self.decomposition else ff_aprs,
self.momentum)
], inputs)
if self.renorm:
l, l_inv = get_inv_sqrt(ff_aprs, self.m_per_group)
ff_mov = (1 - self.epsilon) * self.moving_matrix + tf.eye(
self.m_per_group) * self.epsilon
_, l_mov_inverse = get_inv_sqrt(ff_mov, self.m_per_group)
l_ndiff = K.stop_gradient(l)
return tf.matmul(tf.matmul(l_mov_inverse, l_ndiff), l_inv)
return whitten_matrix
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr * (1. / (1. + self.decay *
K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx()) + 1
# Applies bounds on actual learning rate
step_size = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) /
(1. - K.pow(self.beta_1, t)))
final_lr = self.final_lr * lr / self.base_lr
lower_bound = final_lr * (1. - 1. / (self.gamma * t + 1.))
upper_bound = final_lr * (1. + 1. / (self.gamma * t))
ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
if self.amsbound:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
else:
vhats = [K.zeros(1) for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
# apply weight decay
if self.weight_decay != 0.:
g += self.weight_decay * K.stop_gradient(p)
m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
if self.amsbound:
vhat_t = K.maximum(vhat, v_t)
denom = (K.sqrt(vhat_t) + self.epsilon)
self.updates.append(K.update(vhat, vhat_t))
else:
denom = (K.sqrt(v_t) + self.epsilon)
# Compute the bounds
step_size_p = step_size * K.ones_like(denom)
step_size_p_bound = step_size_p / denom
bounded_lr_t = m_t * K.minimum(
K.maximum(step_size_p_bound, lower_bound), upper_bound)
p_t = p - bounded_lr_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
def call(self, inputs, training=None, mask=None):
q = inputs[0]
k = inputs[1]
v = inputs[2]
qkTensor = K.math_ops.matmul(q,k,transpose_b=True)
scaleTensor = K.math_ops.multiply(K.stop_gradient(1. / K.math_ops.sqrt(self.dk)) , qkTensor)
softMaxTensor = K.softmax(scaleTensor)
drT = self.dropout(softMaxTensor,training = training)
vTensor = K.math_ops.matmul(drT,v)
return vTensor
def style_loss(style, combination, mask_path=None, nb_channels=None):
assert K.ndim(style) == 3
assert K.ndim(combination) == 3
if content_mask_path is not None:
content_mask = K.variable(load_mask(content_mask_path, nb_channels))
combination = combination * K.stop_gradient(content_mask)
del content_mask
if mask_path is not None:
style_mask = K.variable(load_mask(mask_path, nb_channels))
style = style * K.stop_gradient(style_mask)
if content_mask_path is None:
combination = combination * K.stop_gradient(style_mask)
del style_mask
S = gram_matrix(style)
C = gram_matrix(combination)
channels = 3
size = img_width * img_height
return K.sum(K.square(S - C)) / (4. * (channels ** 2) * (size ** 2))
def call(self, inputs, training=None, mask=None):
sentence = inputs[0]
position = inputs[1]
batchTensor = tf.stop_gradient(tf.nn.embedding_lookup(params=self.iniEmbeddingMatrix, ids=sentence))
batchTensor = K.stop_gradient(K.math_ops.multiply(batchTensor,K.math_ops.sqrt(self.dModel)))
positionTensor = self.positionEmbeddingMatrix(position)
positionTensor = K.math_ops.multiply(positionTensor,K.stop_gradient(K.math_ops.sqrt(self.dModel)))
# print(batchTensor.shape)
# print(positionTensor.shape)
eDropTensor = self.embeddingDrop(K.math_ops.multiply(K.math_ops.add(batchTensor,positionTensor),
K.stop_gradient(tf.convert_to_tensor(1. / 2.,dtype=tf.float32))),
training = training)
denseTrans = self.denseTrans(eDropTensor)
thisTransformer = K.identity(denseTrans)
for i in range(self._transformerLayers):
thisTransformer = self.transformerList[i](thisTransformer,training=training)
flattenTensor = self.flat(thisTransformer)
dense1Tensor = self.dense1(flattenTensor)
return tf.nn.softmax(dense1Tensor,axis=-1)
def max_singular_val(w, u, fully_differentiable=False, ip=1):
if not fully_differentiable:
w_ = K.stop_gradient(w)
else:
w_ = w
u = K.expand_dims(u, axis=-1)
u_bar = u
for _ in range(ip):
v_bar = tf.matmul(w_, u_bar, transpose_a=True)
v_bar = K.l2_normalize(v_bar, axis=(-1, -2))
u_bar_raw = tf.matmul(w_, v_bar)
u_bar = K.l2_normalize(u_bar_raw, axis=(-1, -2))
sigma = tf.matmul(u_bar, tf.matmul(w, v_bar), transpose_a=True)
sigma = K.squeeze(sigma, axis=-1)
sigma = K.squeeze(sigma, axis=-1)
u_bar = K.squeeze(u_bar, axis=-1)
return sigma, u_bar
def max_singular_val_for_convolution(w,
u,
fully_differentiable=False,
ip=1,
padding='same',
strides=(1, 1),
data_format='channels_last'):
assert ip >= 1
if not fully_differentiable:
w_ = K.stop_gradient(w)
else:
w_ = w
u_bar = u
for _ in range(ip):
v_bar = K.conv2d(u_bar,
w_,
strides=strides,
data_format=data_format,
padding=padding)
v_bar = K.l2_normalize(v_bar)
u_bar_raw = K.conv2d_transpose(v_bar,
w_,
output_shape=K.int_shape(u),
strides=strides,
data_format=data_format,
padding=padding)
u_bar = K.l2_normalize(u_bar_raw)
u_bar_raw_diff = K.conv2d_transpose(v_bar,
w,
output_shape=K.int_shape(u),
strides=strides,
data_format=data_format,
padding=padding)
sigma = K.sum(u_bar * u_bar_raw_diff)
return sigma, u_bar
def call(self, inputs, **kwargs):
image_shape = K.cast(inputs[0], K.floatx())
boxes = K.stop_gradient(inputs[1])
scores = K.stop_gradient(inputs[2])
fpn = [K.stop_gradient(i) for i in inputs[3:]]
time_distributed = K.ndim(boxes) == 4
if time_distributed:
image_shape = image_shape[1:]
boxes_shape = tf.shape(boxes)
scores_shape = tf.shape(scores)
fpn_shape = [tf.shape(f) for f in fpn]
new_boxes_shape = [-1] + [
boxes_shape[i] for i in range(2, K.ndim(boxes))
]
new_scores_shape = [-1] + [
scores_shape[i] for i in range(2, K.ndim(scores))
]
new_fpn_shape = [[-1] + [f_s[i] for i in range(2, K.ndim(f))]
for f, f_s in zip(fpn, fpn_shape)]
boxes = tf.reshape(boxes, new_boxes_shape)
scores = tf.reshape(scores, new_scores_shape)
fpn = [tf.reshape(f, f_s) for f, f_s in zip(fpn, new_fpn_shape)]
def _roi_align(args):
boxes = args[0]
scores = args[1]
fpn = args[2]
# compute from which level to get features from
target_levels = self.map_to_level(boxes)
# process each pyramid independently
rois, ordered_indices = [], []
for i in range(len(fpn)):
# select the boxes and classification from this pyramid level
indices = tf.where(K.equal(target_levels, i))
ordered_indices.append(indices)
level_boxes = tf.gather_nd(boxes, indices)
fpn_shape = K.cast(K.shape(fpn[i]), dtype=K.floatx())
# convert to expected format for crop_and_resize
x1 = level_boxes[:, 0]
y1 = level_boxes[:, 1]
x2 = level_boxes[:, 2]
y2 = level_boxes[:, 3]
level_boxes = K.stack([
(y1 / image_shape[1] * fpn_shape[0]) / (fpn_shape[0] - 1),
(x1 / image_shape[2] * fpn_shape[1]) / (fpn_shape[1] - 1),
(y2 / image_shape[1] * fpn_shape[0] - 1) /
(fpn_shape[0] - 1),
(x2 / image_shape[2] * fpn_shape[1] - 1) /
(fpn_shape[1] - 1),
],
axis=1)
# append the rois to the list of rois
rois.append(
tf.image.crop_and_resize(
K.expand_dims(fpn[i], axis=0), level_boxes,
tf.zeros((K.shape(level_boxes)[0], ), dtype='int32'),
self.crop_size))
# concatenate rois to one blob
rois = K.concatenate(rois, axis=0)
# reorder rois back to original order
indices = K.concatenate(ordered_indices, axis=0)
rois = tf.scatter_nd(indices, rois, K.cast(K.shape(rois), 'int64'))
return rois
roi_batch = tf.map_fn(_roi_align,
elems=[boxes, scores, fpn],
dtype=K.floatx(),
parallel_iterations=self.parallel_iterations)
if time_distributed:
roi_shape = tf.shape(roi_batch)
new_roi_shape = [boxes_shape[0], boxes_shape[1]] + \
[roi_shape[i] for i in range(1, K.ndim(roi_batch))]
roi_batch = tf.reshape(roi_batch, new_roi_shape)
return roi_batch
def call(self, inputs, **kwargs):
image_shape = K.cast(inputs[0], K.floatx())
boxes = K.stop_gradient(inputs[1])
scores = K.stop_gradient(inputs[2])
fpn = [K.stop_gradient(i) for i in inputs[3:]]
def _roi_align(args):
boxes = args[0]
scores = args[1]
fpn = args[2]
# compute from which level to get features from
target_levels = self.map_to_level(boxes)
# process each pyramid independently
rois, ordered_indices = [], []
for i in range(len(fpn)):
# select the boxes and classification from this pyramid level
indices = tf.where(K.equal(target_levels, i))
ordered_indices.append(indices)
level_boxes = tf.gather_nd(boxes, indices)
fpn_shape = K.cast(K.shape(fpn[i]), dtype=K.floatx())
# convert to expected format for crop_and_resize
x1 = level_boxes[:, 0]
y1 = level_boxes[:, 1]
x2 = level_boxes[:, 2]
y2 = level_boxes[:, 3]
level_boxes = K.stack([
(y1 / image_shape[1] * fpn_shape[0]) / (fpn_shape[0] - 1),
(x1 / image_shape[2] * fpn_shape[1]) / (fpn_shape[1] - 1),
(y2 / image_shape[1] * fpn_shape[0] - 1) / (fpn_shape[0] - 1),
(x2 / image_shape[2] * fpn_shape[1] - 1) / (fpn_shape[1] - 1),
], axis=1)
if(len(fpn[i].get_shape()) >=4):
unstack = tf.unstack(fpn[i], axis=3)
temp_stack=[]
for j in unstack:
temp = tf.image.crop_and_resize(
K.expand_dims(j, axis=3),
level_boxes,
tf.zeros((K.shape(level_boxes)[0],), dtype='int32'),
(self.crop_size[0], self.crop_size[1]))
temp_stack.append(temp)
rois.append(temp_stack)
else:
rois.append(tf.image.crop_and_resize(
K.expand_dims(fpn[i], axis=0),
level_boxes,
tf.zeros((K.shape(level_boxes)[0],), dtype='int32'),
self.crop_size
))
# concatenate rois to one blob
rois = K.concatenate(rois, axis=0)
# reorder rois back to original order
indices = K.concatenate(ordered_indices, axis=0)
rois = tf.scatter_nd(indices, rois, K.cast(K.shape(rois), 'int64'))
return rois
roi_batch = tf.map_fn(
_roi_align,
elems=[boxes, scores, fpn],
dtype=K.floatx(),
parallel_iterations=self.parallel_iterations
)
return roi_batch
def _mean_abs(x, axis=None, keepdims=False):
return K.stop_gradient(K.mean(K.abs(x), axis=axis, keepdims=keepdims))
def round_through(x):
'''Element-wise rounding to the closest integer with full gradient propagation.
A trick from [Sergey Ioffe](http://stackoverflow.com/a/36480182)
'''
rounded = K.round(x)
return x + K.stop_gradient(rounded - x)
