def __call__(self, x):
xshape = K.int_shape(x)
if self.axis is 'last':
x = K.reshape(x, (-1, xshape[-1]))
x /= K.sqrt(K.sum(K.square(x), axis=0, keepdims=True))
xx = K.dot(K.transpose(x), x)
return self.gamma * K.sum(K.log(1.0 + K.exp(self.lam * (xx - 1.0))) * (1.0 - K.eye(xshape[-1])))
elif self.axis is 'first':
x = K.reshape(x, (xshape[0], -1))
x /= K.sqrt(K.sum(K.square(x), axis=1, keepdims=True))
xx = K.dot(x, K.transpose(x))
return self.gamma * K.sum(K.log(1.0 + K.exp(self.lam * (xx - 1.0))) * (1.0 - K.eye(xshape[0])))
def __loss(y_true, y_pred):
kernel_cs_forward, kernel_cs_backward = [], []
for (forward, backward) in layers:
kernel_c_forward = forward.cell.trainable_weights[1][:, rnn_units * 2:rnn_units * 3]
kernel_c_backward = backward.cell.trainable_weights[1][:, rnn_units * 2:rnn_units * 3]
kernel_cs_forward.append(K.reshape(kernel_c_forward, (rnn_units * rnn_units,)))
kernel_cs_backward.append(K.reshape(kernel_c_backward, (rnn_units * rnn_units,)))
phi_forward = K.stack(kernel_cs_forward)
phi_backward = K.stack(kernel_cs_backward)
loss_sim_forward = K.sum(K.square(K.dot(phi_forward, K.transpose(phi_forward)) - K.eye(len(layers))))
loss_sim_backward = K.sum(K.square(K.dot(phi_backward, K.transpose(phi_backward)) - K.eye(len(layers))))
loss_cat = keras.losses.categorical_crossentropy(y_true, y_pred)
return loss_cat + lmbd * (loss_sim_forward + loss_sim_backward)
def _pairwise_distances(self, inputs: List[Tensor]) -> Tensor:
emb_c, emb_r = inputs
bs = K.shape(emb_c)[0]
embeddings = K.concatenate([emb_c, emb_r], 0)
dot_product = K.dot(embeddings, K.transpose(embeddings))
square_norm = K.batch_dot(embeddings, embeddings, axes=1)
distances = K.transpose(square_norm) - 2.0 * dot_product + square_norm
distances = K.slice(distances, (0, bs), (bs, bs))
distances = K.clip(distances, 0.0, None)
mask = K.cast(K.equal(distances, 0.0), K.dtype(distances))
distances = distances + mask * 1e-16
distances = K.sqrt(distances)
distances = distances * (1.0 - mask)
return distances
def get_feature_represent(x, layer_names, model):
'''图片的特征图表示
参数
----------------------------------------------
x : 输入，
这里并没有使用，可以看作一个输入的标识
layer_names : list
CNN网络层的名字
model : CNN模型
返回值
----------------------------------------------
feature_matrices : list
经过CNN卷积层的特征表示，这里大小是(filter个数, feature map的长*宽)
'''
feature_matrices = []
for ln in layer_names:
select_layer = model.get_layer(ln)
feature_raw = select_layer.output
feature_raw_shape = K.shape(feature_raw).eval(session=tf_session)
N_l = feature_raw_shape[-1]
M_l = feature_raw_shape[1]*feature_raw_shape[2]
feature_matrix = K.reshape(feature_raw, (M_l, N_l))
feature_matrix = K.transpose(feature_matrix)
feature_matrices.append(feature_matrix)
return feature_matrices
def A_network_output(x):
# The input of this layer is [L, mu, a] in concatenated form. We first split
# those up.
idx = 0
L_flat = x[:, idx:idx + (self.nb_actions * self.nb_actions + self.nb_actions) / 2]
idx += (self.nb_actions * self.nb_actions + self.nb_actions) / 2
mu = x[:, idx:idx + self.nb_actions]
idx += self.nb_actions
a = x[:, idx:idx + self.nb_actions]
idx += self.nb_actions
# Create L and L^T matrix, which we use to construct the positive-definite matrix P.
Ls = []
LTs = []
for idx in xrange(self.batch_size):
L = K.zeros((self.nb_actions, self.nb_actions))
L = T.set_subtensor(L[np.tril_indices(self.nb_actions)], L_flat[idx, :])
diag = K.exp(T.diag(L))
L = T.set_subtensor(L[np.diag_indices(self.nb_actions)], diag)
Ls.append(L)
LTs.append(K.transpose(L))
# TODO: diagonal elements exp
L = K.pack(Ls)
LT = K.pack(LTs)
P = K.batch_dot(L, LT, axes=(1, 2))
assert K.ndim(P) == 3
# Combine a, mu and P into a scalar (over the batches).
A = -.5 * K.batch_dot(K.batch_dot(a - mu, P, axes=(1, 2)), a - mu, axes=1)
assert K.ndim(A) == 2
return A
def gram_matrix(x):
assert Kr.ndim(x) == 3
features = Kr.batch_flatten(x)
gram = Kr.dot(features, Kr.transpose(features))
return gram
def gram_matrix(x):
#change height,width,depth to depth, height, width, it could be 2,1,0 too
#maybe 2,0,1 is more efficient due to underlying memory layout
features = K.permute_dimensions(x, (2,0,1))
#batch flatten make features become 2D array
features = K.batch_flatten(features)
return K.dot(features, K.transpose(features)) / x.get_shape().num_elements()
def gram_matrix(x):
if K.image_dim_ordering() == "th":
features = K.batch_flatten(x)
else:
features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
gram = K.dot(features, K.transpose(features))
return gram
def call(self, y, mask=None):
'''
parameter y is supposed to have twice the length of the STF
'''
# Compute mask
y1, y2 = T.split(K.transpose(y), [self.output_dim, self.output_dim], 2, axis=0)
mask1 = K.abs(y1) / (K.abs(y1) + K.abs(y2) + 1)
mask2 = K.abs(y2) / (K.abs(y1) + K.abs(y2) + 1)
mask = K.concatenate([mask1, mask2])
# Apply mask
sft = shared(self.sfts[Mask_Data_Callback.idx])
X1 = sft * K.transpose(mask1)
X2 = sft * K.transpose(mask2)
out = K.concatenate([X1, X2], axis=1)
return out
def gram_matrix(x):
assert K.ndim(x) == 3
if K.image_data_format() == 'channels_first':
features = K.batch_flatten(x)
else:
features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
gram = K.dot(features, K.transpose(features))
return gram
def R(vects):
"""
Calculates the cosine similarity of two vectors.
:param vects: a list of two vectors.
:return: the cosine similarity of two vectors.
"""
(x, y) = vects
return backend.dot(x, backend.transpose(y)) / (x.norm(2) * y.norm(2)) # See equation (4)
def attention_control(args):
x, dense_2 = args
find_att = K.reshape(x, (15, 15, 10))
find_att = K.transpose(find_att[:, :, :])
find_att = K.mean(find_att, axis=0)
find_att = find_att / K.sum(find_att, axis=0)
find_att = K.repeat_elements(find_att, 32, axis=0)
find_att = K.reshape(find_att, (1, 32, 15, 15))
return find_att
def gram_matrix(img): # input is (H, W, C) (C = # feature maps) # we first need to convert it to (C, H*W) X = K.batch_flatten(K.permute_dimensions(img, (2, 0, 1))) # now, calculate the gram matrix # gram = XX^T / N # the constant is not important since we'll be weighting these G = K.dot(X, K.transpose(X)) / img.get_shape().num_elements() return G
def gram_matrix(x):
"""
the gram matrix of an image tensor (feature-wise outer product)
:param x: The tensor contains image features
:return: A gram matrix
"""
if K.ndim(x) == 4:
x = x[0, :, :, :]
assert K.ndim(x) == 3
features = K.batch_flatten(x)
gram = K.dot(features, K.transpose(features))
return gram
def get_pmi(self):
print 'feature_sums:', self.feature_sums.get_shape()
print 'feature_covariance_sums:', self.feature_covariance_sums.get_shape()
feature_sums = K.reshape(self.feature_sums, (self.input_space_dim, 1))
products = K.dot(feature_sums, K.transpose(feature_sums))
print 'products:', products.get_shape()
pmi = products * self.feature_covariance_sums
pmi = K.log(pmi + K.epsilon())
pmi *= self.feature_covariance_sums
return pmi
def KLdivergence(P, Y):
alpha = low_dim - 1.
sum_Y = K.sum(K.square(Y), axis=1)
eps = K.variable(10e-15)
D = sum_Y + K.reshape(sum_Y, [-1, 1]) - 2 * K.dot(Y, K.transpose(Y))
Q = K.pow(1 + D / alpha, -(alpha + 1) / 2)
Q *= K.variable(1 - np.eye(batch_size))
Q /= K.sum(Q)
Q = K.maximum(Q, eps)
C = K.log((P + eps) / (Q + eps))
C = K.sum(P * C)
return C
def call(self, x, mask=None):
batch_placeholder = K.cast(x, 'int32')[0]
s, o, p = [batch_placeholder[i] for i in range(3)]
s2v = K.gather(self.E, s)
o2v = K.gather(self.E, o)
r2v = K.gather(self.R, p)
# print(K.shape(s2v).eval())
# print(self.E[[0]].shape.eval())
def ccorr(a, b):
return self.ccorr1d_sc(a, b, border_mode='half')
eta = K.dot(K.transpose(r2v), ccorr(s2v, o2v))
return eta
def tsne(P, activations):
# d = K.shape(activations)[1]
d = 2 # TODO: should set this automatically, but the above is very slow for some reason
n = 1000 # TODO: should set this automatically
v = d - 1.
eps = K.variable(10e-15) # needs to be at least 10e-8 to get anything after Q /= K.sum(Q)
sum_act = K.sum(K.square(activations), axis=1)
Q = K.reshape(sum_act, [-1, 1]) + -2 * K.dot(activations, K.transpose(activations))
Q = (sum_act + Q) / v
Q = K.pow(1 + Q, -(v + 1) / 2)
Q *= K.variable(1 - np.eye(n))
Q /= K.sum(Q)
Q = K.maximum(Q, eps)
C = K.log((P + eps) / (Q + eps))
C = K.sum(P * C)
return C
def __call__(self, loss):
power = 9 # number of iterations of the power method
W = self.p
WW = K.dot(K.transpose(W), W)
dim1, dim2 = K.eval(K.shape(WW))
k = self.k
o = np.ones(dim1) # initial values for the dominant eigenvector
# power method for approximating the dominant eigenvector:
domin_eigenvect = K.dot(WW, o)
for n in range(power - 1):
domin_eigenvect = K.dot(WW, domin_eigenvect)
WWd = K.dot(WW, domin_eigenvect)
domin_eigenval = K.dot(WWd, domin_eigenvect) / K.dot(domin_eigenvect, domin_eigenvect) # the corresponding dominant eigenvalue
regularized_loss = loss + (domin_eigenval ** 0.5) * self.k # multiplied by the given regularization gain
return K.in_train_phase(regularized_loss, loss)
def gram_matrix(x): """ Computes the outer-product of the input tensor x. Input ----- - x: input tensor of shape (C x H x W) Returns ------- - x . x^T Note that this can be computed efficiently if x is reshaped as a tensor of shape (C x H*W). """ # assert K.ndim(x) == 3 if K.image_dim_ordering() == 'th': features = K.batch_flatten(x) else: features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1))) return K.dot(features, K.transpose(features))
def pmi_regularizer(self, loss):
# input_space_dim = stack_size * self.nb_row * self.nb_col
# kernel_shape = (self.nb_row, self.nb_col, stack_size, self.nb_filter)
# x_flat = K.reshape(x, (-1, input_space_dim))
# match: (self.nb_row, self.nb_col, stack_size) gets resized to input_space_dim
# in both cases (same order in kernel and x)
k_mat = K.reshape(self.kernel, (self.input_space_dim, self.nb_filter))
identity = K.variable(np.identity(self.input_space_dim))
print 'k_mat:', k_mat.get_shape()
print 'input_space_dim:', self.input_space_dim
print k_mat.get_shape()
embeddings = K.dot(identity, k_mat)
pmi_star = K.dot(embeddings, K.transpose(k_mat))
print 'pmi_star:', pmi_star.get_shape()
pmi = self.get_pmi()
# potential trick: multiply loss below to give more weight to lower layers
return loss + 0.00000001 * K.mean(K.abs(pmi - pmi_star))
def attention_control(args):
# Why do we need dense_2 here!?
x, dense_2 = args
find_att = K.reshape(x, (15, 15, 10))
find_att = K.transpose(find_att[:, :, :]) # 10 x 15 x 15
find_att = K.mean(find_att, axis=0) # 15 x 15
# WTF ???
find_att = find_att / K.sum(find_att, axis=0) # 15 x 15
# TODO ??? maybe BUG: copy across channels, but he lose channel axis
find_att = K.repeat_elements(find_att, 32, axis=0)
find_att = K.reshape(find_att, (1, 32, 15, 15))
# x, dense_2 = args
# find_att = K.reshape(x, (15, 15, 10))
# find_att = K.transpose(find_att[:, :, :]) # 10 x 15 x 15
# # find average attention here across all feature maps
# mean_att = K.mean(find_att, axis=0) # 15 x 15
# mean_att = K.reshape(mean_att, (1, 15, 15))
# # copy attention mask across all feature maps
# rep_mean_att = K.repeat_elements(mean_att, 32, axis=0)
# focus = K.reshape(rep_mean_att, (1, 32, 15, 15))
# return focus
return find_att
def orth_reg(W):
WT = K.transpose(W)
X = K.dot(WT, W)
X = X * (1. - ada_eye(X))
return 1e-3 * K.sum(X * X)
def __call__(self, p):
print "here"
u,s,v = T.nlinalg.svd(p)
return K.dot(u,K.transpose(v))
def gramian(filters):
c_filters = K.batch_flatten(
K.permute_dimensions(K.squeeze(filters, axis=0),
pattern=(2, 0, 1)))
return K.dot(c_filters, K.transpose(c_filters))
def grammian_matrix(matrix):
flattened_matrix = K.batch_flatten(K.permute_dimensions(matrix, (2, 0, 1)))
matrix_transpose_dot = K.dot(flattened_matrix, K.transpose(flattened_matrix))
element_count = matrix.get_shape().num_elements()
return matrix_transpose_dot / element_count
def __call__(self, p):
print("here")
u, s, v = T.nlinalg.svd(p)
return K.dot(u, K.transpose(v))
def yolo_head(feats, anchors, num_classes):
"""Convert final layer features to bounding box parameters.
Parameters
----------
feats : tensor
Final convolutional layer features.
anchors : array-like
Anchor box widths and heights.
num_classes : int
Number of target classes.
Returns
-------
box_xy : tensor
x, y box predictions adjusted by spatial location in conv layer.
box_wh : tensor
w, h box predictions adjusted by anchors and conv spatial resolution.
box_conf : tensor
Probability estimate for whether each box contains any object.
box_class_pred : tensor
Probability distribution estimate for each box over class labels.
"""
num_anchors = len(anchors)
# Reshape to batch, height, width, num_anchors, box_params.
anchors_tensor = K.reshape(K.variable(anchors), [1, 1, 1, num_anchors, 2])
# Static implementation for fixed models.
# TODO: Remove or add option for static implementation.
# _, conv_height, conv_width, _ = K.int_shape(feats)
# conv_dims = K.variable([conv_width, conv_height])
# Dynamic implementation of conv dims for fully convolutional model.
conv_dims = K.shape(feats)[1:3] # assuming channels last
# In YOLO the height index is the inner most iteration.
conv_height_index = K.arange(0, stop=conv_dims[0])
conv_width_index = K.arange(0, stop=conv_dims[1])
conv_height_index = K.tile(conv_height_index, [conv_dims[1]])
# TODO: Repeat_elements and tf.split doesn't support dynamic splits.
# conv_width_index = K.repeat_elements(conv_width_index, conv_dims[1], axis=0)
conv_width_index = K.tile(
K.expand_dims(conv_width_index, 0), [conv_dims[0], 1])
conv_width_index = K.flatten(K.transpose(conv_width_index))
conv_index = K.transpose(K.stack([conv_height_index, conv_width_index]))
conv_index = K.reshape(conv_index, [1, conv_dims[0], conv_dims[1], 1, 2])
conv_index = K.cast(conv_index, K.dtype(feats))
feats = K.reshape(
feats, [-1, conv_dims[0], conv_dims[1], num_anchors, num_classes + 5])
conv_dims = K.cast(K.reshape(conv_dims, [1, 1, 1, 1, 2]), K.dtype(feats))
# Static generation of conv_index:
# conv_index = np.array([_ for _ in np.ndindex(conv_width, conv_height)])
# conv_index = conv_index[:, [1, 0]] # swap columns for YOLO ordering.
# conv_index = K.variable(
# conv_index.reshape(1, conv_height, conv_width, 1, 2))
# feats = Reshape(
# (conv_dims[0], conv_dims[1], num_anchors, num_classes + 5))(feats)
box_xy = K.sigmoid(feats[..., :2])
box_wh = K.exp(feats[..., 2:4])
box_confidence = K.sigmoid(feats[..., 4:5])
box_class_probs = K.softmax(feats[..., 5:])
# Adjust preditions to each spatial grid point and anchor size.
# Note: YOLO iterates over height index before width index.
box_xy = (box_xy + conv_index) / conv_dims
box_wh = box_wh * anchors_tensor / conv_dims
return box_xy, box_wh, box_confidence, box_class_probs
def get_output(self, train=False):
X = self.get_input(train)
if self.pretrain or self.output_reconstruction:
output = self.reconstruction_activation(K.dot(self.activation(K.dot(X, self.W)), K.transpose(self.W)))
return output
else:
output = self.activation(K.dot(X, self.W))
return output
def power_iteration(W, u):
# Accroding the paper, we only need to do power iteration one time.
_u = u
_v = _l2normalize(K.dot(_u, K.transpose(W)))
_u = _l2normalize(K.dot(_v, W))
return _u, _v
def gram_matrix(x):
features = backend.batch_flatten(backend.permute_dimensions(x, (2, 0, 1)))
gram = backend.dot(features, backend.transpose(features))
return gram
def call(self, inputs):
input_shape = K.shape(inputs)
batch_size = input_shape[0]
if self.data_format == 'channels_first':
h_axis, w_axis = 2, 3
else:
h_axis, w_axis = 1, 2
height, width = input_shape[h_axis], input_shape[w_axis]
kernel_h, kernel_w = self.kernel_size
stride_h, stride_w = self.strides
if self.output_padding is None:
out_pad_h = out_pad_w = None
else:
out_pad_h, out_pad_w = self.output_padding
# Infer the dynamic output shape:
out_height = conv_utils.deconv_length(height, stride_h, kernel_h,
self.padding, out_pad_h)
out_width = conv_utils.deconv_length(width, stride_w, kernel_w,
self.padding, out_pad_w)
if self.data_format == 'channels_first':
output_shape = (batch_size, self.filters, out_height, out_width)
else:
output_shape = (batch_size, out_height, out_width, self.filters)
# Spectral Normalization
def _l2normalize(v, eps=1e-12):
return v / (K.sum(v**2)**0.5 + eps)
def power_iteration(W, u):
# Accroding the paper, we only need to do power iteration one time.
_u = u
_v = _l2normalize(K.dot(_u, K.transpose(W)))
_u = _l2normalize(K.dot(_v, W))
return _u, _v
W_shape = self.kernel.shape.as_list()
# Flatten the Tensor
W_reshaped = K.reshape(self.kernel, [-1, W_shape[-1]])
_u, _v = power_iteration(W_reshaped, self.u)
# Calculate Sigma
sigma = K.dot(_v, W_reshaped)
sigma = K.dot(sigma, K.transpose(_u))
# normalize it
W_bar = W_reshaped / sigma
# reshape weight tensor
if training in {0, False}:
W_bar = K.reshape(W_bar, W_shape)
else:
with tf.control_dependencies([self.u.assign(_u)]):
W_bar = K.reshape(W_bar, W_shape)
self.kernel = W_bar
outputs = K.conv2d_transpose(inputs,
self.kernel,
output_shape,
self.strides,
padding=self.padding,
data_format=self.data_format)
if self.use_bias:
outputs = K.bias_add(outputs,
self.bias,
data_format=self.data_format)
if self.activation is not None:
return self.activation(outputs)
return outputs
def trend_model(thetas, backcast_length, forecast_length, is_forecast):
p = thetas.shape[-1]
t = linear_space(backcast_length, forecast_length, fwd_looking=is_forecast)
T = K.transpose(K.stack([t**i for i in range(p)], axis=0))
T = K.cast(T, np.float32)
return K.dot(thetas, K.transpose(T))
def yolo_head(feats, anchors, num_classes):
"""Convert final layer features to bounding box parameters.
Parameters
----------
feats : tensor
Final convolutional layer features.
anchors : array-like
Anchor box widths and heights.
num_classes : int
Number of target classes.
Returns
-------
box_xy : tensor
x, y box predictions adjusted by spatial location in conv layer.
box_wh : tensor
w, h box predictions adjusted by anchors and conv spatial resolution.
box_conf : tensor
Probability estimate for whether each box contains any object.
box_class_pred : tensor
Probability distribution estimate for each box over class labels.
"""
num_anchors = len(anchors)
# Reshape to batch, height, width, num_anchors, box_params.
anchors_tensor = K.reshape(K.variable(anchors), [1, 1, 1, num_anchors, 2])
# Static implementation for fixed models.
# TODO: Remove or add option for static implementation.
# _, conv_height, conv_width, _ = K.int_shape(feats)
# conv_dims = K.variable([conv_width, conv_height])
# Dynamic implementation of conv dims for fully convolutional model.
conv_dims = K.shape(feats)[1:3] # assuming channels last
# In YOLO the height index is the inner most iteration.
conv_height_index = K.arange(0, stop=conv_dims[0])
conv_width_index = K.arange(0, stop=conv_dims[1])
conv_height_index = K.tile(conv_height_index, [conv_dims[1]])
# TODO: Repeat_elements and tf.split doesn't support dynamic splits.
# conv_width_index = K.repeat_elements(conv_width_index, conv_dims[1], axis=0)
conv_width_index = K.tile(K.expand_dims(conv_width_index, 0),
[conv_dims[0], 1])
conv_width_index = K.flatten(K.transpose(conv_width_index))
conv_index = K.transpose(K.stack([conv_height_index, conv_width_index]))
conv_index = K.reshape(conv_index, [1, conv_dims[0], conv_dims[1], 1, 2])
conv_index = K.cast(conv_index, K.dtype(feats))
feats = K.reshape(
feats, [-1, conv_dims[0], conv_dims[1], num_anchors, num_classes + 5])
conv_dims = K.cast(K.reshape(conv_dims, [1, 1, 1, 1, 2]), K.dtype(feats))
# Static generation of conv_index:
# conv_index = np.array([_ for _ in np.ndindex(conv_width, conv_height)])
# conv_index = conv_index[:, [1, 0]] # swap columns for YOLO ordering.
# conv_index = K.variable(
# conv_index.reshape(1, conv_height, conv_width, 1, 2))
# feats = Reshape(
# (conv_dims[0], conv_dims[1], num_anchors, num_classes + 5))(feats)
box_xy = K.sigmoid(feats[..., :2])
box_wh = K.exp(feats[..., 2:4])
box_confidence = K.sigmoid(feats[..., 4:5])
box_class_probs = K.softmax(feats[..., 5:])
# Adjust preditions to each spatial grid point and anchor size.
# Note: YOLO iterates over height index before width index.
box_xy = (box_xy + conv_index) / conv_dims
box_wh = box_wh * anchors_tensor / conv_dims
return box_xy, box_wh, box_confidence, box_class_probs
def transpose(X):
return K.transpose(x)
def gram_matrix(x):
features = backend.batch_flatten(backend.permute_dimensions(x, (2, 0, 1)))
gram = backend.dot(features, backend.transpose(features))
return gram
def gram_matrix(x):
assert K.ndim(x) == 3
features = K.batch_flatten(x)
gram = K.dot(features, K.transpose(features))
return gram
def call(self, x):
if self._axes is None:
return K.transpose(x)
else:
return K.permute_dimensions(x, self._axes)
def gram_matrix(x):
# Rearrange features in required order
features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
gram = K.dot(features, K.transpose(features))
return gram
def get_Gram_matrix(F):
G = K.dot(F, K.transpose(F))
return G
def call(self, inputs, mask=None, **kwargs):
# [N, max_len, emb_dim] [vacab_size, emb_dim]
inputs, embeddings = inputs
outputs = K.bias_add(K.dot(inputs, K.transpose(embeddings)), self.bias)
return keras.activations.softmax(outputs)
def get_dists_backend(X):
norms = K.expand_dims(K.sum(K.square(X), axis=1),1)
return norms + K.transpose(norms) - 2*K.dot(X,K.transpose(X))
def build_gram_matrix(x):
features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
gram_matrix = K.dot(features, K.transpose(features))
return gram_matrix
def task_finish(y_true, y_pred):
return K.mean(K.greater(K.dot(K.softmax(y_pred), K.transpose(y_true)),.4), axis=-1)
def cosine_distance(jd):
jd = K.l2_normalize(jd, axis=-1)
jt_six = K.l2_normalize(title_embedding, axis=-1)
return K.dot(jd, K.transpose(jt_six))
def gram_matrix(img):
X = k.batch_flatten(k.permute_dimensions(img,(2,0,1)))
G = k.dot(X,k.transpose(X))/img.get_shape().num_elements()
return G
def __call__(self, w):
m = K.dot(K.transpose(w), w) - K.eye(K.int_shape(w)[1])
return self.reg_weight * frobenius_norm(m)
def restricted_tangent_distance(signals, protos, subspaces, bounds,
squared=False,
epsilon=K.epsilon()):
# Note: subspaces is always assumed as transposed and must be orthogonal!
# shape(signals): batch x proto_number x channels x dim1 x dim2 x ... x dimN
# shape(protos): proto_number x dim1 x dim2 x ... x dimN
# shape(subspaces): (optional [proto_number]) x prod(dim1 * dim2 * ... * dimN) x prod(projected_atom_shape)
signal_shape = mixed_shape(signals)
shape = tuple([i if isinstance(i, int) else None for i in signal_shape])
if not equal_shape((shape[1],) + shape[3:], K.int_shape(protos)):
raise ValueError("The shape of signals[2:] must be equal protos. You provide: signals.shape[2:]="
+ str((shape[1],) + shape[3:]) + " != protos.shape=" + str(K.int_shape(protos)))
with K.name_scope('tangent_distance'):
signal_shape = mixed_shape(signals)
atom_axes = list(range(3, len(signal_shape)))
signals = K.permute_dimensions(signals, [0, 2, 1] + atom_axes)
diff = signals - protos
if K.ndim(subspaces) == 2:
with K.name_scope('tangentspace_parameters'):
diff = K.reshape(diff, (signal_shape[0] * signal_shape[2], signal_shape[1], -1))
params = K.dot(diff, subspaces)
with K.name_scope('restricted_parameters'):
in_space = K.cast(K.less_equal(K.abs(params), bounds), dtype=params.dtype)
params = in_space * params + K.sign(params) * (1 - in_space) * bounds
with K.name_scope('tangentspace_projections'):
projected_diff = K.dot(params, K.transpose(subspaces))
projected_diff = K.reshape(projected_diff,
(signal_shape[0], signal_shape[2], signal_shape[1]) + signal_shape[3:])
matching_protos = protos + projected_diff
diss = p_norm(signals - matching_protos,
order_p=2, axis=atom_axes, squared=squared, keepdims=False, epsilon=epsilon)
return K.permute_dimensions(diss, [0, 2, 1])
else: # local or capsule_wise
with K.name_scope('tangentspace_parameters'):
diff = K.reshape(diff, (signal_shape[0] * signal_shape[2], signal_shape[1], -1))
diff = K.permute_dimensions(diff, [1, 0, 2])
params = K.batch_dot(diff, subspaces)
with K.name_scope('restricted_parameters'):
bounds = K.expand_dims(bounds, 1)
in_space = K.cast(K.less_equal(K.abs(params), bounds), dtype=params.dtype)
params = in_space * params + K.sign(params) * (1 - in_space) * bounds
with K.name_scope('tangentspace_projections'):
projected_diff = K.batch_dot(params, subspaces, [2, 2])
projected_diff = K.permute_dimensions(projected_diff, [1, 0, 2])
projected_diff = K.reshape(projected_diff,
(signal_shape[0], signal_shape[2], signal_shape[1]) + signal_shape[3:])
matching_protos = protos + projected_diff
diss = p_norm(signals - matching_protos,
order_p=2, axis=atom_axes, squared=squared, keepdims=False, epsilon=epsilon)
return K.permute_dimensions(diss, [0, 2, 1])
def call(self, inputs, mask=None):
# Retrieve the layer input
input0, arc_tensor_in, arc_tensor_out, label_tensor_in, label_tensor_out, mask_in, mask_out, mask_loop = inputs
batch_size = tf.shape(input0)[0]
seq_len = int(input0.shape[1])
#print('batch_size',batch_size,'seq_len',seq_len)
max_degree = 1
label_tensor_in = tf.reshape(label_tensor_in, [-1, 1])
label_tensor_out = tf.reshape(label_tensor_out, [-1, 1])
mask_in = tf.reshape(mask_in, [-1, 1])
mask_out = tf.reshape(mask_out, [-1, 1])
mask_loop = tf.reshape(mask_loop, [-1, 1])
input_ = tf.reshape(
input0, [batch_size * seq_len, self.num_inputs]) # [b* t, h]
if self.in_arcs:
input_in = K.dot(input_, self.V_in) # [b* t, h] * [h,h] = [b*t, h]
second_in = K.gather(self.b_in, K.transpose(label_tensor_in)[0])
in_ = tf.reshape((input_in + second_in),
[batch_size, seq_len, 1, self.num_units])
# compute gate weights
input_in_gate = K.dot(
input_, self.V_in_gate) # [b* t, h] * [h,h] = [b*t, h]
second_in_gate = K.gather(self.b_in_gate,
K.transpose(label_tensor_in)[0])
in_gate = tf.reshape((input_in_gate + second_in_gate),
[batch_size, seq_len, 1])
max_degree += 1
if self.out_arcs:
input_out = K.dot(input_,
self.V_out) # [b* t, h] * [h,h] = [b* t, h]
second_out = K.gather(self.b_out, K.transpose(label_tensor_out)[0])
degr = K.cast(tf.shape(input_out)[0] / batch_size / seq_len,
dtype='int32')
max_degree += degr
out_ = tf.reshape((input_out + second_out),
[batch_size, seq_len, degr, self.num_units])
# compute gate weights
input_out_gate = K.dot(
input_, self.V_out_gate) # [b* t, h] * [h,h] = [b* t, h]
second_out_gate = K.gather(self.b_out_gate,
K.transpose(label_tensor_out)[0])
out_gate = tf.reshape((input_out_gate + second_out_gate),
[batch_size, seq_len, degr])
same_input = K.dot(input0, self.W_self_loop)
same_input = tf.reshape(same_input, (batch_size, seq_len, -1))
same_input = tf.reshape(
same_input,
(batch_size, seq_len, 1, tf.shape(self.W_self_loop)[1]))
same_input_gate = K.dot(input0, self.W_self_loop_gate)
same_input_gate = tf.reshape(same_input_gate, (batch_size, seq_len))
same_input_gate = tf.reshape(same_input_gate,
(batch_size, seq_len, -1))
if self.in_arcs and self.out_arcs:
potentials = K.concatenate([in_, out_, same_input],
axis=2) # [b, t, mxdeg, h]
potentials_gate = K.concatenate(
[in_gate, out_gate, same_input_gate],
axis=2) # [b, t, mxdeg, h]
mask_soft = K.concatenate([mask_in, mask_out, mask_loop],
axis=1) # [b* t, mxdeg]
elif self.out_arcs:
potentials = K.concatenate([out_, same_input],
axis=2) # [b, t, 2*mxdeg+1, h]
potentials_gate = K.concatenate([out_gate, same_input_gate],
axis=2) # [b, t, mxdeg, h]
mask_soft = K.concatenate([mask_out, mask_loop],
axis=1) # [b* t, mxdeg]
elif self.in_arcs:
potentials = K.concatenate([in_, same_input],
axis=2) # [b, t, 2*mxdeg+1, h]
potentials_gate = K.concatenate([in_gate, same_input_gate],
axis=2) # [b, t, mxdeg, h]
mask_soft = K.concatenate([mask_in, mask_loop],
axis=1) # [b* t, mxdeg]
potentials_ = K.permute_dimensions(potentials,
(3, 0, 1, 2)) # [h, b, t, mxdeg]
potentials_resh = K.reshape(potentials_,
(self.num_units, batch_size * seq_len,
max_degree)) # [h, b * t, mxdeg]
potentials_r = K.reshape(
potentials_gate,
(batch_size * seq_len, max_degree)) # [h, b * t, mxdeg]
# calculate the gate
mask_soft = K.cast(mask_soft, dtype='float32')
probs_det_ = K.sigmoid(potentials_r) * mask_soft # [b * t, mxdeg]
potentials_masked = potentials_resh * mask_soft * probs_det_ # [h, b * t, mxdeg]
potentials_masked_ = K.sum(potentials_masked, axis=2) # [h, b * t]
#potentials_masked__ = T.switch(potentials_masked_ > 0, potentials_masked_, 0) # ReLU
potentials_masked_ = K.relu(potentials_masked_)
#potentials_masked__=K.relu(potentials_masked_)
result_ = K.permute_dimensions(potentials_masked_,
(1, 0)) # [b * t, h]
result_ = K.reshape(
result_, (batch_size, seq_len, self.num_units)) # [ b, t, h]
#result = result_ * mask.dimshuffle(0, 1, 'x') # [b, t, h]
return result_
def call(self, x):
C = K.expand_dims(self.centers)
H = K.transpose(C-K.transpose(x))
return K.exp(-self.betas * K.sum(H**2, axis=1))
def power_iteration(W, u):
_u = u
_v = _l2normalize(K.dot(_u, K.transpose(W)))
_u = _l2normalize(K.dot(_v, W))
return _u, _v
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
y_pred_f = K.flatten(y_pred)
return (2. * K.dot(y_true_f, K.transpose(y_pred_f)) +
smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)
def call(self, inputs):
def _l2normalize(v, eps=1e-12):
return v / (K.sum(v ** 2) ** 0.5 + eps)
def power_iteration(W, u):
_u = u
_v = _l2normalize(K.dot(_u, K.transpose(W)))
_u = _l2normalize(K.dot(_v, W))
return _u, _v
if self.spectral_normalization:
W_shape = self.kernel.shape.as_list()
#Flatten the Tensor
W_reshaped = K.reshape(self.kernel, [-1, W_shape[-1]])
_u, _v = power_iteration(W_reshaped, self.u)
#Calculate Sigma
sigma=K.dot(_v, W_reshaped)
sigma=K.dot(sigma, K.transpose(_u))
#normalize it
W_bar = W_reshaped / sigma
#reshape weight tensor
if training in {0, False}:
W_bar = K.reshape(W_bar, W_shape)
else:
with tf.control_dependencies([self.u.assign(_u)]):
W_bar = K.reshape(W_bar, W_shape)
#update weitht
self.kernel = W_bar
if self.rank == 1:
outputs = K.conv1d(
inputs,
self.kernel,
strides=self.strides[0],
padding=self.padding,
data_format=self.data_format,
dilation_rate=self.dilation_rate[0])
if self.rank == 2:
outputs = K.conv2d(
inputs,
self.kernel,
strides=self.strides,
padding=self.padding,
data_format=self.data_format,
dilation_rate=self.dilation_rate)
if self.rank == 3:
outputs = K.conv3d(
inputs,
self.kernel,
strides=self.strides,
padding=self.padding,
data_format=self.data_format,
dilation_rate=self.dilation_rate)
if self.use_bias:
outputs = K.bias_add(
outputs,
self.bias,
data_format=self.data_format)
if self.activation is not None:
return self.activation(outputs)
return outputs
def __multiply_tensors(cls, x):
features = backend.batch_flatten(backend.permute_dimensions(x, (2, 0, 1)))
return backend.dot(features, backend.transpose(features))
def gram_matrix(x):
assert K.ndim(x) == 3
features = K.batch_flatten(K.permute_dimensions(x, (2, 0, 1)))
gram = K.dot(features, K.transpose(features))
return gram
def loss(y_true, y_pred, alpha):
mse = K.mean(K.square(y_pred - K.transpose(y_pred)), axis=-1)
return binary_crossentropy_with_nan(y_true, y_pred) + alpha * mse
def gram_matrix(x):
assert K.ndim(x) == 3
features = K.batch_flatten(x)
gram = K.dot(features - 1, K.transpose(features - 1))
return gram
def kappa_metric(t, x):
u = 0.5 * K.sum(K.square(x - t))
v = K.dot(K.transpose(x), t - K.mean(t))
return v / (v + u)
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
y_pred_f = K.flatten(y_pred)
return (2. * K.dot(y_true_f, K.transpose(y_pred_f)) + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)
def gram_matrix(X):
_X = K.squeeze(X, 0)
features = K.batch_flatten(K.permute_dimensions(_X, (2, 0, 1)))
gram = K.dot(features, K.transpose(features))
return K.expand_dims(gram / K.cast(K.prod(_X.shape), 'float32'), axis=0)
