def call(self, x, **kwargs):
assert isinstance(x, list)
inp_a, inp_b = x
last_state = K.expand_dims(inp_b[:, -1, :], 1)
m = []
for i in range(self.output_dim):
outp_a = inp_a * self.W[i]
outp_last = last_state * self.W[i]
outp_a = K.l2_normalize(outp_a, -1)
outp_last = K.l2_normalize(outp_last, -1)
outp = K.batch_dot(outp_a, outp_last, axes=[2, 2])
m.append(outp)
if self.output_dim > 1:
persp = K.concatenate(m, 2)
else:
persp = m[0]
return [persp, persp]
def call(self, inputs, **kwargs):
q, k = inputs
q = K.repeat_elements(q, K.int_shape(k)[1], axis=1)
a = K.concatenate([q, k, q - k, q * k], axis=-1)
a = self.dense1(a)
a = self.dense2(a)
a = self.dense3(a)
a = tf.transpose(a, (0, 2, 1))
# a_output = K.dot(a, k)
a_output = tf.matmul(a, k)
return a_output
def _produce_context_embeddings(self, query_vector, sentence_vector):
# assume transformer layer follows a BERT architecture
assert(self.cls_token_id is not None and self.sep_token_id is not None and self.pad_token_id is not None)
batch_dim = tf.shape(query_vector)[0]
cls_input = K.expand_dims(tf.ones((batch_dim,), dtype="int32")*tf.constant(self.cls_token_id))
sep_input = K.expand_dims(tf.ones((batch_dim,), dtype="int32")*tf.constant(self.sep_token_id))
_input = K.concatenate([cls_input, query_vector, sep_input, sentence_vector, sep_input])
_out = self.context_embedding_layer(_input)
context_query = _out[:,1:self.query_max_elements+1,:]
context_sentence = _out[:,self.query_max_elements+2:self.query_max_elements+2+self.setence_max_elements,:]
return context_query, context_sentence
def call(self, inputs, mask=None):
features, feature_graph_index = inputs
feature_graph_index = tf.reshape(feature_graph_index, (-1, ))
_, _, count = tf.unique_with_counts(feature_graph_index)
m = kb.dot(features, self.m_weight)
if self.use_bias:
m += self.m_bias
self.h = tf.zeros(
tf.stack([
tf.shape(input=features)[0],
tf.shape(input=count)[0], self.n_hidden
]))
self.c = tf.zeros(
tf.stack([
tf.shape(input=features)[0],
tf.shape(input=count)[0], self.n_hidden
]))
q_star = tf.zeros(
tf.stack([
tf.shape(input=features)[0],
tf.shape(input=count)[0], 2 * self.n_hidden
]))
for i in range(self.T):
self.h, c = self._lstm(q_star, self.c)
e_i_t = tf.reduce_sum(
input_tensor=m *
repeat_with_index(self.h, feature_graph_index),
axis=-1)
exp = tf.exp(e_i_t)
# print('exp shape ', exp.shape)
seg_sum = tf.transpose(a=tf.math.segment_sum(
tf.transpose(a=exp, perm=[1, 0]), feature_graph_index),
perm=[1, 0])
seg_sum = tf.expand_dims(seg_sum, axis=-1)
# print('seg_sum shape', seg_sum.shape)
interm = repeat_with_index(seg_sum, feature_graph_index)
# print('interm shape', interm.shape)
a_i_t = exp / interm[..., 0]
# print(a_i_t.shape)
r_t = tf.transpose(a=tf.math.segment_sum(
tf.transpose(a=tf.multiply(m, a_i_t[:, :, None]),
perm=[1, 0, 2]), feature_graph_index),
perm=[1, 0, 2])
q_star = kb.concatenate([self.h, r_t], axis=-1)
return q_star
def build_model_lstm_gru_attn(n_seq, d_model, n_code, d_code, n_output=1):
"""
RNN Model whth Embedding
:param n_seq: number of vocab
:param d_model: hidden size
:param n_code: number of code
:param d_code: dim of code embed
:param n_output: number of output
:return model: model object
"""
inputs_1 = tf.keras.layers.Input((n_seq, d_model)) # (bs, n_seq, d_model)
inputs_2 = tf.keras.layers.Input((n_seq, )) # (bs, 1)
code_embed = tf.keras.layers.Embedding(n_code, d_code)
code_hidden = code_embed(inputs_2) # (bs, n_seq, d_code)
hidden = K.concatenate([inputs_1, code_hidden],
axis=-1) # (bs, n_seq, d_model + d_code)
lstm = tf.keras.layers.LSTM(units=64,
return_sequences=True,
activation=tf.nn.relu) # (bs, n_seq, units)
hidden_lstm = lstm(hidden) # (bs, n_seq, units)
gru = tf.keras.layers.GRU(units=64,
return_sequences=True,
activation=tf.nn.relu) # (bs, n_seq, units)
hidden_gru = gru(hidden_lstm) # (bs, n_seq, units)
attn = DotProductAttention()
attn_lstm = attn((hidden_lstm, hidden_lstm, hidden_lstm))
attn_gru = attn((hidden_gru, hidden_gru, hidden_gru))
hidden_lstm = tf.keras.layers.GlobalMaxPool1D()(hidden_lstm) # (bs, units)
hidden_gru = tf.keras.layers.GlobalMaxPool1D()(hidden_gru) # (bs, units)
attn_lstm = tf.keras.layers.GlobalMaxPool1D()(attn_lstm) # (bs, units)
attn_gru = tf.keras.layers.GlobalMaxPool1D()(attn_gru) # (bs, units)
hidden = tf.concat([hidden_lstm, hidden_gru, attn_lstm, attn_gru],
axis=-1) # (bs, units * 4)
hidden = tf.keras.layers.Dense(64, activation=tf.nn.relu)(hidden)
output_dense = tf.keras.layers.Dense(n_output)
outputs = output_dense(hidden) # (bs, n_output)
model = tf.keras.Model(inputs=(inputs_1, inputs_2), outputs=outputs)
return model
def call(self, inputs):
categorical_inputs, numerical_inputs = inputs
if categorical_inputs and numerical_inputs and \
categorical_inputs[0].shape[-1] != numerical_inputs[0].shape[-1] \
and self._numerical_interactive is True:
raise ValueError('If `fm_numerical_interactive` is True, '
'categorical_inputs`s shape must equals to numerical_inputs`s shape')
if self._numerical_interactive is True:
exp_inputs = [K.expand_dims(x, axis=1) for x in categorical_inputs+numerical_inputs]
else:
exp_inputs = [K.expand_dims(x, axis=1) for x in categorical_inputs]
concat_inputs = K.concatenate(exp_inputs, axis=1)
square_inputs = K.square(K.sum(concat_inputs, axis=1))
sum_inputs = K.sum(K.square(concat_inputs), axis=1)
cross_term = square_inputs - sum_inputs
outputs = 0.5 * K.sum(cross_term, axis=1, keepdims=True)
return outputs
def __init__(self, content: np.ndarray, style: np.ndarray):
tf.compat.v1.disable_eager_execution()
K.set_floatx('float64')
self.content = content
self.style = style
print(" Building transfer model.")
self.contentTensor = K.variable(self.content)
self.styleTensor = K.variable(self.style)
self.genTensor = K.placeholder((1, CONTENT_IMG_H, CONTENT_IMG_W, 3))
self.inputTensor = K.concatenate(
[self.contentTensor, self.styleTensor, self.genTensor], axis=0)
self.model = vgg19.VGG19(include_top=False,
input_tensor=self.inputTensor)
self.totalLoss = self.constructTotalLoss()
self.gradient = self.constructGradient()
self.kerasFunction = self.constructKerasFunction()
self.runOutput = None
def densenet(shape=None):
image_input = layers.Input(shape)
x = Conv2D(16, (3, 3), padding='same', activation='relu')(image_input)
for i in range(3):
conv = Conv2D(16 * (i + 1), (3, 3), padding='same',
activation='relu')(x)
conv = MaxPooling2D(pool_size=2, padding='same', strides=(1, 1))(conv)
x = concatenate([x, conv])
x = Flatten()(x)
x = Dense(6, activation='softmax')(x)
model = Model(image_input, x)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['acc'])
return model
def call(self, inputs):
recent_input = inputs[0]
first_input = inputs[1]
if self.activation == 'sigmoid':
segmentation = K.sigmoid(recent_input[:, self.seg_channel, :, :, :])
else:
raise Exception(f'do not support : {self.activation}')
intensity = recent_input[:, self.int_channel, :, :, :]
# Adding channel
segmentation = K.expand_dims(segmentation, axis=1)
intensity = K.expand_dims(intensity, axis=1)
# residual
residual_intensity = first_input - intensity
return K.concatenate([residual_intensity,segmentation], axis=1)
def _full_quadratic_interaction(self, x):
"""
This option generates full-quadratic interactions, which include all
linear, bilinear and quadratic terms. It does *not* include an
intercept. Let :math:`b` and :math:`n` be the batch size and number of
features. Then, the input shape is :math:`(b, n)` and the output shape
is :math:`(b, (n + 1) (n + 2) / 2 - 1))`.
**Note:** This option requires the `tensorflow` backend.
"""
ones = K.ones_like(K.expand_dims(x[:, 0], axis=1))
x = K.concatenate([ones, x])
x2 = tf.einsum('ij,ik->ijk', x, x) # full outer product w/ dupes
x2 = tf.map_fn(self._triu_slice, x2) # deduped bi-linear interactions
return x2
def yolo_head(feats, anchors, num_classes, input_shape, calc_loss=False):
"""
:param feats: (N, 13, 13, 3 * (5+n_class)), ...
:param anchors: (3, 2)
:param num_classes: 15
:param input_shape: (416, 416)
:param calc_loss:
:return:
"""
# 3
num_anchors = len(anchors)
# Reshape to batch, height, width, num_anchors, box_params.
# (1, 1, 1, 3, 2)
anchors_tensor = K.reshape(K.constant(anchors),
[1, 1, 1, num_anchors, 2])
# (13, 13)
grid_shape = K.shape(feats)[1:3] # height, width
#
grid_y = K.tile(
K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]),
[1, grid_shape[1], 1, 1])
grid_x = K.tile(
K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]),
[grid_shape[0], 1, 1, 1])
grid = K.concatenate([grid_x, grid_y])
# (13, 13, 1, 2)
grid = K.cast(grid, K.floatx())
# (N, 13, 13, 3 * 15)
feats = K.reshape(
feats,
[-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5])
box_xy = (K.sigmoid(feats[..., :2]) + grid) / K.cast(
grid_shape[::-1], K.dtype(feats))
box_wh = K.exp(feats[..., 2:4]) * anchors_tensor / K.cast(
input_shape[::-1], K.dtype(feats))
box_confidence = K.sigmoid(feats[..., 4:5])
box_class_probs = K.sigmoid(feats[..., 5:])
if calc_loss:
# (13, 13, 1, 2), (N, 13, 13, 3, 15), (N, 13, 13, 3, 2), (N, 13, 13, 3, 2)
return grid, feats, box_xy, box_wh
# (N, 13, 13, 3, 2), (N, 13, 13, 3, 2), (N, 13, 13, 3, 1), (N, 13, 13, 3, 10)
return box_xy, box_wh, box_confidence, box_class_probs
def yolo_head(feats, anchors, num_classes, input_shape, calc_loss=False):
num_anchors = len(anchors)
#---------------------------------------------------#
# [1, 1, 1, num_anchors, 2]
#---------------------------------------------------#
feats = tf.convert_to_tensor(feats)
anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2])
#---------------------------------------------------#
# 获得x，y的网格
# (13, 13, 1, 2)
#---------------------------------------------------#
grid_shape = K.shape(feats)[1:3] # height, width
grid_y = K.tile(K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]),
[1, grid_shape[1], 1, 1])
grid_x = K.tile(K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]),
[grid_shape[0], 1, 1, 1])
grid = K.concatenate([grid_x, grid_y])
grid = K.cast(grid, K.dtype(feats))
#---------------------------------------------------#
# 将预测结果调整成(batch_size,13,13,3,85)
# 85可拆分成4 + 1 + 80
# 4代表的是中心宽高的调整参数
# 1代表的是框的置信度
# 80代表的是种类的置信度
#---------------------------------------------------#
feats = K.reshape(feats, [-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5])
#---------------------------------------------------#
# 将预测值调成真实值
# box_xy对应框的中心点
# box_wh对应框的宽和高
#---------------------------------------------------#
box_xy = (K.sigmoid(feats[..., :2]) + grid) / K.cast(grid_shape[...,::-1], K.dtype(feats))
box_wh = K.exp(feats[..., 2:4]) * anchors_tensor / K.cast(input_shape[...,::-1], K.dtype(feats))
box_confidence = K.sigmoid(feats[..., 4:5])
box_class_probs = K.sigmoid(feats[..., 5:])
#---------------------------------------------------------------------#
# 在计算loss的时候返回grid, feats, box_xy, box_wh
# 在预测的时候返回box_xy, box_wh, box_confidence, box_class_probs
#---------------------------------------------------------------------#
if calc_loss == True:
return grid, feats, box_xy, box_wh
return box_xy, box_wh, box_confidence, box_class_probs
def _build_fusion(self, encoder_input, encoder_output):
batch, height, width, channels = Kbackend.int_shape(encoder_output)
# Fusion Layer
if encoder_input.shape[-1] == 1:
repeat_input = Kbackend.concatenate(
[encoder_input, encoder_input, encoder_input], axis=-1)
embs = self.inception_resnet_v2_model(repeat_input)
else:
embs = self.inception_resnet_v2_model(encoder_input)
embs = tf.keras.layers.GlobalAveragePooling2D()(embs)
embs = RepeatVector(height * width)(embs)
embs = Reshape((height, width, embs.shape[-1]))(embs)
embs = concatenate([encoder_output, embs], axis=-1)
return embs
def yolo3_decode(self,
feats,
anchors,
num_classes,
input_shape,
scale_x_y=None):
"""Decode final layer features to bounding box parameters."""
num_anchors = len(anchors)
# Reshape to batch, height, width, num_anchors, box_params.
anchors_tensor = K.reshape(K.constant(anchors),
[1, 1, 1, num_anchors, 2])
grid_shape = K.shape(feats)[1:3] # height, width
grid_y = K.tile(
K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]),
[1, grid_shape[1], 1, 1])
grid_x = K.tile(
K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]),
[grid_shape[0], 1, 1, 1])
grid = K.concatenate([grid_x, grid_y])
grid = K.cast(grid, K.dtype(feats))
feats = K.reshape(
feats,
[-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5])
# Adjust preditions to each spatial grid point and anchor size.
if scale_x_y:
# Eliminate grid sensitivity trick involved in YOLOv4
#
# Reference Paper & code:
# "YOLOv4: Optimal Speed and Accuracy of Object Detection"
# https://arxiv.org/abs/2004.10934
# https://github.com/opencv/opencv/issues/17148
#
box_xy_tmp = K.sigmoid(
feats[..., :2]) * scale_x_y - (scale_x_y - 1) / 2
box_xy = (box_xy_tmp + grid) / (
K.cast(grid_shape[..., ::-1], K.dtype(feats)) + K.epsilon())
else:
box_xy = (K.sigmoid(feats[..., :2]) + grid) / (
K.cast(grid_shape[..., ::-1], K.dtype(feats)) + K.epsilon())
box_wh = K.exp(feats[..., 2:4]) * anchors_tensor / (
K.cast(input_shape[..., ::-1], K.dtype(feats)) + K.epsilon())
return feats, box_xy, box_wh
def call(self, inputs, mask=None):
input_shape = K.int_shape(inputs)
if input_shape[-1] == 1:
inputs = K.squeeze(inputs, axis=-1)
input_shape = input_shape[:-1]
if len(input_shape) > 2:
original_inputs = inputs
inputs = last_dim_flatten(inputs)
if mask is not None:
mask = last_dim_flatten(mask)
# Now we have both inputs and mask with shape (?, num_options), and can do a softmax.
softmax_result = masked_softmax(inputs, mask)
if len(input_shape) > 2:
original_shape = K.shape(original_inputs)
input_shape = K.concatenate([[-1], original_shape[1:]], 0)
softmax_result = K.reshape(softmax_result, input_shape)
return softmax_result
def evaluate_batch_item(batch_item_detections,
num_classes,
max_objects_per_class=20,
max_objects=100,
iou_threshold=0.5,
score_threshold=0.1):
batch_item_detections = tf.boolean_mask(
batch_item_detections,
tf.greater(batch_item_detections[:, 4], score_threshold))
detections_per_class = []
for cls_id in range(num_classes):
# (num_keep_this_class_boxes, 4) score 大于 score_threshold 的当前 class 的 boxes
class_detections = tf.boolean_mask(
batch_item_detections, tf.equal(batch_item_detections[:, 5],
cls_id))
nms_keep_indices = tf.image.non_max_suppression(
class_detections[:, :4],
class_detections[:, 4],
max_objects_per_class,
iou_threshold=iou_threshold)
class_detections = K.gather(class_detections, nms_keep_indices)
detections_per_class.append(class_detections)
batch_item_detections = K.concatenate(detections_per_class, axis=0)
def filter():
nonlocal batch_item_detections
_, indices = tf.nn.top_k(batch_item_detections[:, 4], k=max_objects)
batch_item_detections_ = tf.gather(batch_item_detections, indices)
return batch_item_detections_
def pad():
nonlocal batch_item_detections
batch_item_num_detections = tf.shape(batch_item_detections)[0]
batch_item_num_pad = tf.maximum(
max_objects - batch_item_num_detections, 0)
batch_item_detections_ = tf.pad(tensor=batch_item_detections,
paddings=[[0, batch_item_num_pad],
[0, 0]],
mode='CONSTANT',
constant_values=0.0)
return batch_item_detections_
batch_item_detections = tf.cond(
tf.shape(batch_item_detections)[0] >= 100, filter, pad)
return batch_item_detections
def styleTransfer(cData, sData, tData):
print(" Building transfer model.")
contentTensor = K.variable(cData)
styleTensor = K.variable(sData)
genTensor = K.placeholder((1, CONTENT_IMG_H, CONTENT_IMG_W, 3))
inputTensor = K.concatenate([contentTensor, styleTensor, genTensor], axis=0)
model = vgg19.VGG19(include_top=False, weights="imagenet", input_tensor=inputTensor)
outputDict = dict([(layer.name, layer.output) for layer in model.layers])
print(" VGG19 model loaded.")
loss = 0.0
styleLayerNames = ["block1_conv1", "block2_conv1", "block3_conv1", "block4_conv1", "block5_conv1"]
contentLayerName = "block5_conv2"
print(" Calculating content loss.")
contentLayer = outputDict[contentLayerName]
contentOutput = contentLayer[0, :, :, :]
genOutput = contentLayer[2, :, :, :]
loss += CONTENT_WEIGHT * contentLoss(contentOutput, genOutput)
print(" Calculating style loss.")
for layer_name in styleLayerNames:
layer_features = outputDict[layer_name]
style_reference_features = layer_features[1, :, :, :]
combination_features = layer_features[2, :, :, :]
sl = styleLoss(style_reference_features, combination_features)
loss = loss + (STYLE_WEIGHT / len(styleLayerNames)) * sl
loss += TOTAL_WEIGHT * totalLoss(genTensor)
gradient = K.gradients(loss, genTensor)
outputs = [loss]
if isinstance(gradient, (list, tuple)):
outputs += gradient
else:
outputs.append(gradient)
global f_outputs
f_outputs = K.function([genTensor], outputs)
print(" Beginning transfer.")
evaluator = Evaluator()
for i in range(TRANSFER_ROUNDS):
index = i + 1
print(" Step %d." % i)
tData, tLoss, info = fmin_l_bfgs_b(evaluator.loss, tData.flatten(), fprime=evaluator.grads, maxfun=20)
print(" Loss: %f." % tLoss)
img = deprocessImage(tData.copy())
saveFile = "image" + str(index) + ".jpg"
imageio.imwrite(saveFile, img)
print(" Image saved to \"%s\"." % saveFile)
print(" Transfer complete.")
def inverse_dft_model():
# inp = Input(shape=(N_ADSORP,), name='generator_input', batch_size=generator_batchsize)
inp = Input(shape=(N_ADSORP, ),
batch_size=generator_batchsize,
name='generator_input')
x1 = Dense(GRID_SIZE * GRID_SIZE * 64, name='fc1',
activation='relu')(inp[:, :20])
x2 = Dense(GRID_SIZE * GRID_SIZE * 64, name='fc2',
activation='relu')(inp[:, 20:])
x1 = Reshape((GRID_SIZE, GRID_SIZE, 64))(x1)
x2 = Reshape((GRID_SIZE, GRID_SIZE, 64))(x2)
x1 = BatchNormalization()(x1)
x2 = BatchNormalization()(x2)
x1 = Conv2DTranspose(64, 20, strides=1, padding='same',
activation='relu')(x1)
x1 = BatchNormalization()(x1)
x2 = Conv2DTranspose(64, 20, strides=1, padding='same',
activation='relu')(x2)
x2 = BatchNormalization()(x2)
x1 = Conv2DTranspose(64, 20, strides=1, padding='same',
activation='relu')(x1)
x1 = BatchNormalization()(x1)
x2 = Conv2DTranspose(64, 20, strides=1, padding='same',
activation='relu')(x2)
x2 = BatchNormalization()(x2)
x1 = Conv2D(1, 3, strides=1, padding='same', activation='relu')(x1)
x1 = BatchNormalization()(x1)
x2 = Conv2D(1, 3, strides=1, padding='same', activation='relu')(x2)
x2 = BatchNormalization()(x2)
x = K.concatenate((x1, x2), axis=-1)
out = Conv2D(1,
3,
strides=1,
padding='same',
activation=binary_sigmoid,
name='generator_conv')(x)
out = Reshape((GRID_SIZE, GRID_SIZE))(out)
model = Model(inputs=inp, outputs=out, name='generator_model')
return model
def call(self, x):
# print("!",x.shape)
E = K.reshape(x[:, :, 0], (-1, self.gs, 1))
p1 = K.reshape(x[:, :, 1], (-1, self.gs, 1))
p2 = K.reshape(x[:, :, 2], (-1, self.gs, 1))
p3 = K.reshape(x[:, :, 3], (-1, self.gs, 1))
pt = K.sqrt(p1**2 + p2**2)
p = K.sqrt(pt**2 + p3**2)
iszero = p**2 + E**2
iszero = 1 - K.relu(1 - self.numericC * iszero) + K.relu(
-self.numericC * iszero)
#return iszero
eta = iszero * 0.5 * K.log(0.0000000001 + (p + p3) /
(p - p3 + 0.0000000001))
#phi=iszero*t.math.acos(p3/(p+0.0000001))
phi = iszero * t.math.atan2(p2, p1)
#print("eta",eta.shape,"phi",phi.shape)
meta = K.mean(eta, axis=-2)
mphi = K.mean(phi, axis=-2)
#print("meta",meta.shape,"mphi",mphi.shape)
#deta=eta#-meta##not sure if abs here
#dphi=phi#-mphi##not sure here either
deta = iszero * K.permute_dimensions(
K.permute_dimensions(eta, (1, 0, 2)) - meta,
(1, 0, 2)) #not sure if abs here required
dphi = iszero * K.permute_dimensions(
K.permute_dimensions(phi, (1, 0, 2)) - mphi,
(1, 0, 2)) #also not sure here either
ret = K.concatenate((iszero, deta, dphi),
axis=-1) #adding iszero for numerical reasons
#print(ret.shape,x.shape)
#exit()
return ret
def call(self, x, **kwargs):
"""
Apply MeanAggregation on input tensors, x
Args:
x: List of Keras Tensors with the following elements
- x[0]: tensor of self features shape (n_batch, n_head, n_feat)
- x[1+r]: tensors of neighbour features each of shape (n_batch, n_head, n_neighbour[r], n_feat[r])
Returns:
Keras Tensor representing the aggregated embeddings in the input.
"""
# Calculate the mean vectors over the neigbours of each relation (edge) type
neigh_agg_by_relation = []
for r in range(self.nr):
# The neighbour input tensors for relation r
z = x[1 + r]
# If there are neighbours aggregate over them
if z.shape[2] > 0:
z_agg = K.dot(K.mean(z, axis=2), self.w_neigh[r])
# Otherwise add a synthetic zero vector
else:
z_shape = K.shape(z)
w_shape = self.half_output_dim
z_agg = tf.zeros((z_shape[0], z_shape[1], w_shape))
neigh_agg_by_relation.append(z_agg)
# Calculate the self vector shape (n_batch, n_head, n_out_self)
from_self = K.dot(x[0], self.w_self)
# Sum the contributions from all neighbour averages shape (n_batch, n_head, n_out_neigh)
from_neigh = sum(neigh_agg_by_relation) / self.nr
# Concatenate self + neighbour features, shape (n_batch, n_head, n_out)
total = K.concatenate(
[from_self, from_neigh], axis=2
) # YT: this corresponds to concat=Partial
# TODO: implement concat=Full and concat=False
return self.act((total + self.bias) if self.has_bias else total)
def call(self, inputs, training=None, mask=None):
kwargs = {}
if has_arg(self.layer.call, 'training'):
kwargs['training'] = training
if has_arg(self.layer.call, 'mask') and mask is not None:
kwargs['mask'] = mask
if self.hidden_dim is None:
outputs = [K.expand_dims(layer.call(inputs, **kwargs)) for layer in self.layers]
else:
outputs = []
for i, layer in enumerate(self.layers):
begin = i * self.hidden_dim
end = begin + self.hidden_dim
transformed = K.dot(inputs, self.W[:, begin:end])
if self.use_bias:
transformed += self.b[begin:end]
outputs.append(K.expand_dims(layer.call(transformed, **kwargs)))
return K.concatenate(outputs, axis=-1)
def call(self, x):
# x [n, A]
# T [A, B, C]
# M [n, B, C]
M = tf.tensordot(x, self.T, axes=[[1], [0]])
# M1 [n, B, C, 1]
M1 = K.expand_dims(M, 3)
# M2 [1, B, C, n]
M2 = K.permute_dimensions(M1, [3, 1, 2, 0])
diffs = M1 - M2
# Sum over the features in each vector
# [n, B, C, n] -> [n, B, n].
l1 = K.sum(K.abs(diffs), axis=2)
c = K.exp(-l1)
# Sum over each sample in the minibatch
# [n, B, n] -> [n, B].
o = K.sum(c, axis=2)
return K.concatenate([x, o], 1)
def call(self, inputs):
'''
Args:
inputs: [cs0, st0, cs1, st1, ... , risk0, risk1, ...]
'''
risks = list()
total_loss = 0
for i_event in range(self.n_events):
cs = inputs[i_event*2]
st = inputs[i_event*2 + 1]
risk = inputs[self.n_events * 2 + i_event]
risks.append(risk)
total_loss += self.negative_hazard_log_likelihood(cs, st, risk) * self.event_weights[i_event] / self.n_events
self.add_loss(total_loss)
# only output risks
return K.concatenate(risks, -1)
def call(self, input):
idx1 = self.S * self.S * self.C
idx2 = idx1 + self.S * self.S * self.B
# class probabilities
class_probs = K.reshape(input[:, :idx1], (K.shape(input)[0],) + tuple([self.S, self.S, self.C]))
class_probs = K.softmax(class_probs)
#confidence
confs = K.reshape(input[:, idx1:idx2], (K.shape(input)[0],) + tuple([self.S, self.S, self.B]))
confs = K.sigmoid(confs)
# boxes
boxes = K.reshape(input[:, idx2:], (K.shape(input)[0],) + tuple([self.S, self.S, self.B * 4]))
boxes = K.sigmoid(boxes)
outputs = K.concatenate([class_probs, confs, boxes])
return outputs
def __call__(self, inputs, **kwargs):
if not self.built:
self._maybe_build(inputs)
# check that the implementation matches exactly py torch.
keys = self.keys_fc(inputs)
queries = self.queries_fc(inputs)
values = self.values_fc(inputs)
logits = K.batch_dot(queries, K.permute_dimensions(keys, (0, 2, 1)))
mask = K.ones_like(logits) * np.triu(
(-np.inf) * np.ones(logits.shape.as_list()[1:]), k=1)
logits = mask + logits
probs = Softmax(axis=-1,
name="Softmax_SnailAttn")(logits / self.sqrt_k)
read = K.batch_dot(probs, values)
output = K.concatenate([inputs, read], axis=-1)
return output
def call(self, x):
y_embed = K.cumsum(K.ones_like(x[:, :, :, 0]), 1)
x_embed = K.cumsum(K.ones_like(x[:, :, :, 0]), 2)
if self.normalize:
eps = 1e-6
y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale
x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale
dim_t = K.arange(self.num_pos_feats, dtype='float')
dim = self.temperature ** (2 * (dim_t // 2) /self.num_pos_feats)
pos_x = x_embed[:, :, :, None] / dim_t
pos_y = y_embed[:, :, :, None] / dim_t
pos_x = K.stack((K.sin(pos_x[:, :, :, 0::2]), K.cos(pos_x[:, :, :, 1::2])), axis=4)
b, h, w, d, c = K.int_shape(pos_x)
pos_x = K.reshape(pos_x, (-1, h, w, d*c))
pos_y = K.stack((K.sin(pos_y[:, :, :, 0::2]), K.cos(pos_y[:, :, :, 1::2])), axis=4)
pos_y = K.reshape(pos_y, (-1, h, w, d*c))
pos = K.permute_dimensions(K.concatenate((pos_y, pos_x), axis=3), (0, 3, 1, 2))
return pos
def _find_maxima(x, coordinate_scale=1, confidence_scale=255.0):
x = K.cast(x, K.floatx())
col_max = K.max(x, axis=1)
row_max = K.max(x, axis=2)
maxima = K.max(col_max, 1)
maxima = K.expand_dims(maxima, -2) / confidence_scale
cols = K.cast(K.argmax(col_max, -2), K.floatx())
rows = K.cast(K.argmax(row_max, -2), K.floatx())
cols = K.expand_dims(cols, -2) * coordinate_scale
rows = K.expand_dims(rows, -2) * coordinate_scale
maxima = K.concatenate([cols, rows, maxima], -2)
return maxima
def call(self, inputs):
# calculate mean value for each pixel across channels
mean = backend.mean(inputs, axis=0, keepdims=True)
# calculate squared differences
squ_diffs = backend.square(inputs - mean)
# variance
mean_sq_diff = backend.mean(squ_diffs, axis=0, keepdims=True)
mean_sq_diff += 1e-8
# stdev
stdev = backend.sqrt(mean_sq_diff)
# calculate mean standard deviation across each pixel
mean_pix = backend.mean(stdev, keepdims=True)
shape = backend.shape(inputs)
output = backend.tile(mean_pix, (shape[0], shape[1], shape[2], 1))
# concatenate with the output
combined = backend.concatenate([inputs, output], axis=-1)
return combined
def call(self, inputs):
features = inputs[0]
fltr = inputs[1]
if not K.is_sparse(fltr):
fltr = ops.dense_to_sparse(fltr)
features_neigh = self.aggregate_op(
tf.gather(features, fltr.indices[:, -1]), fltr.indices[:, -2])
output = K.concatenate([features, features_neigh])
output = K.dot(output, self.kernel)
if self.use_bias:
output = K.bias_add(output, self.bias)
if self.activation is not None:
output = self.activation(output)
output = K.l2_normalize(output, axis=-1)
return output
def get_anchors_and_decode(feats, anchors, num_classes, input_shape, calc_loss=False):
num_anchors = len(anchors)
#------------------------------------------#
# grid_shape指的是特征层的高和宽
#------------------------------------------#
grid_shape = K.shape(feats)[1:3]
#--------------------------------------------------------------------#
# 获得各个特征点的坐标信息。生成的shape为(13, 13, num_anchors, 2)
#--------------------------------------------------------------------#
grid_x = K.tile(K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]), [grid_shape[0], 1, num_anchors, 1])
grid_y = K.tile(K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]), [1, grid_shape[1], num_anchors, 1])
grid = K.cast(K.concatenate([grid_x, grid_y]), K.dtype(feats))
#---------------------------------------------------------------#
# 将先验框进行拓展，生成的shape为(13, 13, num_anchors, 2)
#---------------------------------------------------------------#
anchors_tensor = K.reshape(K.constant(anchors), [1, 1, num_anchors, 2])
anchors_tensor = K.tile(anchors_tensor, [grid_shape[0], grid_shape[1], 1, 1])
#---------------------------------------------------#
# 将预测结果调整成(batch_size,13,13,3,85)
# 85可拆分成4 + 1 + 80
# 4代表的是中心宽高的调整参数
# 1代表的是框的置信度
# 80代表的是种类的置信度
#---------------------------------------------------#
feats = K.reshape(feats, [-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5])
#------------------------------------------#
# 对先验框进行解码，并进行归一化
#------------------------------------------#
box_xy = (K.sigmoid(feats[..., :2]) + grid) / K.cast(grid_shape[::-1], K.dtype(feats))
box_wh = K.exp(feats[..., 2:4]) * anchors_tensor / K.cast(input_shape[::-1], K.dtype(feats))
#------------------------------------------#
# 获得预测框的置信度
#------------------------------------------#
box_confidence = K.sigmoid(feats[..., 4:5])
box_class_probs = K.sigmoid(feats[..., 5:])
#---------------------------------------------------------------------#
# 在计算loss的时候返回grid, feats, box_xy, box_wh
# 在预测的时候返回box_xy, box_wh, box_confidence, box_class_probs
#---------------------------------------------------------------------#
if calc_loss == True:
return grid, feats, box_xy, box_wh
return box_xy, box_wh, box_confidence, box_class_probs
