def create_network(self):
# ref_img = img_to_array(load_img(P.target_mask_path))
# img_nrows, img_ncols = ref_img.shape[:2]
# Create tensor variables for images
images = K.concatenate([self.style_image, self.target_image, self.content_image], axis=0)
# Create tensor variables for masks
raw_style_mask, raw_target_mask = load_mask_labels(self.img_nrows, self.img_ncols)
style_mask = K.variable(raw_style_mask.astype('float32'))
target_mask = K.variable(raw_target_mask.astype('float32'))
masks = K.concatenate([style_mask, target_mask], axis=0)
# image model as VGG19
with tf.name_scope("VGG"):
self.image_model = vgg19.VGG19(include_top=False, input_tensor=images)
# mask model as a series of pooling
with tf.name_scope('mask_model'):
mask_input = tf.keras.layers.Input(tensor=masks, shape=(None, None, None), name='mask_input')
x = mask_input
for layer in self.image_model.layers[1:]:
name = 'mask_%s' % layer.name
if 'conv' in layer.name:
x = tf.keras.layers.AveragePooling2D((3, 3), padding='same', strides=(
1, 1), name=name)(x)
elif 'pool' in layer.name:
x = tf.keras.layers.AveragePooling2D((2, 2), name=name)(x)
self.mask_model = tf.keras.Model(mask_input, x)
def yolo_correct_boxes(box_xy, box_wh, input_shape,
image_shape): #调整框的相对大小以适应原始图像的长宽比
'''Get corrected boxes'''
box_yx = box_xy[..., ::-1]
box_hw = box_wh[..., ::-1]
input_shape = K.cast(input_shape, K.dtype(box_yx))
image_shape = K.cast(image_shape, K.dtype(box_yx))
new_shape = K.round(image_shape * K.min(input_shape / image_shape))
offset = (input_shape - new_shape) / 2. / input_shape
scale = input_shape / new_shape
box_yx = (box_yx - offset) * scale
box_hw *= scale
box_mins = box_yx - (box_hw / 2.)
box_maxes = box_yx + (box_hw / 2.)
boxes = K.concatenate([
box_mins[..., 0:1], # y_min
box_mins[..., 1:2], # x_min
box_maxes[..., 0:1], # y_max
box_maxes[..., 1:2] # x_max
])
# Scale boxes back to original image shape.
boxes *= K.concatenate([image_shape, image_shape])
return boxes
def step(dec_input, states):
(prev_output, prev_attention, prev_alignment, prev_attn_rnn_state,
prev_dec_rnn1_state, prev_dec_rnn2_state) = states
dec_input = K.switch(training, dec_input, prev_output)
prenet_out = self.prenet(dec_input)
cell_inputs = K.concatenate([prenet_out, prev_attention], axis=-1)
cell_out, next_attn_rnn_state = self.attn_rnn_cell(
cell_inputs, [prev_attn_rnn_state])
next_attention, next_alignment = self.attention_mechanism(
[cell_out, values, keys])
concatenated = K.concatenate([next_attention, cell_out], axis=-1)
projected = self.projection(concatenated)
dec_rnn1_out, next_dec_rnn1_state = self.decoderRNNCell1(
projected, [prev_dec_rnn1_state])
res_conn1 = projected + dec_rnn1_out
dec_rnn2_out, next_dec_rnn2_state = self.decoderRNNCell2(
res_conn1, [prev_dec_rnn2_state])
res_conn2 = res_conn1 + dec_rnn2_out
next_output = self.output_projection(res_conn2)
return [next_output, next_alignment], [
next_output, next_attention, next_alignment,
next_attn_rnn_state, next_dec_rnn1_state, next_dec_rnn2_state
]
def call(self, inputs, **kwargs):
inputs, memory_length = inputs
memory_length = K.cast(memory_length[0][0], 'int32')
batch_size = K.cast(K.shape(inputs)[0], 'int32')
seq_len = K.cast(K.shape(inputs)[1], 'int32')
# Build new memory
pad = K.tile(inputs[0:1, ...], (self.batch_size - batch_size, 1, 1))
padded = K.concatenate([inputs, pad], axis=0) # (self.batch_size, seq_len, output_dim)
new_memory = K.concatenate([self.memory, padded], axis=1) # (self.batch_size, self.memory_len + self.target_len + seq_len, ...)
new_memory = tf.slice( # (self.batch_size, self.memory_len + self.target_len, output_dim)
new_memory,
(0, seq_len, 0),
(self.batch_size, self.memory_len + self.target_len, self.output_dim),
)
self.add_update(K.update(self.memory, new_memory), inputs)
# Build output
old_memory = tf.slice( # (batch_size, memory_length, output_dim)
new_memory,
(0, K.maximum(0, self.memory_len + self.target_len - seq_len - memory_length), 0),
(batch_size, K.minimum(self.memory_len, memory_length), self.output_dim),
)
return old_memory
def update_mask(self, padding_mask, dataset_batch):
"""Calculate and cache the amount of padding required for a batch."""
original_batch_size = self.get_real_batch_size(dataset_batch)
missing_count = self.padded_batch_size - original_batch_size
mask = K.concatenate([array_ops.ones(original_batch_size),
array_ops.zeros(missing_count)], axis=0)
return K.concatenate([padding_mask, mask], axis=0)
def call(self, inputs, **kwargs):
embedded_split = K.reshape(inputs, shape=(self.N, self.M, -1))
center = K.l2_normalize(K.mean(embedded_split, axis=1), axis=-1)
center_except = K.l2_normalize(K.reshape(
K.sum(embedded_split, axis=1, keepdims=True) - embedded_split,
shape=(self.N * self.M, -1)),
axis=-1)
similarity = K.concatenate([
K.concatenate([
K.sum(center_except[i * self.M:(i + 1) * self.M, :] *
embedded_split[j, :, :],
axis=1,
keepdims=True) if i == j else K.sum(
center[i:(i + 1), :] * embedded_split[j, :, :],
axis=1,
keepdims=True) for i in range(self.N)
],
axis=1) for j in range(self.N)
],
axis=0)
similarity = self.w * similarity + self.b
return similarity
def YOLOCorrectBoxes(box_xy, box_wh, input_shape, image_shape):
'''Get Corrected Boxes.'''
box_yx = box_xy[..., ::-1]
box_hw = box_wh[..., ::-1]
input_shape = K.cast(input_shape, K.dtype(box_yx))
image_shape = K.cast(image_shape, K.dtype(box_yx))
new_shape = K.round(image_shape * K.min(input_shape / image_shape))
offset = (input_shape - new_shape) / 2. / input_shape
scale = input_shape / new_shape
box_yx = (box_yx - offset) * scale
box_hw *= scale
box_mins = box_yx - (box_hw / 2.)
box_max = box_yx + (box_hw / 2.)
boxes = K.concatenate([
box_mins[..., 0:1], #y min
box_mins[..., 1:2], #x min
box_max[..., 0:1], #y max
box_max[..., 1:2] #x max
])
#Scale boxes back to original image shape
boxes *= K.concatenate([image_shape, image_shape])
return boxes
def yolo_eval(yolo_outputs,
anchors,
num_classes,
image_shape,
score_threshold,
iou_threshold,
max_boxes=20):
"""Evaluate YOLO model on given input and return nms filtered boxes."""
num_layers = len(yolo_outputs) # This refers to the different scales at which yolo detects objects
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] if num_layers == 3 else [[3, 4, 5], [1, 2, 3]] # default or tiny yolo
input_shape = K.shape(yolo_outputs[0])[1:3] * 32
boxes = []
box_scores = []
box_classes = []
for l in range(num_layers):
_boxes, _box_scores, _box_classes = process_yolo_layer_output(yolo_outputs[l], anchors[anchor_mask[l]], num_classes, input_shape, image_shape)
boxes.append(_boxes)
box_scores.append(_box_scores)
box_classes.append(_box_classes)
boxes = K.concatenate(boxes, axis=0)
box_scores = K.concatenate(box_scores, axis=0)
box_classes = K.concatenate(box_classes, axis=0)
scores_, boxes_, classes_ = non_max_suppression(box_scores, boxes, box_classes, max_boxes, iou_threshold, score_threshold)
return boxes_, scores_, classes_
def call(self, inputs,**kwargs):
if K.ndim(inputs[0]) != 3:
raise ValueError("Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs)))
embed_list = inputs
row = []
col = []
num_inputs = len(embed_list)
for i in range(num_inputs - 1):
for j in range(i + 1, num_inputs):
row.append(i)
col.append(j)
p = K.concatenate([embed_list[idx] for idx in row],axis=1) # batch num_pairs k
q = K.concatenate([embed_list[idx] for idx in col],axis=1) # Reshape([num_pairs, self.embedding_size])
#-------------------------
if self.kernel_type == 'mat':
p = tf.expand_dims(p, 1)
# k k* pair* k
# batch * pair
kp = tf.reduce_sum(
# batch * pair * k
tf.multiply(
# batch * pair * k
tf.transpose(
# batch * k * pair
tf.reduce_sum(
# batch * k * pair * k
tf.multiply(
p, self.kernel),
-1),
[0, 2, 1]),
q),
-1)
else:
# 1 * pair * (k or 1)
k = tf.expand_dims(self.kernel, 0)
# batch * pair
kp = tf.reduce_sum(p * q * k, -1)
# p q # b * p * k
return kp
def call(self, inputs, mask=None, training=None):
inputs, relatives, memories, bias_context, bias_relative = inputs
full = K.concatenate([memories, inputs], axis=1) # (batch, prev_len + seq_len, units)
w_q = K.dot(inputs, self.kernel_q) # (batch, seq_len, units)
w_kv = K.dot(full, self.kernel_kv) # (batch, prev_len + seq_len, units * 2)
w_r = K.dot(relatives, self.kernel_r) # (batch, prev_len + seq_len, units)
if self.use_bias:
w_q = K.bias_add(w_q, self.bias_q)
w_kv = K.bias_add(w_kv, self.bias_kv)
w_r = K.bias_add(w_r, self.bias_r)
if self.activation is not None:
w_q = self.activation(w_q)
w_kv = self.activation(w_kv)
w_r = self.activation(w_r)
w_k = w_kv[:, :, :self.units] # (batch, prev_len + seq_len, units)
w_v = w_kv[:, :, self.units:] # (batch, prev_len + seq_len, units)
w_qc = K.bias_add(w_q, bias_context)
w_qc = self._reshape_to_batches(w_qc) # (batch * n_head, seq_len, units_head)
w_k = self._reshape_to_batches(w_k) # (batch * n_head, prev_len + seq_len, units_head)
a_context = K.batch_dot(w_qc, w_k, axes=2) # (batch * n_head, seq_len, prev_len + seq_len)
w_qr = K.bias_add(w_q, bias_relative)
w_qr = self._reshape_to_batches(w_qr) # (batch * n_head, seq_len, units_head)
w_r = self._reshape_to_batches(w_r) # (batch * n_head, prev_len + seq_len, units_head)
a_relative = K.batch_dot(w_qr, w_r, axes=2) # (batch * n_head, seq_len, prev_len + seq_len)
a_relative = self._relative_shift(a_relative) # (batch * n_head, seq_len, prev_len + seq_len)
att = (a_context + a_relative) / K.sqrt(K.constant(self.units_head, dtype=K.floatx()))
exp = K.exp(att - K.max(att, axis=-1, keepdims=True))
q_len, k_len = K.shape(w_q)[1], K.shape(w_k)[1]
indices = K.expand_dims(K.arange(0, k_len), axis=0)
upper = K.expand_dims(K.arange(k_len - q_len, k_len), axis=-1)
exp *= K.expand_dims(K.cast(indices <= upper, K.floatx()), axis=0)
if mask is not None and mask[0] is not None:
mask = K.cast(mask[0], K.floatx())
mask = K.concatenate([K.ones_like(memories[:, :, 0]), mask], axis=1)
exp *= K.expand_dims(self._reshape_mask(mask), axis=1)
att = exp / K.sum(exp, axis=-1, keepdims=True)
if self.att_drop_layer is not None:
att = self.att_drop_layer(att, training=training)
w_v = self._reshape_to_batches(w_v) # (batch * n_head, prev_len + seq_len, units_head)
w_o = K.batch_dot(att, w_v) # (batch * n_head, seq_len, units_head)
w_o = self._reshape_from_batches(w_o) # (batch, seq_len, units)
w_o = K.dot(w_o, self.kernel_o) # (batch, seq_len, units)
if self.use_bias:
w_o = K.bias_add(w_o, self.bias_o)
if self.activation is not None:
w_o = self.activation(w_o)
# Add shape information to tensor when using `tf.keras`
input_shape = K.int_shape(inputs)
if input_shape[1] is not None:
w_o = K.reshape(w_o, (-1,) + input_shape[1:])
return w_o
def _roi_align(args):
boxes = args[0]
scores = args[1]
fpn = args[2]
# compute from which level to get features from
target_levels = self.map_to_level(boxes)
# process each pyramid independently
rois, ordered_indices = [], []
for i in range(len(fpn)):
# select the boxes and classification from this pyramid level
indices = tf.where(K.equal(target_levels, i))
ordered_indices.append(indices)
level_boxes = tf.gather_nd(boxes, indices)
fpn_shape = K.cast(K.shape(fpn[i]), dtype=K.floatx())
# convert to expected format for crop_and_resize
x1 = level_boxes[:, 0]
y1 = level_boxes[:, 1]
x2 = level_boxes[:, 2]
y2 = level_boxes[:, 3]
level_boxes = K.stack([
(y1 / image_shape[1] * fpn_shape[0]) / (fpn_shape[0] - 1),
(x1 / image_shape[2] * fpn_shape[1]) / (fpn_shape[1] - 1),
(y2 / image_shape[1] * fpn_shape[0] - 1) / (fpn_shape[0] - 1),
(x2 / image_shape[2] * fpn_shape[1] - 1) / (fpn_shape[1] - 1),
], axis=1)
if(len(fpn[i].get_shape()) >=4):
unstack = tf.unstack(fpn[i], axis=3)
temp_stack=[]
for j in unstack:
temp = tf.image.crop_and_resize(
K.expand_dims(j, axis=3),
level_boxes,
tf.zeros((K.shape(level_boxes)[0],), dtype='int32'),
(self.crop_size[0], self.crop_size[1]))
temp_stack.append(temp)
rois.append(temp_stack)
else:
rois.append(tf.image.crop_and_resize(
K.expand_dims(fpn[i], axis=0),
level_boxes,
tf.zeros((K.shape(level_boxes)[0],), dtype='int32'),
self.crop_size
))
# concatenate rois to one blob
rois = K.concatenate(rois, axis=0)
# reorder rois back to original order
indices = K.concatenate(ordered_indices, axis=0)
rois = tf.scatter_nd(indices, rois, K.cast(K.shape(rois), 'int64'))
return rois
def call(self, inputs):
outputs = []
if self.data_format == 'channels_first':
count = 0
for c in range(self.input_spec.axes[1]):
input = inputs[:, c:c+1, ...]
for d in range(self.depth_multiplier):
output = K.conv3d(input
, self.depthwise_kernels[count]
, padding=self.padding
, data_format=self.data_format
, dilation_rate=self.dilation_rate)
if self.use_bias:
output = K.bias_add(output
, self.biases[count]
, data_format=self.data_format)
outputs.append(output)
count +=1
outputs = K.concatenate(outputs, axis=1)
else:
count = 0
for c in range(self.input_spec.axes[4]):
input = inputs[:, c:c + 1, ...]
for d in range(self.depth_multiplier):
output = K.conv3d(input
, self.depthwise_kernels[count]
, padding=self.padding
, data_format=self.data_format
, dilation_rate=self.dilation_rate)
if self.use_bias:
output = K.bias_add(output
, self.biases[count]
, data_format=self.data_format)
outputs.append(output)
count += 1
outputs = K.concatenate(outputs, axis=4)
outputs = K.conv3d(outputs
, self.pointwise_kernel
, padding=self.padding
, data_format=self.data_format
, dilation_rate=self.dilation_rate)
if self.activation is not None:
return self.activation(outputs)
return outputs
def call(self, input):
input_shape = backend.shape(input)
num_rows = input_shape[1]
num_cols = input_shape[2]
row_length = backend.cast(num_rows,
'float32') / (2 * self.circle_number)
col_length = backend.cast(num_cols,
'float32') / (2 * self.circle_number)
outputs = []
pool_circle = []
for jy in range(self.circle_number * 2):
for ix in range(self.circle_number * 2):
x1 = ix * col_length
x2 = ix * col_length + col_length
y1 = jy * row_length
y2 = jy * row_length + row_length
x1 = backend.cast(backend.round(x1), 'int32')
x2 = backend.cast(backend.round(x2), 'int32')
y1 = backend.cast(backend.round(y1), 'int32')
y2 = backend.cast(backend.round(y2), 'int32')
new_shape = [input_shape[0], y2 - y1, x2 - x1, input_shape[3]]
x_crop = input[:, y1:y2, x1:x2, :]
xm = backend.reshape(x_crop, new_shape)
if self.pool_mode == 'avg':
pooled_val = backend.mean(xm, axis=(1, 2))
else:
pooled_val = backend.max(xm, axis=(1, 2))
pool_circle.append(
backend.reshape(xm, (input_shape[0], -1, input_shape[3])))
circle_index = self._circle_index(self.circle_number)
for cidx in circle_index:
circle_val = [pool_circle[idx] for idx in cidx]
if self.pool_mode == 'avg':
pooled_val = backend.mean(backend.concatenate(circle_val,
axis=1),
axis=1)
else:
pooled_val = backend.max(backend.concatenate(circle_val,
axis=1),
axis=1)
outputs.append(pooled_val)
outputs = backend.concatenate(outputs)
outputs = backend.reshape(
outputs,
(input_shape[0], self.nb_channels * self.num_outputs_per_channel))
return outputs
def YOLOEval(yolo_outputs,
anchors,
num_classes,
image_shape,
max_boxes=20,
score_threshold=.6,
iou_threshold=.5):
'''Returns evaluated filtered boxes based on given input.'''
num_layers = len(yolo_outputs)
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]
] if num_layers == 3 else [[3, 4, 5], [1, 2, 3]]
input_shape = K.shape(yolo_outputs[0])[1:3] * 32
boxes = []
box_scores = []
for i in range(num_layers):
_boxes, _box_scores = YOLOBoxesAndScores(yolo_outputs[i],
anchors[anchor_mask[i]],
num_classes, input_shape,
image_shape)
boxes.append(_boxes)
box_scores.append(_box_scores)
boxes = K.concatenate(boxes, axis=0)
box_scores = K.concatenate(box_scores, axis=0)
mask = box_scores >= score_threshold
max_boxes_tensor = K.constant(max_boxes, dtype='int32')
boxes_, scores_, classes_ = [], [], []
for i in range(num_classes):
_class_boxes = tf.boolean_mask(boxes, mask[:, i])
_class_boxes_scores = tf.boolean_mask(box_scores[:, i], mask[:, i])
_nms_index = tf.image.non_max_suppression(_class_boxes,
_class_boxes_scores,
max_boxes_tensor,
iou_threshold=iou_threshold)
_class_boxes = K.gather(_class_boxes, _nms_index)
_class_boxes_scores = K.gather(_class_boxes_scores, _nms_index)
_classes = K.ones_like(_class_boxes_scores, dtype='int32') * i
boxes_.append(_class_boxes)
scores_.append(_class_boxes_scores)
classes_.append(_classes)
boxes_ = K.concatenate(boxes_, axis=0)
scores_ = K.concatenate(scores_, axis=0)
classes_ = K.concatenate(classes_, axis=0)
return boxes_, scores_, classes_
def call(self, inputs, states, training=None):
count, state_in = states
(p_zs, p_mus, p_sigmas, p_hs, p_flatten_memory,
q_zs, q_mus, q_sigmas, q_hs, q_flatten_memory
) = array_ops.split(
state_in,
[self.units, self.units, self.units, self.units,
self.units * self.num_memory_slots,
self.units, self.units, self.units, self.units,
self.units * self.num_memory_slots],
axis=-1)
# prior_inputs = q_hs
prior_inputs = K.concatenate([q_hs, p_hs])
(p_next_zs, p_next_mus, p_next_sigmas,
p_next_hs, p_next_flatten_memory) = self._call_one_layer(
prior_inputs, p_flatten_memory, training, self.p_ws)
p_next_zs = tf.where_v2(
tf.floormod(count, self.time_scale) > 0,
p_zs, p_next_zs)
p_next_mus = tf.where_v2(
tf.floormod(count, self.time_scale) > 0,
p_mus, p_next_mus)
p_next_sigmas = tf.where_v2(
tf.floormod(count, self.time_scale) > 0,
p_sigmas, p_next_sigmas)
p_next_hs = tf.where_v2(
tf.floormod(count, self.time_scale) > 0,
p_hs, p_next_hs)
p_next_flatten_memory = tf.where_v2(
tf.floormod(count, self.time_scale) > 0,
p_flatten_memory, p_next_flatten_memory)
posterior_inputs = K.concatenate(
[inputs, p_next_mus, p_next_sigmas, q_zs])
(q_next_zs, q_next_mus, q_next_sigmas,
q_next_hs, q_next_flatten_memory) = self._call_one_layer(
posterior_inputs, q_flatten_memory, training, self.q_ws)
state_out = K.concatenate(
[p_next_zs, p_next_mus, p_next_sigmas,
p_next_hs, p_next_flatten_memory,
q_next_zs, q_next_mus, q_next_sigmas,
q_next_hs, q_next_flatten_memory])
return ({"p_zs": p_next_zs,
"p_mus": p_next_mus,
"p_sigmas": p_next_sigmas,
"q_zs": q_next_zs,
"q_mus": q_next_mus,
"q_sigmas": q_next_sigmas},
[count + 1.0, state_out])
def call(self, inputs, **kwargs):
boxes, other = inputs
if K.ndim(boxes) == 3:
boxes_shape = K.shape(boxes)
other_shape = K.shape(other)
other = K.reshape(other, (boxes_shape[0], boxes_shape[1], -1))
return K.concatenate([boxes, other], axis=2)
elif K.ndim(boxes) == 4:
boxes_shape = K.shape(boxes)
other_shape = K.shape(other)
other = K.reshape(
other, (boxes_shape[0], boxes_shape[1], boxes_shape[2], -1))
return K.concatenate([boxes, other], axis=3)
def yolo_eval(
yolo_outputs, #通过nms生成相对大小的预测框
anchors,
num_classes,
image_shape,
max_boxes=50,
score_threshold=.6,
iou_threshold=.5):
"""Evaluate YOLO model on given input and return filtered boxes."""
num_layers = len(yolo_outputs)
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] if num_layers == 3 else [[
3, 4, 5
], [1, 2, 3]] # default setting
input_shape = K.shape(yolo_outputs[0])[1:3] * 32
boxes = []
box_scores = []
for l in range(num_layers):
_boxes, _box_scores = yolo_boxes_and_scores(yolo_outputs[l],
anchors[anchor_mask[l]],
num_classes, input_shape,
image_shape)
boxes.append(_boxes)
box_scores.append(_box_scores)
boxes = K.concatenate(boxes, axis=0)
box_scores = K.concatenate(box_scores, axis=0)
mask = box_scores >= score_threshold
max_boxes_tensor = K.constant(max_boxes, dtype='int32')
boxes_ = []
scores_ = []
classes_ = []
for c in range(num_classes):
# TODO: use keras backend instead of tf.
class_boxes = tf.boolean_mask(boxes, mask[:, c])
class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c])
nms_index = tf.image.non_max_suppression(class_boxes,
class_box_scores,
max_boxes_tensor,
iou_threshold=iou_threshold)
class_boxes = K.gather(class_boxes, nms_index)
class_box_scores = K.gather(class_box_scores, nms_index)
classes = K.ones_like(class_box_scores, 'int32') * c
boxes_.append(class_boxes)
scores_.append(class_box_scores)
classes_.append(classes)
boxes_ = K.concatenate(boxes_, axis=0)
scores_ = K.concatenate(scores_, axis=0)
classes_ = K.concatenate(classes_, axis=0)
return boxes_, scores_, classes_
def yolo_head(feats, anchors, num_classes, input_shape, calc_loss=False):
"""Convert final layer features to bounding box parameters."""
num_anchors = len(anchors)
# Reshape to batch, height, width, num_anchors, box_params.
anchors_tensor = K.reshape(K.constant(anchors), [1, 1, 1, num_anchors, 2])
grid_shape = K.shape(feats)[1:3] # height, width
grid_y = K.tile(K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]),
[1, grid_shape[1], 1, 1])
grid_x = K.tile(K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]),
[grid_shape[0], 1, 1, 1])
grid = K.concatenate([grid_x, grid_y])
grid = K.cast(grid, K.dtype(feats))
feats = K.reshape(
feats,
[-1, grid_shape[0], grid_shape[1], num_anchors, num_classes + 5])
# Adjust preditions to each spatial grid point and anchor size.
box_xy = (K.sigmoid(feats[..., :2]) + grid) / K.cast(
grid_shape[::-1], K.dtype(feats))
box_wh = K.exp(feats[..., 2:4]) * anchors_tensor / K.cast(
input_shape[::-1], K.dtype(feats))
box_confidence = K.sigmoid(feats[..., 4:5])
box_class_probs = K.sigmoid(feats[..., 5:])
if calc_loss == True:
return grid, feats, box_xy, box_wh
return box_xy, box_wh, box_confidence, box_class_probs
def call(self, inputs, mask=None, **kwargs):
input_fw = inputs
input_bw = inputs
for i in range(self.layers):
output_fw = self.fw_lstm[i](input_fw)
output_bw = self.bw_lstm[i](input_bw)
output_bw = Lambda(lambda x: K.reverse(x, 1),
mask=lambda inputs, mask: mask)(output_bw)
if i >= self.layers - self.res_layers:
output_fw += input_fw
output_bw += input_bw
input_fw = output_fw
input_bw = output_bw
output_fw = input_fw
output_bw = input_bw
if self.merge_mode == "fw":
output = output_fw
elif self.merge_mode == "bw":
output = output_bw
elif self.merge_mode == 'concat':
output = K.concatenate([output_fw, output_bw])
elif self.merge_mode == 'sum':
output = output_fw + output_bw
elif self.merge_mode == 'ave':
output = (output_fw + output_bw) / 2
elif self.merge_mode == 'mul':
output = output_fw * output_bw
elif self.merge_mode is None:
output = [output_fw, output_bw]
return output
def simple_context(X, mask):
desc, head = X[:, :parameters.max_len_desc, :], X[:, parameters.
max_len_desc:, :]
head_activations, head_words = head[:, :, :parameters.
activation_rnn_size], head[:, :,
parameters.
activation_rnn_size:]
desc_activations, desc_words = desc[:, :, :parameters.
activation_rnn_size], desc[:, :,
parameters.
activation_rnn_size:]
activation_energies = K.batch_dot(head_activations,
desc_activations,
axes=(2, 2))
activation_energies = activation_energies + -1e20 * K.expand_dims(
1. - K.cast(mask[:, :parameters.max_len_desc], 'float32'), 1)
activation_energies = K.reshape(activation_energies,
(-1, parameters.max_len_desc))
activation_weights = K.softmax(activation_energies)
activation_weights = K.reshape(
activation_weights,
(-1, parameters.max_len_head, parameters.max_len_desc))
desc_avg_word = K.batch_dot(activation_weights, desc_words, axes=(2, 1))
return K.concatenate((desc_avg_word, head_words))
def _diag_gradients2(ys, xs, order=1):
"""Returns the gradients of y in `ys` w.r.t. x in `xs`.
`ys` and `xs` are each a Tensor or a list of tensors.
# Arguments
ys: A tensor or list of tesnors to be differentiated.
xs: A tensor or list of tensors to be used for differentiation.
order: Order of differentiation.
# Returns
A list of `D^n y / Dx^n` for each y and x in `ys` and `xs`.
"""
assert ys.shape.as_list() == xs.shape.as_list(), \
'Supported when X and Y has the same dimensions - ' + \
'Xs:{}, Ys:{}'.format(xs.shape.as_list(), ys.shape.as_list())
splitted_ys = tf.split(ys, num_or_size_splits=ys.shape[-1], axis=-1)
ds = []
for j, y in enumerate(splitted_ys):
ds.append(y)
for i in range(order):
ds[-1] = unpack_singleton(
tf.gradients(
ds[-1],
xs,
unconnected_gradients='zero',
# colocate_gradients_with_ops=True, TF: V1.14.0
))
ds[-1] = ds[-1][:, j:j + 1]
# The output is a tensor.
ds = K.concatenate(ds, -1)
return ds
def _pad(self, y):
if self.N > self.num_leaves:
# pads the encoding with zeros in the place of non-leaf nodes
# cast in case our labels are ints
y = tf.cast(y, self.p.dtype)
P = K.tile(self.p, (K.shape(y)[0], 1))
return K.concatenate((y, P))
def call(self, x, mask=None):
'''
shape=(batch_size,new_time_step,filters)
x_cont=Tensor("layer_dropout_5/cond/Identity:0", shape=(None, None, 128), dtype=float32)
x_ques=Tensor("layer_dropout_11/cond/Identity:0", shape=(None, None, 128), dtype=float32)
c_mask=Tensor("batch_slice_4/Slice:0", shape=(None, None), dtype=bool)#
q_mask=Tensor("batch_slice_5/Slice:0", shape=(None, None), dtype=bool)
'''
x_cont, x_ques, c_mask, q_mask = x
# get similarity matrix S
##K.dot(x_cont, self.W0)维度变化： [batch_size,time_step,dim] *[dim,1] =[batch_size,time_step,1]
subres0 = K.tile(K.dot(x_cont, self.W0), [1, 1, self.q_maxlen])
subres1 = K.tile(
K.permute_dimensions(K.dot(x_ques, self.W1), pattern=(0, 2, 1)),
[1, self.c_maxlen, 1])
subres2 = K.batch_dot(x_cont * self.W2,
K.permute_dimensions(x_ques, pattern=(0, 2, 1)))
S = subres0 + subres1 + subres2
S += self.bias
q_mask = tf.expand_dims(q_mask, 1)
#默认是对最后一维度，即axis=-1
S_ = tf.nn.softmax(self.mask_logits(S, q_mask))
c_mask = tf.expand_dims(c_mask, 2)
S_T = K.permute_dimensions(
tf.nn.softmax(self.mask_logits(S, c_mask), axis=1), (0, 2, 1))
c2q = tf.matmul(S_, x_ques)
q2c = tf.matmul(tf.matmul(S_, S_T), x_cont)
result = K.concatenate([x_cont, c2q, x_cont * c2q, x_cont * q2c],
axis=-1)
return result
def call(self, inputs, **kwargs):
boxes, other = inputs
boxes_shape = K.shape(boxes)
n = int(K.ndim(boxes) - 1)
other_shape = tuple([boxes_shape[i] for i in range(n)] + [-1])
other = K.reshape(other, other_shape)
return K.concatenate([boxes, other], axis=K.ndim(boxes) - 1)
def call(self, inputs, mask=None):
# output = softmax(score)
k, q = inputs
if len(q.shape) == 2:
q = K.expand_dims(q, axis=1)
# k: (?, K_LEN, EMBED_DIM,)
# q: (?, Q_LEN, EMBED_DIM,)
# score: (?, Q_LEN, K_LEN,)
if self.score_function == 'scaled_dot_product':
kt = K.permute_dimensions(k, (0, 2, 1))
qkt = K.batch_dot(q, kt)
score = qkt / self.EMBED_DIM
elif self.score_function == 'mlp':
kq = K.concatenate([k, q], axis=1)
kqw2 = K.tanh(K.dot(kq, self.W2))
score = K.permute_dimensions(K.dot(self.W1, kqw2), (1, 0, 2))
elif self.score_function == 'bi_linear':
qw = K.dot(q, self.W)
kt = K.permute_dimensions(k, (0, 2, 1))
score = K.batch_dot(qw, kt)
else:
raise RuntimeError('invalid score_function')
score = K.softmax(score)
# if mask is not None:
# score *= K.cast(mask[0], K.floatx())
# output: (?, Q_LEN, EMBED_DIM,)
output = K.batch_dot(score, k)
return output
def call(self, inputs, output_shape=None):
updates, mask = inputs[0], inputs[1]
mask = tf.cast(mask, 'int32')
input_shape = tf.shape(updates, out_type='int32')
# calculation new shape
if output_shape is None:
output_shape = (input_shape[0], input_shape[1] * self.size[0],
input_shape[2] * self.size[1], input_shape[3])
# calculation indices for batch, height, width and feature maps
one_like_mask = K.ones_like(mask, dtype='int32')
batch_shape = K.concatenate([[input_shape[0]], [1], [1], [1]], axis=0)
batch_range = K.reshape(tf.range(output_shape[0], dtype='int32'),
shape=batch_shape)
b = one_like_mask * batch_range
y = mask // (output_shape[2] * output_shape[3])
x = (mask // output_shape[3]) % output_shape[2]
feature_range = tf.range(output_shape[3], dtype='int32')
f = one_like_mask * feature_range
# transpose indices & reshape update values to one dimension
updates_size = tf.size(updates)
indices = K.transpose(
K.reshape(K.stack([b, y, x, f]), [4, updates_size]))
values = K.reshape(updates, [updates_size])
ret = tf.scatter_nd(indices, values, output_shape)
return ret
def positional_signal(hidden_size: int,
length: int,
min_timescale: float = 1.0,
max_timescale: float = 1e4):
"""
Helper function, constructing basic positional encoding.
The code is partially based on implementation from Tensor2Tensor library
https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/layers/common_attention.py
"""
if hidden_size % 2 != 0:
raise ValueError(
f"The hidden dimension of the model must be divisible by 2."
f"Currently it is {hidden_size}")
position = K.arange(0, length, dtype=K.floatx())
num_timescales = hidden_size // 2
log_timescale_increment = K.constant(
(np.log(float(max_timescale) / float(min_timescale)) /
(num_timescales - 1)),
dtype=K.floatx())
inv_timescales = (min_timescale * K.exp(
K.arange(num_timescales, dtype=K.floatx()) * -log_timescale_increment))
scaled_time = K.expand_dims(position, 1) * K.expand_dims(inv_timescales, 0)
signal = K.concatenate([K.sin(scaled_time), K.cos(scaled_time)], axis=1)
return K.expand_dims(signal, axis=0)
def compute_mask(self, inputs, mask=None):
if mask is None:
return None
if not isinstance(mask, list):
raise ValueError('`mask` should be a list.')
if not isinstance(inputs, list):
raise ValueError('`inputs` should be a list.')
if len(mask) != len(inputs):
raise ValueError('The lists `inputs` and `mask` '
'should have the same length.')
if all(m is None for m in mask):
return None
# Make a list of masks while making sure
# the dimensionality of each mask
# is the same as the corresponding input.
masks = []
for input_i, mask_i in zip(inputs, mask):
if mask_i is None:
# Input is unmasked. Append all 1s to masks,
masks.append(array_ops.ones_like(input_i, dtype='bool'))
elif K.ndim(mask_i) < K.ndim(input_i):
# Mask is smaller than the input, expand it
masks.append(array_ops.expand_dims(mask_i, axis=-1))
else:
masks.append(mask_i)
concatenated = K.concatenate(masks, axis=self.axis)
return K.all(concatenated, axis=-1, keepdims=False)
def scale_boxes_to_original_image_size(box_xy, box_wh, image_shape):
"""Scale boxes from internal representation back to original image shape"""
box_mins = box_xy - (box_wh / 2.)
box_maxes = box_xy + (box_wh / 2.)
top = box_mins[..., 1:2]
left = box_mins[..., 0:1]
bottom = box_maxes[..., 1:2]
right = box_maxes[..., 0:1]
boxes = K.concatenate([top, left, bottom, right])
# Scale boxes back to original image shape.
scaling_tensor = K.concatenate([image_shape, image_shape])
boxes = boxes * scaling_tensor
return boxes
def bias_initializer(_, *args, **kwargs):
return K.concatenate([
self.bias_initializer((self.units, ), *args, **kwargs),
initializers.Ones()((self.units, ), *args, **kwargs),
self.bias_initializer((self.units * 2, ), *args,
**kwargs),
])
def compute_mask(self, inputs, mask=None):
if mask is None:
return None
if not isinstance(mask, list):
raise ValueError('`mask` should be a list.')
if not isinstance(inputs, list):
raise ValueError('`inputs` should be a list.')
if len(mask) != len(inputs):
raise ValueError('The lists `inputs` and `mask` '
'should have the same length.')
if all([m is None for m in mask]):
return None
# Make a list of masks while making sure
# the dimensionality of each mask
# is the same as the corresponding input.
masks = []
for input_i, mask_i in zip(inputs, mask):
if mask_i is None:
# Input is unmasked. Append all 1s to masks,
masks.append(array_ops.ones_like(input_i, dtype='bool'))
elif K.ndim(mask_i) < K.ndim(input_i):
# Mask is smaller than the input, expand it
masks.append(array_ops.expand_dims(mask_i, axis=-1))
else:
masks.append(mask_i)
concatenated = K.concatenate(masks, axis=self.axis)
return K.all(concatenated, axis=-1, keepdims=False)
def call(self,
inputs,
training=None,
mask=None,
initial_state=None,
constants=None):
"""`Bidirectional.call` implements the same API as the wrapped `RNN`."""
kwargs = {}
if generic_utils.has_arg(self.layer.call, 'training'):
kwargs['training'] = training
if generic_utils.has_arg(self.layer.call, 'mask'):
kwargs['mask'] = mask
if generic_utils.has_arg(self.layer.call, 'constants'):
kwargs['constants'] = constants
if initial_state is not None and generic_utils.has_arg(
self.layer.call, 'initial_state'):
forward_state = initial_state[:len(initial_state) // 2]
backward_state = initial_state[len(initial_state) // 2:]
y = self.forward_layer.call(inputs, initial_state=forward_state, **kwargs)
y_rev = self.backward_layer.call(
inputs, initial_state=backward_state, **kwargs)
else:
y = self.forward_layer.call(inputs, **kwargs)
y_rev = self.backward_layer.call(inputs, **kwargs)
if self.return_state:
states = y[1:] + y_rev[1:]
y = y[0]
y_rev = y_rev[0]
if self.return_sequences:
y_rev = K.reverse(y_rev, 1)
if self.merge_mode == 'concat':
output = K.concatenate([y, y_rev])
elif self.merge_mode == 'sum':
output = y + y_rev
elif self.merge_mode == 'ave':
output = (y + y_rev) / 2
elif self.merge_mode == 'mul':
output = y * y_rev
elif self.merge_mode is None:
output = [y, y_rev]
# Properly set learning phase
if (getattr(y, '_uses_learning_phase', False) or
getattr(y_rev, '_uses_learning_phase', False)):
if self.merge_mode is None:
for out in output:
out._uses_learning_phase = True
else:
output._uses_learning_phase = True
if self.return_state:
if self.merge_mode is None:
return output + states
return [output] + states
return output
def compute_mask(self, inputs, mask=None):
if mask is None:
return None
if not isinstance(mask, list):
raise ValueError('`mask` should be a list.')
if not isinstance(inputs, list):
raise ValueError('`inputs` should be a list.')
if len(mask) != len(inputs):
raise ValueError('The lists `inputs` and `mask` '
'should have the same length.')
if all([m is None for m in mask]):
return None
masks = [array_ops.expand_dims(m, axis=0) for m in mask if m is not None]
return K.all(K.concatenate(masks, axis=0), axis=0, keepdims=False)
def call(self,
inputs,
training=None,
mask=None,
initial_state=None,
constants=None):
"""`Bidirectional.call` implements the same API as the wrapped `RNN`."""
kwargs = {}
if generic_utils.has_arg(self.layer.call, 'training'):
kwargs['training'] = training
if generic_utils.has_arg(self.layer.call, 'mask'):
kwargs['mask'] = mask
if generic_utils.has_arg(self.layer.call, 'constants'):
kwargs['constants'] = constants
if initial_state is not None and generic_utils.has_arg(
self.layer.call, 'initial_state'):
forward_inputs = [inputs[0]]
backward_inputs = [inputs[0]]
pivot = len(initial_state) // 2 + 1
# add forward initial state
forward_state = inputs[1:pivot]
forward_inputs += forward_state
if self._num_constants is None:
# add backward initial state
backward_state = inputs[pivot:]
backward_inputs += backward_state
else:
# add backward initial state
backward_state = inputs[pivot:-self._num_constants]
backward_inputs += backward_state
# add constants for forward and backward layers
forward_inputs += inputs[-self._num_constants:]
backward_inputs += inputs[-self._num_constants:]
y = self.forward_layer.call(forward_inputs,
initial_state=forward_state, **kwargs)
y_rev = self.backward_layer.call(backward_inputs,
initial_state=backward_state, **kwargs)
else:
y = self.forward_layer.call(inputs, **kwargs)
y_rev = self.backward_layer.call(inputs, **kwargs)
if self.return_state:
states = y[1:] + y_rev[1:]
y = y[0]
y_rev = y_rev[0]
if self.return_sequences:
y_rev = K.reverse(y_rev, 1)
if self.merge_mode == 'concat':
output = K.concatenate([y, y_rev])
elif self.merge_mode == 'sum':
output = y + y_rev
elif self.merge_mode == 'ave':
output = (y + y_rev) / 2
elif self.merge_mode == 'mul':
output = y * y_rev
elif self.merge_mode is None:
output = [y, y_rev]
else:
raise ValueError(
'Unrecognized value for `merge_mode`: %s' % (self.merge_mode))
if self.return_state:
if self.merge_mode is None:
return output + states
return [output] + states
return output
def bias_initializer(_, *args, **kwargs):
return K.concatenate([
self.bias_initializer((self.filters,), *args, **kwargs),
initializers.Ones()((self.filters,), *args, **kwargs),
self.bias_initializer((self.filters * 2,), *args, **kwargs),
])
def _merge_function(self, inputs): return K.concatenate(inputs, axis=self.axis)
def call(self, inputs):
if not isinstance(inputs, list):
raise ValueError('A merge layer should be called ' 'on a list of inputs.')
if self._reshape_required:
reshaped_inputs = []
input_ndims = list(map(K.ndim, inputs))
if None not in input_ndims:
# If ranks of all inputs are available,
# we simply expand each of them at axis=1
# until all of them have the same rank.
max_ndim = max(input_ndims)
for x in inputs:
x_ndim = K.ndim(x)
for _ in range(max_ndim - x_ndim):
x = array_ops.expand_dims(x, axis=1)
reshaped_inputs.append(x)
return self._merge_function(reshaped_inputs)
else:
# Transpose all inputs so that batch size is the last dimension.
# (batch_size, dim1, dim2, ... ) -> (dim1, dim2, ... , batch_size)
transposed = False
for x in inputs:
x_ndim = K.ndim(x)
if x_ndim is None:
x_shape = array_ops.shape(x)
batch_size = x_shape[0]
new_shape = K.concatenate(
[x_shape[1:],
array_ops.expand_dims(batch_size, axis=-1)])
x_transposed = array_ops.reshape(
x,
array_ops.stack(
[batch_size, math_ops.reduce_prod(x_shape[1:])], axis=0))
x_transposed = array_ops.transpose(x_transposed, perm=(1, 0))
x_transposed = array_ops.reshape(x_transposed, new_shape)
reshaped_inputs.append(x_transposed)
transposed = True
elif x_ndim > 1:
dims = list(range(1, x_ndim)) + [0]
reshaped_inputs.append(array_ops.transpose(x, perm=dims))
transposed = True
else:
# We don't transpose inputs if they are 1D vectors or scalars.
reshaped_inputs.append(x)
y = self._merge_function(reshaped_inputs)
y_ndim = K.ndim(y)
if transposed:
# If inputs have been transposed, we have to transpose the output too.
if y_ndim is None:
y_shape = array_ops.shape(y)
y_ndim = array_ops.shape(y_shape)[0]
batch_size = y_shape[y_ndim - 1]
new_shape = K.concatenate([
array_ops.expand_dims(batch_size, axis=-1), y_shape[:y_ndim - 1]
])
y = array_ops.reshape(y, (-1, batch_size))
y = array_ops.transpose(y, perm=(1, 0))
y = array_ops.reshape(y, new_shape)
elif y_ndim > 1:
dims = [y_ndim - 1] + list(range(y_ndim - 1))
y = array_ops.transpose(y, perm=dims)
return y
else:
return self._merge_function(inputs)