def call(self, x):
"""Postprocess part for YOLOv3 model except NMS."""
assert isinstance(x, list)
#num_layers = len(anchors)//3 # default setting
yolo_outputs, image_shape = x
#anchor_mask = [[6,7,8], [3,4,5], [0,1,2]] if num_layers==3 else [[3,4,5], [0,1,2]] # default setting
#input_shape = K.shape(yolo_outputs[0])[1:3] * 32
batch_size = K.shape(image_shape)[0] # batch size, tensor
boxes = []
box_scores = []
for l in range(self.num_layers):
# get anchor set for each feature layer
if self.num_layers == 3: #YOLOv3 arch
if l == 0:
anchorset = self.anchors[6:]
grid_shape = [
self.input_dim[0] // 32, self.input_dim[1] // 32
]
elif l == 1:
anchorset = self.anchors[3:6]
grid_shape = [
self.input_dim[0] // 16, self.input_dim[1] // 16
]
elif l == 2:
anchorset = self.anchors[:3]
grid_shape = [
self.input_dim[0] // 8, self.input_dim[1] // 8
]
elif self.num_layers == 2: # Tiny YOLOv3 arch
if l == 0:
anchorset = self.anchors[3:]
grid_shape = [
self.input_dim[0] // 32, self.input_dim[1] // 32
]
elif l == 1:
anchorset = self.anchors[:3]
grid_shape = [
self.input_dim[0] // 16, self.input_dim[1] // 16
]
else:
raise ValueError('Invalid layer number')
feats = yolo_outputs[l]
# Convert final layer features to bounding box parameters
num_anchors = len(anchorset)
# Reshape to batch, height, width, num_anchors, box_params.
anchors_tensor = K.reshape(K.constant(anchorset),
[1, 1, 1, num_anchors, 2])
#grid_shape = K.shape(feats)[1:3] # height, width
# get total anchor number for each feature layer
total_anchor_num = grid_shape[0] * grid_shape[1] * num_anchors
grid_y = K.tile(
K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]),
[1, grid_shape[1], 1, 1])
grid_x = K.tile(
K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]),
[grid_shape[0], 1, 1, 1])
grid = K.concatenate([grid_x, grid_y])
grid = K.cast(grid, K.dtype(feats))
reshape_feats = K.reshape(feats, [
-1, grid_shape[0], grid_shape[1], num_anchors,
self.num_classes + 5
])
# Adjust preditions to each spatial grid point and anchor size.
box_xy = (K.sigmoid(reshape_feats[..., :2]) + grid) / K.cast(
grid_shape[::-1], K.dtype(reshape_feats))
box_wh = K.exp(reshape_feats[..., 2:4]) * anchors_tensor / K.cast(
self.input_dim[::-1], K.dtype(reshape_feats))
box_confidence = K.sigmoid(reshape_feats[..., 4:5])
box_class_probs = K.sigmoid(reshape_feats[..., 5:])
# correct boxes to the original image shape
input_shape = K.cast(self.input_dim, K.dtype(box_xy))
image_shape = K.cast(image_shape, K.dtype(box_xy))
#new_shape = K.round(image_shape * K.min(input_shape/image_shape))
new_shape = K.cast(image_shape * K.min(input_shape / image_shape),
dtype='int32')
new_shape = K.cast(new_shape, dtype='float32')
offset = (input_shape - new_shape) / 2. / input_shape
scale = input_shape / new_shape
box_xy = (box_xy - offset) * scale
box_wh *= scale
box_mins = box_xy - (box_wh / 2.)
box_maxes = box_xy + (box_wh / 2.)
_boxes = K.concatenate([
box_mins[..., 0:1], # x_min
box_mins[..., 1:2], # y_min
box_maxes[..., 0:1], # x_max
box_maxes[..., 1:2] # y_max
])
# Scale boxes back to original image shape.
_boxes *= K.concatenate([image_shape, image_shape])
# Reshape boxes to flatten the boxes
_boxes = K.reshape(_boxes, [-1, total_anchor_num, 4])
# check if only 1 class for different score
_box_scores = tf.cond(
K.equal(K.constant(value=self.num_classes, dtype='int32'),
1), lambda: box_confidence,
lambda: box_confidence * box_class_probs)
_box_scores = K.reshape(_box_scores,
[-1, total_anchor_num, self.num_classes])
boxes.append(_boxes)
box_scores.append(_box_scores)
# Merge boxes for all feature layers, for further NMS option
boxes = K.concatenate(boxes, axis=1)
box_scores = K.concatenate(box_scores, axis=1)
return boxes, box_scores
def batched_yolo2_postprocess(args,
anchors,
num_classes,
max_boxes=100,
confidence=0.1,
iou_threshold=0.4):
"""Postprocess for YOLOv2 model on given input and return filtered boxes."""
yolo_outputs = args[0]
image_shape = args[1]
input_shape = K.shape(yolo_outputs)[1:3] * 32
batch_size = K.shape(image_shape)[0] # batch size, tensor
boxes, box_scores = batched_yolo2_boxes_and_scores(yolo_outputs, anchors,
num_classes,
input_shape,
image_shape)
mask = box_scores >= confidence
max_boxes_tensor = K.constant(max_boxes, dtype='int32')
def single_image_nms(b, batch_boxes, batch_scores, batch_classes):
boxes_ = []
scores_ = []
classes_ = []
for c in range(num_classes):
# TODO: use keras backend instead of tf.
class_boxes = tf.boolean_mask(boxes[b], mask[b, :, c])
class_box_scores = tf.boolean_mask(box_scores[b, :, c], mask[b, :,
c])
nms_index = tf.image.non_max_suppression(
class_boxes,
class_box_scores,
max_boxes_tensor,
iou_threshold=iou_threshold)
class_boxes = K.gather(class_boxes, nms_index)
class_box_scores = K.gather(class_box_scores, nms_index)
classes = K.ones_like(class_box_scores, 'int32') * c
boxes_.append(class_boxes)
scores_.append(class_box_scores)
classes_.append(classes)
boxes_ = K.concatenate(boxes_, axis=0)
scores_ = K.concatenate(scores_, axis=0)
classes_ = K.concatenate(classes_, axis=0)
batch_boxes = batch_boxes.write(b, boxes_)
batch_scores = batch_scores.write(b, scores_)
batch_classes = batch_classes.write(b, classes_)
return b + 1, batch_boxes, batch_scores, batch_classes
batch_boxes = tf.TensorArray(K.dtype(boxes), size=1, dynamic_size=True)
batch_scores = tf.TensorArray(K.dtype(box_scores),
size=1,
dynamic_size=True)
batch_classes = tf.TensorArray(dtype=tf.int32, size=1, dynamic_size=True)
_, batch_boxes, batch_scores, batch_classes = tf.while_loop(
lambda b, *args: b < batch_size, single_image_nms,
[0, batch_boxes, batch_scores, batch_classes])
batch_boxes = batch_boxes.stack()
batch_scores = batch_scores.stack()
batch_classes = batch_classes.stack()
return batch_boxes, batch_scores, batch_classes
enc_hidden = encoder.initialize_hidden_state()
steps_per_epoch = len(
pairs_final_train) // batch_size # used for caculating number of batches
current_ep = 1
batch_per_epoche = 6
#Reload a checkpoint if training was interrupted
checkpoint.restore(manager.latest_checkpoint)
print("Restored from {}".format(manager.latest_checkpoint))
last_stopped = int(manager.latest_checkpoint[36:])
with open('training_history.pkl', 'rb') as f:
history = pickle.load(f)
batch_loss = K.constant(0)
X, y = [], []
#Iterate over the training epoches
for ep in range(current_ep, EPOCHS - last_stopped):
current_ep = ep
start = time.time()
total_loss = 0
btch = 1
#Iterates over each pair of conversation
for p in pairs_final_train:
#Split the conversation into message and response
question = p[0]
label = p[1]
def _scharr_edges(cls, image, magnitude):
""" Returns a tensor holding modified Scharr edge maps.
Parameters
----------
image: tensor
Image tensor with shape [batch_size, h, w, d] and type float32. The image(s) must be
2x2 or larger.
magnitude: bool
Boolean to determine if the edge magnitude or edge direction is returned
Returns
-------
tensor
Tensor holding edge maps for each channel. Returns a tensor with shape `[batch_size, h,
w, d, 2]` where the last two dimensions hold `[[dy[0], dx[0]], [dy[1], dx[1]], ...,
[dy[d-1], dx[d-1]]]` calculated using the Scharr filter.
"""
# Define vertical and horizontal Scharr filters.
static_image_shape = image.get_shape()
image_shape = K.shape(image)
# 5x5 modified Scharr kernel ( reshape to (5,5,1,2) )
matrix = np.array([[[[0.00070, 0.00070]], [[0.00520, 0.00370]],
[[0.03700, 0.00000]], [[0.00520, -0.0037]],
[[0.00070, -0.0007]]],
[[[0.00370, 0.00520]], [[0.11870, 0.11870]],
[[0.25890, 0.00000]], [[0.11870, -0.1187]],
[[0.00370, -0.0052]]],
[[[0.00000, 0.03700]], [[0.00000, 0.25890]],
[[0.00000, 0.00000]], [[0.00000, -0.2589]],
[[0.00000, -0.0370]]],
[[[-0.0037, 0.00520]], [[-0.1187, 0.11870]],
[[-0.2589, 0.00000]], [[-0.1187, -0.1187]],
[[-0.0037, -0.0052]]],
[[[-0.0007, 0.00070]], [[-0.0052, 0.00370]],
[[-0.0370, 0.00000]], [[-0.0052, -0.0037]],
[[-0.0007, -0.0007]]]])
num_kernels = [2]
kernels = K.constant(matrix, dtype='float32')
kernels = K.tile(kernels, [1, 1, image_shape[-1], 1])
# Use depth-wise convolution to calculate edge maps per channel.
# Output tensor has shape [batch_size, h, w, d * num_kernels].
pad_sizes = [[0, 0], [2, 2], [2, 2], [0, 0]]
padded = tf.pad(
image, # pylint:disable=unexpected-keyword-arg,no-value-for-parameter
pad_sizes,
mode='REFLECT')
output = K.depthwise_conv2d(padded, kernels)
if not magnitude: # direction of edges
# Reshape to [batch_size, h, w, d, num_kernels].
shape = K.concatenate([image_shape, num_kernels], axis=0)
output = K.reshape(output, shape=shape)
output.set_shape(static_image_shape.concatenate(num_kernels))
output = tf.atan(
K.squeeze(output[:, :, :, :, 0] / output[:, :, :, :, 1],
axis=None))
# magnitude of edges -- unified x & y edges don't work well with Neural Networks
return output
def _smooth_labels(y_true, label_smoothing):
num_classes = tf.cast(K.shape(y_true)[-1], dtype=K.floatx())
label_smoothing = K.constant(label_smoothing, dtype=K.floatx())
return y_true * (1.0 - label_smoothing) + label_smoothing / num_classes
def weighted_NAE(yTrue, yPred):
weights = K.constant([.3, .175, .175, .175, .175], dtype=tf.float32)
return K.sum(weights * K.sum(K.abs(yTrue - yPred)) / K.sum(yPred))
def gamma_initializer(shape, dtype=None):
return K.constant(
np.vstack((np.zeros(shape[0]).reshape(1, -1),
np.tril(np.ones((shape - np.array([1, 0])).astype(int))))))
class TestLosses(unittest.TestCase):
truthly = K.constant(np.ones((2, 2, 1)))
falsely = K.constant(np.zeros((2, 2, 1)))
def test_flatten(self):
from tensorflow.keras.losses import mse
loss_function = losses.flatten(mse)
loss = K.eval(loss_function(self.truthly, self.truthly))
self.assertAlmostEqual(loss, 0, 3)
loss = K.eval(loss_function(self.truthly, self.falsely))
self.assertAlmostEqual(loss, 1, 3)
loss = K.eval(loss_function(self.falsely, self.truthly))
self.assertAlmostEqual(loss, 1, 3)
loss = K.eval(loss_function(self.falsely, self.falsely))
self.assertAlmostEqual(loss, 0, 3)
def test_weighted_crossentropy(self):
loss_function = losses.flatten(losses.weighted_crossentropy(weight=(10, 1)))
loss = K.eval(loss_function(self.truthly, self.truthly))
self.assertAlmostEqual(loss, 0, 3)
loss = K.eval(loss_function(self.truthly, self.falsely))
self.assertAlmostEqual(loss, 8.373037, 3)
loss = K.eval(loss_function(self.falsely, self.truthly))
self.assertAlmostEqual(loss, 0.8373037, 3)
loss = K.eval(loss_function(self.falsely, self.falsely))
self.assertAlmostEqual(loss, 0, 3)
def test_nd_mean_squared_error(self):
loss_function = losses.nd_mean_squared_error
loss = K.eval(loss_function(self.truthly, self.truthly))
self.assertAlmostEqual(loss, 0, 3)
loss = K.eval(loss_function(self.truthly, self.falsely))
self.assertAlmostEqual(loss, 1, 3)
loss = K.eval(loss_function(self.falsely, self.truthly))
self.assertAlmostEqual(loss, 1, 3)
loss = K.eval(loss_function(self.falsely, self.falsely))
self.assertAlmostEqual(loss, 0, 3)
def test_nd_mean_squared_logarithmic_error(self):
loss_function = losses.nd_mean_squared_logarithmic_error
loss = K.eval(loss_function(self.truthly, self.truthly))
self.assertAlmostEqual(loss, 0, 3)
loss = K.eval(loss_function(self.truthly, self.falsely))
self.assertAlmostEqual(loss, 0.48045287, 3)
loss = K.eval(loss_function(self.falsely, self.truthly))
self.assertAlmostEqual(loss, 0.48045287, 3)
loss = K.eval(loss_function(self.falsely, self.falsely))
self.assertAlmostEqual(loss, 0, 3)
def test_nd_poisson(self):
loss_function = losses.nd_poisson
loss = K.eval(loss_function(self.truthly, self.truthly))
self.assertAlmostEqual(loss, 0.9999999, 3)
loss = K.eval(loss_function(self.truthly, self.falsely))
self.assertAlmostEqual(loss, 16.118095, 3)
loss = K.eval(loss_function(self.falsely, self.truthly))
self.assertAlmostEqual(loss, 1, 3)
loss = K.eval(loss_function(self.falsely, self.falsely))
self.assertAlmostEqual(loss, 0, 3)
def test_nd_squared_hinge(self):
loss_function = losses.nd_squared_hinge
loss = K.eval(loss_function(self.truthly, self.truthly))
self.assertAlmostEqual(loss, 0, 3)
loss = K.eval(loss_function(self.truthly, self.falsely))
self.assertAlmostEqual(loss, 1, 3)
loss = K.eval(loss_function(self.falsely, self.truthly))
self.assertAlmostEqual(loss, 4, 3)
loss = K.eval(loss_function(self.falsely, self.falsely))
self.assertAlmostEqual(loss, 1, 3)
def test_nd_binary_crossentropy(self):
loss_function = losses.nd_binary_crossentropy
loss = K.eval(loss_function(self.truthly, self.truthly))
self.assertAlmostEqual(loss, 0, 3)
loss = K.eval(loss_function(self.truthly, self.falsely))
self.assertAlmostEqual(loss, 15.424949, 3)
loss = K.eval(loss_function(self.falsely, self.truthly))
self.assertAlmostEqual(loss, 15.333239, 3)
loss = K.eval(loss_function(self.falsely, self.falsely))
self.assertAlmostEqual(loss, 0, 3)
def test_nd_kullback_leibler_divergence(self):
loss_function = losses.nd_kullback_leibler_divergence
loss = K.eval(loss_function(self.truthly, self.truthly))
self.assertAlmostEqual(loss, 0, 3)
loss = K.eval(loss_function(self.truthly, self.falsely))
self.assertAlmostEqual(loss, 64.47238, 3)
loss = K.eval(loss_function(self.falsely, self.truthly))
self.assertAlmostEqual(loss, 0, 3)
loss = K.eval(loss_function(self.falsely, self.falsely))
self.assertAlmostEqual(loss, 0, 3)
def test_nd_mean_absolute_percentage_error(self):
loss_function = losses.nd_mean_absolute_percentage_error
loss = K.eval(loss_function(self.truthly, self.truthly))
self.assertAlmostEqual(loss, 0, 3)
loss = K.eval(loss_function(self.truthly, self.falsely))
self.assertAlmostEqual(loss, 100, 3)
loss = K.eval(loss_function(self.falsely, self.truthly))
self.assertAlmostEqual(loss, 1000000000, 3)
loss = K.eval(loss_function(self.falsely, self.falsely))
self.assertAlmostEqual(loss, 0, 3)
def __init__(self, output_dim, start_temp=10.0, min_temp=0.1, alpha=0.99999, **kwargs):
self.output_dim = output_dim
self.start_temp = start_temp
self.min_temp = K.constant(min_temp)
self.alpha = K.constant(alpha)
super(ConcreteSelect, self).__init__(**kwargs)
def batched_yolo5_postprocess(args,
anchors,
num_classes,
max_boxes=100,
confidence=0.1,
iou_threshold=0.4,
elim_grid_sense=True):
"""Postprocess for YOLOv3 model on given input and return filtered boxes."""
num_layers = len(anchors)//3 # default setting
yolo_outputs = args[:num_layers]
image_shape = args[num_layers]
if num_layers == 3:
anchor_mask = [[6,7,8], [3,4,5], [0,1,2]]
# YOLOv5 enable "elim_grid_sense" by default
scale_x_y = [2.0, 2.0, 2.0] #if elim_grid_sense else [None, None, None]
else:
anchor_mask = [[3,4,5], [0,1,2]]
scale_x_y = [1.05, 1.05] #if elim_grid_sense else [None, None]
input_shape = K.shape(yolo_outputs[0])[1:3] * 32
batch_size = K.shape(image_shape)[0] # batch size, tensor
# print("yolo_outputs",yolo_outputs)
boxes = []
box_scores = []
for l in range(num_layers):
_boxes, _box_scores = batched_yolo5_boxes_and_scores(yolo_outputs[l],
anchors[anchor_mask[l]], num_classes, input_shape, image_shape, scale_x_y=scale_x_y[l])
boxes.append(_boxes)
box_scores.append(_box_scores)
boxes = K.concatenate(boxes, axis=1)
box_scores = K.concatenate(box_scores, axis=1)
mask = box_scores >= confidence
max_boxes_tensor = K.constant(max_boxes, dtype='int32')
def single_image_nms(b, batch_boxes, batch_scores, batch_classes):
boxes_ = []
scores_ = []
classes_ = []
for c in range(num_classes):
# TODO: use keras backend instead of tf.
class_boxes = tf.boolean_mask(boxes[b], mask[b, :, c])
class_box_scores = tf.boolean_mask(box_scores[b, :, c], mask[b, :, c])
nms_index = tf.image.non_max_suppression(
class_boxes, class_box_scores, max_boxes_tensor, iou_threshold=iou_threshold)
class_boxes = K.gather(class_boxes, nms_index)
class_box_scores = K.gather(class_box_scores, nms_index)
classes = K.ones_like(class_box_scores, 'int32') * c
boxes_.append(class_boxes)
scores_.append(class_box_scores)
classes_.append(classes)
boxes_ = K.concatenate(boxes_, axis=0)
scores_ = K.concatenate(scores_, axis=0)
classes_ = K.concatenate(classes_, axis=0)
batch_boxes = batch_boxes.write(b, boxes_)
batch_scores = batch_scores.write(b, scores_)
batch_classes = batch_classes.write(b, classes_)
return b+1, batch_boxes, batch_scores, batch_classes
batch_boxes = tf.TensorArray(K.dtype(boxes), size=1, dynamic_size=True)
batch_scores = tf.TensorArray(K.dtype(box_scores), size=1, dynamic_size=True)
batch_classes = tf.TensorArray(dtype=tf.int32, size=1, dynamic_size=True)
_, batch_boxes, batch_scores, batch_classes = tf.while_loop(lambda b,*args: b<batch_size, single_image_nms, [0, batch_boxes, batch_scores, batch_classes])
batch_boxes = batch_boxes.stack()
batch_scores = batch_scores.stack()
batch_classes = batch_classes.stack()
return batch_boxes, batch_scores, batch_classes
def yolo_eval(yolo_outputs,
anchors,
num_classes,
image_shape,
max_boxes=20,
score_threshold=.6,
iou_threshold=.5,
eager=False):
if eager:
image_shape = K.reshape(yolo_outputs[-1], [-1])
num_layers = len(yolo_outputs) - 1
else:
# 获得特征层的数量
num_layers = len(yolo_outputs)
# 特征层1对应的anchor是678
# 特征层2对应的anchor是345
# 特征层3对应的anchor是012
anchor_mask = [[3, 4, 5], [0, 1, 2]]
input_shape = K.shape(yolo_outputs[0])[1:3] * 32
boxes = []
box_scores = []
# 对每个特征层进行处理
for l in range(num_layers):
_boxes, _box_scores = yolo_boxes_and_scores(yolo_outputs[l],
anchors[anchor_mask[l]],
num_classes, input_shape,
image_shape)
boxes.append(_boxes)
box_scores.append(_box_scores)
# 将每个特征层的结果进行堆叠
boxes = K.concatenate(boxes, axis=0)
box_scores = K.concatenate(box_scores, axis=0)
mask = box_scores >= score_threshold
max_boxes_tensor = K.constant(max_boxes, dtype='int32')
boxes_ = []
scores_ = []
classes_ = []
for c in range(num_classes):
# 取出所有box_scores >= score_threshold的框，和成绩
class_boxes = tf.boolean_mask(boxes, mask[:, c])
class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c])
# 非极大抑制，去掉box重合程度高的那一些
nms_index = tf.image.non_max_suppression(class_boxes,
class_box_scores,
max_boxes_tensor,
iou_threshold=iou_threshold)
# 获取非极大抑制后的结果
# 下列三个分别是
# 框的位置，得分与种类
class_boxes = K.gather(class_boxes, nms_index)
class_box_scores = K.gather(class_box_scores, nms_index)
classes = K.ones_like(class_box_scores, 'int32') * c
boxes_.append(class_boxes)
scores_.append(class_box_scores)
classes_.append(classes)
boxes_ = K.concatenate(boxes_, axis=0)
scores_ = K.concatenate(scores_, axis=0)
classes_ = K.concatenate(classes_, axis=0)
return boxes_, scores_, classes_
def call(self, x):
# x = x + K.constant(self.pe[:, :x.shape[1].value])
x = x + K.constant(self.pe[:, :x.shape[1]])
return self.dropout(x)
def quantized_model_debug(model, X_test, plot=False):
"""Debugs and plots model weights and activations."""
outputs = []
output_names = []
for layer in model.layers:
if layer.__class__.__name__ in REGISTERED_LAYERS:
output_names.append(layer.name)
outputs.append(layer.output)
model_debug = Model(inputs=model.inputs, outputs=outputs)
y_pred = model_debug.predict(X_test)
print("{:30} {: 8.4f} {: 8.4f}".format(
"input", np.min(X_test), np.max(X_test)))
for n, p in zip(output_names, y_pred):
layer = model.get_layer(n)
if (layer.__class__.__name__ in "QActivation" or
layer.__class__.__name__ in "QAdaptiveActivation"):
alpha = get_weight_scale(layer.activation, p)
else:
alpha = 1.0
print(
"{:30} {: 8.4f} {: 8.4f}".format(n, np.min(p / alpha),
np.max(p / alpha)),
end="")
if alpha != 1.0:
print(" a[{: 8.4f} {:8.4f}]".format(np.min(alpha), np.max(alpha)))
if plot and layer.__class__.__name__ in [
"QConv1D", "QConv2D", "QConv2DTranspose", "QDense", "QActivation",
"QAdaptiveActivation", "QSimpleRNN", "QLSTM", "QGRU", "QBidirectional",
"QSeparableConv1D", "QSeparableConv2D"
]:
plt.hist(p.flatten(), bins=25)
plt.title(layer.name + "(output)")
plt.show()
alpha = None
if layer.__class__.__name__ not in [
"QConv2DBatchnorm", "QDepthwiseConv2DBatchnorm"]:
weights_to_examine = layer.get_weights()
else:
weights_to_examine = layer.get_folded_weights()
for i, weights in enumerate(weights_to_examine):
if hasattr(layer, "get_quantizers") and layer.get_quantizers()[i]:
weights = K.eval(layer.get_quantizers()[i](K.constant(weights)))
if i == 0 and layer.__class__.__name__ in [
"QConv1D", "QConv2D", "QConv2DTranspose", "QDense",
"QSimpleRNN", "QLSTM", "QGRU",
"QSeparableConv1D", "QSeparableConv2D",
"QConv2DBatchnorm", "QDepthwiseConv2DBatchnorm"
]:
alpha = get_weight_scale(layer.get_quantizers()[i], weights)
# if alpha is 0, let's remove all weights.
alpha_mask = (alpha == 0.0)
weights = np.where(alpha_mask, weights * alpha, weights / alpha)
if plot:
plt.hist(weights.flatten(), bins=25)
plt.title(layer.name + "(weights)")
plt.show()
print(" ({: 8.4f} {: 8.4f})".format(np.min(weights), np.max(weights)),
end="")
if alpha is not None and isinstance(alpha, np.ndarray):
print(" a({: 10.6f} {: 10.6f})".format(
np.min(alpha), np.max(alpha)), end="")
print("")
def build_models(self):
min_action = -1.5
max_action = 1.5
initializer_1 = tf.keras.initializers.random_uniform(minval=-0.03,
maxval=0.03)
action_constant_1 = constant([0, 1, 0])
action_constant_2 = constant([1, 0.5, 1])
action_constant_3 = constant([0, 0.5, 0])
in_history = Input(shape=[self.seq_size, self.h, self.w,
1]) # batch, sequence_size, h, w, 1
in_vel = Input(shape=[
self.action_size,
]) # batch, action_size
image_process = BatchNormalization()(in_history)
image_process = TimeDistributed(
Conv2D(16, (3, 3),
activation='elu',
padding='same',
kernel_initializer='he_normal'))(image_process)
#72 128
image_process = TimeDistributed(
Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal'))(image_process)
#70 126
image_process = TimeDistributed(
Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal'))(image_process)
#68 124
image_process = TimeDistributed(MaxPooling2D((2, 2)))(image_process)
#34 62
image_process = TimeDistributed(
Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal'))(image_process)
#32 60
image_process = TimeDistributed(
Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal'))(image_process)
#30 58
image_process = TimeDistributed(MaxPooling2D((2, 2)))(image_process)
#15 29
image_process = TimeDistributed(
Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal'))(image_process)
#13 27
image_process = TimeDistributed(
Conv2D(32, (4, 4),
activation='elu',
kernel_initializer='he_normal'))(image_process)
#10 24
image_process = TimeDistributed(MaxPooling2D((2, 2)))(image_process)
#5 12
image_process = TimeDistributed(
Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal'))(image_process)
#3 10
image_process = TimeDistributed(
Conv2D(8, (1, 1), activation='elu',
kernel_initializer='he_normal'))(image_process)
image_process = TimeDistributed(Flatten())(image_process)
image_process = GRU(48, kernel_initializer='he_normal',
use_bias=False)(image_process)
image_process = BatchNormalization()(image_process)
image_process = Activation('tanh')(image_process)
# vel process
vel_process = Dense(48, kernel_initializer='he_normal',
use_bias=False)(in_vel)
vel_process = BatchNormalization()(vel_process)
vel_process = Activation('tanh')(vel_process)
#add
shared = Add()([image_process, vel_process])
#actor
action = Dense(32, kernel_initializer='he_normal')(shared)
action = BatchNormalization()(action)
action = ELU()(action)
action = Dense(32, kernel_initializer='he_normal')(action)
action = BatchNormalization()(action)
action = ELU()(action)
action = Dense(self.action_size,
kernel_initializer=initializer_1)(action)
action = Lambda(lambda x: clip(x, min_action, max_action))(action)
action = Lambda(lambda x: x + action_constant_1)(action)
action = Lambda(lambda x: x * action_constant_2)(action)
action = Lambda(lambda x: x + action_constant_3)(action)
#critic
value = Dense(32, kernel_initializer='he_normal')(shared)
value = BatchNormalization()(value)
value = ELU()(value)
value = Dense(32, kernel_initializer='he_normal')(value)
value = BatchNormalization()(value)
value = ELU()(value)
value = Dense(1)(value)
actor = Model(inputs=[in_history, in_vel], outputs=action)
critic = Model(inputs=[in_history, in_vel], outputs=value)
whole = Model(inputs=[in_history, in_vel], outputs=[action, value])
return actor, critic, whole
def deprocess_image(x):
# Remove zero-center by mean pixel
x[:, :, 0] += 103.939
x[:, :, 1] += 116.779
x[:, :, 2] += 123.68
# 'BGR'->'RGB'
x = x[:, :, ::-1]
x = np.clip(x, 0, 255).astype('uint8')
return x
# %%
# 学習済みのvgg19ネットワークを読み込み，3つの画像に適用
target_image = K.constant(preprocess_image(target_image_path))
style_reference_image = K.constant(
preprocess_image(style_reference_image_path))
# This placeholder will contain our generated image
combination_image = K.placeholder((1, img_height, img_width, 3))
# We combine the 3 images into a single batch
input_tensor = K.concatenate(
[target_image, style_reference_image, combination_image], axis=0)
# We build the VGG19 network with our batch of 3 images as input.
# The model will be loaded with pre-trained ImageNet weights.
model = vgg19.VGG19(input_tensor=input_tensor,
weights='imagenet',
include_top=False)
def yolo_loss(inputs, num_anchors):
ignore_thresh = .5 # Порог вероятности обнаружения объекта
num_layers = num_anchors // 3 # Подсчитываем количество анкоров на каждом уровне сетки
y_pred = inputs[:num_layers] # Из входных данных выцепляем посчитанные моделью значения
y_true = inputs[num_layers:] # Из входных данных выцепляем эталонные значения
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] # Задаем маску анкоров для каждого уровня сеток
# Получаем размерность входного изображения ( (13 х 13) * 32 = (416 х 416)) и приводим к типу элемента y_true[0]
input_shape = K.cast(K.shape(y_pred[0])[1:3] * 32, K.dtype(y_true[0]))
# Получаем двумерный массив, соответствующий размерностям сеток ((13, 13), (26, 26), (52, 52))
grid_shapes = [K.cast(K.shape(y_pred[l])[1:3], K.dtype(y_true[0])) for l in range(num_layers)]
loss = 0 # Значение ошибки
# Считываем количество элементов
m = K.shape(y_pred[0])[0] # Размер пакета
batch_size = K.cast(m, K.dtype(y_pred[0])) # Преобразуем к типу y_pred[0]
for l in range(num_layers): # Пробегаем по всем трем уровням сеток
# Получаем маску для сетки l-го уровня по вероятности определения объекта (5-ый параметр в списке общих параметров).
# В массиве object_mask будут значения, которые соответствуют только вероятности обнаружения объекта
object_mask = y_true[l][..., 4:5] # Вернется набор данных вида ([0][0][0][0]...[1]...[0])
# Получаем аналогичную выборку для сетки l-го уровня с OHE (где записана позиция нашего класса)
# В массиве true_class будут значения, которые соответствуют только OHE представлению класса для данного уровня анкоров
true_class = y_true[l][..., 5:] # Вернется набор данных вида ([0][0][0][0]...[1]...[0])
num_sub_anchors = len(anchors[anchor_mask[l]]) # Получаем количество анкоров для отдельного уровян сетки (3)
# Решейпим анкоры отдельного уровня сетки и записываем в переменную anchors_tensor
anchors_tensor = K.reshape(K.constant(anchors[anchor_mask[l]]), [1, 1, 1, num_sub_anchors, 2])
# Создаем двумерный массив grid со значениями [[[0, 0] , [0, 1] , [0, 2] , ... , [0, k]],
# [[1, 0] , [1, 1] , [1, 2] , ... , [1 ,k]],
# ...
# [[k, 0] , [k, 1] , [k, 2] , ... , [k, k]]]
# где k - размерность сетки. Массив хранит индексы ячеек сетки
grid_shape = K.shape(y_pred[l])[1:3] # Получаем ширину и высоту сетки
grid_y = K.tile(K.reshape(K.arange(0, stop=grid_shape[0]), [-1, 1, 1, 1]),[1, grid_shape[1], 1, 1]) # Создаем вертикальную линию
grid_x = K.tile(K.reshape(K.arange(0, stop=grid_shape[1]), [1, -1, 1, 1]),[grid_shape[0], 1, 1, 1]) # Создаем горизонтальную линию
grid = K.concatenate([grid_x, grid_y]) # Объединяем
grid = K.cast(grid, K.dtype(y_pred[l])) # Приводим к типу y_pred[l]
# Решейпим y_pred[l]
feats = K.reshape(y_pred[l], [-1, grid_shape[0], grid_shape[1], num_sub_anchors, num_classes + 5])
# Считаем ошибку в определении координат центра объекта
# Получаем координаты центра объекта из спредиктенного значения
pred_xy = (K.sigmoid(feats[..., :2]) + grid) / K.cast(grid_shape[::-1], K.dtype(feats))
# Производим обратные вычисления для оригинальных значений из y_true для координат центра объекта
true_xy = y_true[l][..., :2] * grid_shapes[l][::-1] - grid # Реальные координаты центра bounding_box
box_loss_scale = 2 - y_true[l][...,2:3] * y_true[l][...,3:4] # чем больше бокс, тем меньше ошибка
# binary_crossentropy для истинного значения и спредиктенного (obect_mask для подсчета только требуемого значения)
xy_loss = object_mask * box_loss_scale * K.binary_crossentropy(true_xy, feats[...,0:2], from_logits=True)
# Считаем ошибку в определении координат ширины и высоты
# Получаем значения ширины и высоты изображения из спредиктенного значения
pred_wh = K.exp(feats[..., 2:4]) * anchors_tensor / K.cast(input_shape[::-1], K.dtype(feats))
# Производим обратные вычисления для оригинальных значений из y_true для ширины и высоты объекта
true_wh = K.log(y_true[l][..., 2:4] / anchors[anchor_mask[l]] * input_shape[::-1])
# Оставляем значение высоты и ширины только у тех элементов, где object_mask = 1
true_wh = K.switch(object_mask, true_wh, K.zeros_like(true_wh))
# Считаем значение ошибки в определении высоты и ширины
wh_loss = object_mask * box_loss_scale * 0.5 * K.square(true_wh-feats[...,2:4])
# Объединяем значения в один массив
pred_box = K.concatenate([pred_xy, pred_wh])
# Считаем ошибку в определении обнаружения какого-либо класса
# Для этого вначале надо отсечь все найденные объекты, вероятность которых меньше установленного значения ignore_thresh
# Определяем массив, который будет хранить данные о неподходящих значениях
ignore_mask = tf.TensorArray(K.dtype(y_true[0]), size=1, dynamic_size=True)
object_mask_bool = K.cast(object_mask, 'bool') # Приводим тип object_mask к типу 'bool'
# Функция, определяющая данные, которые требуется игнорировать
# Пробегаем по всем элементам пакета (b<m)
# Получаем параметры реального bounding_box для текущей ячейки
# Считаем IoU реального и спредиктенного
# В зависимости от best_iou < ignore_thresh помечаем его как верно распознанный или неверено
def loop_body(
b,
ignore_mask
):
# в true_box запишутся первые 4 параметра (центр, высота и ширина объекта) того элемента, значение которого в object_mask_bool равно True
true_box = tf.boolean_mask(y_true[l][b,...,0:4], object_mask_bool[b,...,0])
# Подсчитываем iou для спредиктенной ограничивающей рамки (pred_box) и оригинальной (true_box)
iou = calc_iou(pred_box[b], true_box)
# Находим лучшую ограничивающую рамку
best_iou = K.max(iou, axis=-1)
# Записываем в ignore_mask true или false в зависимости от (best_iou < ignore_thresh)
ignore_mask = ignore_mask.write(b, K.cast(best_iou < ignore_thresh, K.dtype(true_box)))
return b+1, ignore_mask # Увеличиваем счетчик на единицу и возвращаем ignore_mask
# Пробегаем в цикле по всем элементам в пределах значения m (m = batch size)
_, ignore_mask = tf.while_loop(lambda b,*args: b<m, loop_body, [0, ignore_mask])
ignore_mask = ignore_mask.stack() # Приводим ignore_mask к тензору
ignore_mask = K.expand_dims(ignore_mask, -1) # Добавляем еще одну размерность в конце ignore_mask
# Считаем значение ошибки
# 1 компонент - для значений, которые были верно спредиктены
# 2 компонент - для значения, которые были неверно спредиктены
confidence_loss = (
object_mask * K.binary_crossentropy(object_mask, feats[...,4:5], from_logits=True) +
(1-object_mask) * K.binary_crossentropy(object_mask, feats[...,4:5], from_logits=True) * ignore_mask
)
# Считаем ошибку в определении класса объекта
class_loss = object_mask * K.binary_crossentropy(true_class, feats[...,5:], from_logits=True)
# Считаем суммарную ошибку
xy_loss = K.sum(xy_loss) / batch_size
wh_loss = K.sum(wh_loss) / batch_size
confidence_loss = K.sum(confidence_loss) / batch_size
class_loss = K.sum(class_loss) / batch_size
loss += xy_loss + wh_loss + confidence_loss + class_loss
return loss # Возвращаем значение ошибки
def _smooth_labels(y_true, label_smoothing):
label_smoothing = K.constant(label_smoothing, dtype=K.floatx())
return y_true * (1.0 - label_smoothing) + 0.5 * label_smoothing
def weighted_acc(y_true, y_pred):
y_true_digit = K.argmax(y_true,axis=-1)
y_pred_digit = K.argmax(y_pred,axis=-1)
mask = tf.subtract(K.constant(1.0,dtype=tf.float32),y_pred[:,:,:,-1])
true_mat = K.cast(K.equal(y_true_digit,y_pred_digit),K.floatx())
return K.sum(tf.multiply(true_mat,mask))/K.sum(mask+1e-12)
def build(self, input_shape):
#assert len(input_shape) >= 2
self.input_dim = input_shape[-1]
self.input_spec = InputSpec(min_ndim=2, axes={-1: self.input_dim}) #never used for another layer
mu_curent, si_current = mu_si_initializer(self.init_mu_current, self.init_sigma_current, self.input_dim,
self.units, verbose=self.verbose)
#print("mu_curent",mu_curent)
self.mu_current = self.add_weight(shape=(self.units,),
initializer=constant(mu_curent),
name="Mu_current",
trainable=self.train_mu)
self.sigma_current = self.add_weight(shape=(self.units,),
initializer=constant(si_current),
name="Sigma_current",
regularizer=self.si_regularizer,
trainable=self.train_sigma)
mu_prev, si_prev = mu_si_initializer(self.init_mu_prev, self.init_sigma_prev, self.input_dim,
self.units, verbose=self.verbose)
#print("mu_prev",mu_prev)
self.mu_prev = self.add_weight(shape=(self.units,),
initializer=constant(mu_prev),
name="Mu_prev",
trainable=self.train_mu)
self.sigma_prev = self.add_weight(shape=(self.units,),
initializer=constant(si_prev),
name="Sigma_prev",
regularizer=self.si_regularizer,
trainable=self.train_sigma)
idxs_current = np.linspace(0, 1.0,self.input_dim) # current previ lazım mı
idxs_current = idxs_current.astype(dtype='float32')
self.idxs_current = K.constant(value=idxs_current, shape=(self.input_dim,),
name="idxs_current")
idxs_prev = np.linspace(0, 1.0,self.units) # current previ lazım mı
idxs_prev = idxs_prev.astype(dtype='float32')
self.idxs_prev = K.constant(value=idxs_prev, shape=(self.units,), #outputa göre
name="idxs_prev")
MIN_SI = 0.01 # zero or below si will crashed calc_u
MAX_SI = 1.0
# create shared vars.
self.MIN_SI = np.float32(MIN_SI)#, dtype='float32')
self.MAX_SI = np.float32(MAX_SI)#, dtype='float32')
print("input_shape= ",input_shape[-1])
print("units= ",self.units)
self.W_current = self.add_weight(shape=(input_shape[-1],self.units),
initializer='uniform',
name='W_current')
self.W_previous = self.add_weight(
shape=(self.units,self.units),
initializer='uniform',
name='W_previous')
self.built = True
# Keras에서 Attention value를 계산하는 절차를 확인한다.
import numpy as np
from tensorflow.keras.layers import Dot, Activation
import tensorflow.keras.backend as K
e = np.array([[[0.3, 0.2, 0.1], [0.3, 0.6, 0.5], [0.3, 0.8, 0.2],
[0.7, 0.2, 0.1]]])
d = np.array([[[0.1, 0.2, 0.3], [0.3, 0.2, 0.5], [0.1, 0.8, 0.4],
[0.4, 0.2, 0.3]]])
e.shape, d.shape
te = K.constant(e)
td = K.constant(d)
dot_product = Dot(axes=(2, 2))([te, td]) # (None, 4, 4)
dot_product
attn_score = Activation('softmax')(dot_product)
attn_score
attn_val = Dot(axes=(2, 1))([attn_score, te])
attn_val
# attn_score 까지는 확실하므로, attn_val 부분만 수동으로 확인해 본다.
score = attn_score.numpy()
score
for n in range(4):
e1 = e[0, 0, :]
v1 = score[0, n, 0] * e1
def call(self, x, mask=None):
'''
Return an anchor box tensor based on the shape of the input tensor.
The logic implemented here is identical to the logic in the module `ssd_box_encode_decode_utils.py`.
Note that this tensor does not participate in any graph computations at runtime. It is being created
as a constant once during graph creation and is just being output along with the rest of the model output
during runtime. Because of this, all logic is implemented as Numpy array operations and it is sufficient
to convert the resulting Numpy array into a Keras tensor at the very end before outputting it.
Arguments:
x (tensor): 4D tensor of shape `(batch, channels, height, width)` if `dim_ordering = 'th'`
or `(batch, height, width, channels)` if `dim_ordering = 'tf'`. The input for this
layer must be the output of the localization predictor layer.
'''
# Compute box width and height for each aspect ratio
# The shorter side of the image will be used to compute `w` and `h` using `scale` and `aspect_ratios`.
size = min(self.img_height, self.img_width)
# Compute the box widths and and heights for all aspect ratios
wh_list = []
for ar in self.aspect_ratios:
if (ar == 1):
# Compute the regular anchor box for aspect ratio 1.
box_height = box_width = self.this_scale * size
wh_list.append((box_width, box_height))
if self.two_boxes_for_ar1:
# Compute one slightly larger version using the geometric mean of this scale value and the next.
box_height = box_width = np.sqrt(
self.this_scale * self.next_scale) * size
wh_list.append((box_width, box_height))
else:
box_height = self.this_scale * size / np.sqrt(ar)
box_width = self.this_scale * size * np.sqrt(ar)
wh_list.append((box_width, box_height))
wh_list = np.array(wh_list)
# We need the shape of the input tensor
if K.image_data_format() == 'tf':
batch_size, feature_map_channels, feature_map_height, feature_map_width = x.shape
else: # Not yet relevant since TensorFlow is the only supported backend right now, but it can't harm to have this in here for the future
batch_size, feature_map_height, feature_map_width, feature_map_channels = x.shape
# Compute the grid of box center points. They are identical for all aspect ratios.
# Compute the step sizes, i.e. how far apart the anchor box center points will be vertically and horizontally.
if (self.this_steps is None):
step_height = self.img_height / feature_map_height
step_width = self.img_width / feature_map_width
else:
if isinstance(self.this_steps,
(list, tuple)) and (len(self.this_steps) == 2):
step_height = self.this_steps[0]
step_width = self.this_steps[1]
elif isinstance(self.this_steps, (int, float)):
step_height = self.this_steps
step_width = self.this_steps
# Compute the offsets, i.e. at what pixel values the first anchor box center point will be from the top and from the left of the image.
if (self.this_offsets is None):
offset_height = 0.5
offset_width = 0.5
else:
if isinstance(self.this_offsets,
(list, tuple)) and (len(self.this_offsets) == 2):
offset_height = self.this_offsets[0]
offset_width = self.this_offsets[1]
elif isinstance(self.this_offsets, (int, float)):
offset_height = self.this_offsets
offset_width = self.this_offsets
# Now that we have the offsets and step sizes, compute the grid of anchor box center points.
cy = np.linspace(offset_height * step_height,
(offset_height + feature_map_height - 1) *
step_height, feature_map_height)
cx = np.linspace(offset_width * step_width,
(offset_width + feature_map_width - 1) * step_width,
feature_map_width)
cx_grid, cy_grid = np.meshgrid(cx, cy)
cx_grid = np.expand_dims(
cx_grid, -1
) # This is necessary for np.tile() to do what we want further down
cy_grid = np.expand_dims(
cy_grid, -1
) # This is necessary for np.tile() to do what we want further down
# Create a 4D tensor template of shape `(feature_map_height, feature_map_width, n_boxes, 4)`
# where the last dimension will contain `(cx, cy, w, h)`
boxes_tensor = np.zeros(
(feature_map_height, feature_map_width, self.n_boxes, 4))
boxes_tensor[:, :, :, 0] = np.tile(cx_grid,
(1, 1, self.n_boxes)) # Set cx
boxes_tensor[:, :, :, 1] = np.tile(cy_grid,
(1, 1, self.n_boxes)) # Set cy
boxes_tensor[:, :, :, 2] = wh_list[:, 0] # Set w
boxes_tensor[:, :, :, 3] = wh_list[:, 1] # Set h
# Convert `(cx, cy, w, h)` to `(xmin, xmax, ymin, ymax)`
boxes_tensor = convert_coordinates(boxes_tensor,
start_index=0,
conversion='centroids2corners')
# If `clip_boxes` is enabled, clip the coordinates to lie within the image boundaries
if self.clip_boxes:
x_coords = boxes_tensor[:, :, :, [0, 2]]
x_coords[x_coords >= self.img_width] = self.img_width - 1
x_coords[x_coords < 0] = 0
boxes_tensor[:, :, :, [0, 2]] = x_coords
y_coords = boxes_tensor[:, :, :, [1, 3]]
y_coords[y_coords >= self.img_height] = self.img_height - 1
y_coords[y_coords < 0] = 0
boxes_tensor[:, :, :, [1, 3]] = y_coords
# If `normalize_coords` is enabled, normalize the coordinates to be within [0,1]
if self.normalize_coords:
boxes_tensor[:, :, :, [0, 2]] /= self.img_width
boxes_tensor[:, :, :, [1, 3]] /= self.img_height
# TODO: Implement box limiting directly for `(cx, cy, w, h)` so that we don't have to unnecessarily convert back and forth.
if self.coords == 'centroids':
# Convert `(xmin, ymin, xmax, ymax)` back to `(cx, cy, w, h)`.
boxes_tensor = convert_coordinates(boxes_tensor,
start_index=0,
conversion='corners2centroids',
border_pixels='half')
elif self.coords == 'minmax':
# Convert `(xmin, ymin, xmax, ymax)` to `(xmin, xmax, ymin, ymax).
boxes_tensor = convert_coordinates(boxes_tensor,
start_index=0,
conversion='corners2minmax',
border_pixels='half')
# Create a tensor to contain the variances and append it to `boxes_tensor`. This tensor has the same shape
# as `boxes_tensor` and simply contains the same 4 variance values for every position in the last axis.
variances_tensor = np.zeros_like(
boxes_tensor
) # Has shape `(feature_map_height, feature_map_width, n_boxes, 4)`
variances_tensor += self.variances # Long live broadcasting
# Now `boxes_tensor` becomes a tensor of shape `(feature_map_height, feature_map_width, n_boxes, 8)`
boxes_tensor = np.concatenate((boxes_tensor, variances_tensor),
axis=-1)
# Now prepend one dimension to `boxes_tensor` to account for the batch size and tile it along
# The result will be a 5D tensor of shape `(batch_size, feature_map_height, feature_map_width, n_boxes, 8)`
boxes_tensor = np.expand_dims(boxes_tensor, axis=0)
boxes_tensor = K.tile(K.constant(boxes_tensor, dtype='float32'),
(K.shape(x)[0], 1, 1, 1, 1))
return boxes_tensor
def loss(self, x: np.ndarray, y: np.ndarray, reduction: str = "none", **kwargs) -> np.ndarray:
"""
Compute the loss of the neural network for samples `x`.
:param x: Samples of shape (nb_samples, nb_features) or (nb_samples, nb_pixels_1, nb_pixels_2,
nb_channels) or (nb_samples, nb_channels, nb_pixels_1, nb_pixels_2).
:param y: Target values (class labels) one-hot-encoded of shape `(nb_samples, nb_classes)` or indices
of shape `(nb_samples,)`.
:param reduction: Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'.
'none': no reduction will be applied
'mean': the sum of the output will be divided by the number of elements in the output,
'sum': the output will be summed.
:return: Loss values.
:rtype: Format as expected by the `model`
"""
if not self._losses:
raise NotImplementedError("loss method is only supported for keras versions >= 2.3.1")
if self.is_tensorflow:
import tensorflow.keras.backend as k
else:
import keras.backend as k
x_preprocessed, y_preprocessed = self._apply_preprocessing(x, y, fit=False)
shape_match = [i is None or i == j for i, j in zip(self._input_shape, x_preprocessed.shape[1:])]
if not all(shape_match):
raise ValueError(
"Error when checking x: expected preprocessed x to have shape {} but got array with "
"shape {}.".format(self._input_shape, x_preprocessed.shape[1:])
)
# Adjust the shape of y for loss functions that do not take labels in one-hot encoding
if self._reduce_labels:
y_preprocessed = np.argmax(y_preprocessed, axis=1)
predictions = self._model.predict(x_preprocessed)
if self._orig_loss and hasattr(self._orig_loss, "reduction"):
prev_reduction = self._orig_loss.reduction
self._orig_loss.reduction = self._losses.Reduction.NONE
loss = self._orig_loss(y_preprocessed, predictions)
self._orig_loss.reduction = prev_reduction
else:
prev_reduction = []
predictions = k.constant(predictions)
y_preprocessed = k.constant(y_preprocessed)
for loss_function in self._model.loss_functions:
prev_reduction.append(loss_function.reduction)
loss_function.reduction = self._losses.Reduction.NONE
loss = self._loss_function(y_preprocessed, predictions)
for i, loss_function in enumerate(self._model.loss_functions):
loss_function.reduction = prev_reduction[i]
loss_value = k.eval(loss)
if reduction == "none":
pass
elif reduction == "mean":
loss_value = np.mean(loss_value, axis=0)
elif reduction == "sum":
loss_value = np.sum(loss_value, axis=0)
return loss_value
def call(self, x):
t = K.constant([0, 1], dtype="float32")
#return tf.contrib.integrate.odeint(self.ode_func, x, t, rtol=1e-3, atol=1e-3)[1] #for tensorflow 1.x
return tfs.integrate.odeint(self.ode_func, x, t, rtol=1e-3,
atol=1e-3)[1]
def reconstruct_content(content: np.ndarray,
layer_name: str = 'block4_conv2',
optimize: Callable = L_BFGS(),
iterations: int = 10,
noise_range: tuple = (0, 1),
callback: Callable = None):
"""
Reconstruct the given content image at the given VGG19 layer.
Args:
content: the content image to reconstruct
layer_name: the layer to reconstruct the content from
optimize: the optimization method for minimizing the content loss
iterations: the number of iterations to run the optimizer
noise_range: the range of values for initializing random noise
callback: the callback for iterations of gradient descent
Returns:
the reconstructed content image based on the VGG19 response
at the given layer name
"""
# disable eager mode for this operation
tf.compat.v1.disable_eager_execution()
# normalize the image's RGB values about the RGB channel means for the
# ImageNet dataset
content = normalize(content[None, ...].astype('float'))
# load the content image into Keras as a constant, it never changes
content = K.constant(content, name='Content')
# create a placeholder for the trained image, this variable trains
canvas = K.placeholder(content.shape, name='Canvas')
# combine the content and canvas tensors along the frame axis (0) into a
# 4D tensor of shape [2, height, width, channels]
input_tensor = K.concatenate([content, canvas], axis=0)
# build the model with the 4D input tensor of content and canvas
model = VGG19(include_top=False, input_tensor=input_tensor, pooling='avg')
# extract the layer's out that we have interest in for reconstruction
layer = model.get_layer(layer_name)
# calculate the loss between the output of the layer on the content (0)
# and the canvas (1)
loss = content_loss(layer.output[0], layer.output[1])
# calculate the gradients
grads = K.gradients(loss, canvas)[0]
# generate the iteration function for gradient descent optimization
step = K.function([canvas], [loss, grads])
# generate random noise with the given noise range
noise = np.random.uniform(*noise_range, size=content.shape)
# optimize the white noise to reconstruct the content
image = optimize(noise, canvas.shape, step, iterations, callback)
# clear the Keras session
K.clear_session()
# de-normalize the image (from ImageNet means) and convert back to binary
return np.clip(denormalize(image.reshape(canvas.shape)[0]), 0,
255).astype('uint8')
def __call__(self, shape, dtype=None):
return K.constant(self.value / shape[0], shape=shape, dtype=dtype)
def __init__(self, epsilon):
self.K_epsilon = K.constant(epsilon)
super(BatchNormLayer, self).__init__()
import tensorflow.keras.backend as K x = img1 y = img1_y x = K.constant(x) x = np.reshape(x, ((1, ) + x.shape)) y = np.reshape(y, ((1, ) + y.shape)) y = np.float32(y) y = K.constant(y) demo.compile(optimizer="adam", loss=loss_region, metrics=["accuracy"]) print(x.shape) print(y.shape) demo.fit(x, y, batch_size=1, epochs=1, verbose=True)
def _construct_inference_tensors(*,
restored_model,
num_of_anchors,
anchors,
model_image_width,
model_image_height,
prob_detection_threshold=0.25,
nms_iou_threshold=0.5):
"""
Constructs input tensors (placeholders) and output tensors that are used for inference.
:param restored_model Keras model restored from the Darknet
:param num_of_anchors number of anchors used in the architecture
:param anchors anchors used in the architecture (expected shape=(num_of_anchors, 2), first dimension is width)
:param model_image_width width of the image used by model (needs to be divisible by 32)
:param model_image_height height of the image used by model (needs to be divisible by 32)
:param model_image_height height of the image used by model (needs to be divisible by 32)
:param prob_detection_threshold threshold for detecting object
:param nms_iou_threshold threshold for non-max suppresion
:return (out_tensors, input_tensors)
- out_tensors - (picked_boxes, picked_classes, picked_scores)
- picked_boxes = Tensor of (left, top, bottom, right)
- picked_classes = Tensor of ints
- picked_score = Tensor of floats
- input_tensors = (model_input, orig_image_width, orig_image_height)
- orig_image_width - Placeholder for original image width (before resizing)
- orig_image_height - Placeholder for original image height (before resizing)
- model_input - Placeholder for image pixels
"""
start = time.time()
boxes = []
prob_class = []
placeholder_orig_image_width = K.placeholder(shape=(1, ))
placeholder_orig_image_height = K.placeholder(shape=(1, ))
for yolo_head_idx in range(len(restored_model.output)):
yolo_head = restored_model.output[yolo_head_idx]
yolo_head_shape = K.shape(yolo_head)
yolo_head_num_of_cols, yolo_head_num_of_rows = yolo_head_shape[
2], yolo_head_shape[1]
curr_yolo_head = K.reshape(yolo_head, [
-1, yolo_head_num_of_cols, yolo_head_num_of_rows, num_of_anchors,
NUM_OF_BOX_PARAMS + NUM_OF_CLASSES
])
grid = construct_grid(yolo_head_shape[1], yolo_head_shape[2])
grid = K.cast(grid, dtype=K.dtype(curr_yolo_head))
grid_size = K.cast([yolo_head_num_of_cols, yolo_head_num_of_rows],
dtype=K.dtype(curr_yolo_head))
curr_boxes_xy = (K.sigmoid(curr_yolo_head[..., :2]) + grid) / grid_size
curr_boxes_wh = K.exp(curr_yolo_head[...,
2:4]) * anchors[yolo_head_idx]
curr_prob_obj = K.sigmoid(curr_yolo_head[..., 4:5])
curr_prob_class = K.sigmoid(curr_yolo_head[..., 5:])
curr_prob_detected_class = curr_prob_obj * curr_prob_class
boxes.append(
get_corrected_boxes(
box_width=curr_boxes_wh[..., 0:1],
box_height=curr_boxes_wh[..., 1:2],
box_x=curr_boxes_xy[..., 0:1],
box_y=curr_boxes_xy[..., 1:2],
orig_image_shape=(placeholder_orig_image_width,
placeholder_orig_image_height),
model_image_shape=(model_image_width, model_image_height)))
curr_prob_detected_class = K.reshape(curr_prob_detected_class,
[-1, NUM_OF_CLASSES])
prob_class.append(curr_prob_detected_class)
prob_class = K.concatenate(prob_class, axis=0)
boxes = K.concatenate(boxes, axis=0)
mask = prob_class >= prob_detection_threshold
max_boxes_tensor = K.constant(20, dtype='int32')
picked_boxes = []
picked_scores = []
picked_classes = []
for c in range(NUM_OF_CLASSES):
class_boxes = tf.boolean_mask(boxes, mask[:, c])
class_box_scores = tf.boolean_mask(prob_class[:, c], mask[:, c])
nms_index = tf.image.non_max_suppression(
class_boxes,
class_box_scores,
max_boxes_tensor,
iou_threshold=nms_iou_threshold)
class_boxes = K.gather(class_boxes, nms_index)
class_box_scores = K.gather(class_box_scores, nms_index)
classes = K.ones_like(class_box_scores, 'int32') * c
picked_boxes.append(class_boxes)
picked_scores.append(class_box_scores)
picked_classes.append(classes)
picked_boxes = K.concatenate(picked_boxes, axis=0)
picked_scores = K.concatenate(picked_scores, axis=0)
picked_classes = K.concatenate(picked_classes, axis=0)
out_tensors = [picked_boxes, picked_scores, picked_classes]
print(f'Took {time.time() - start} seconds to construct network.')
input_tensors = [
restored_model.input, placeholder_orig_image_width,
placeholder_orig_image_height
]
return out_tensors, input_tensors
def std_noise(dummy_input):
batch_shape = K.shape(dummy_input)[:1]
full_shape = K.constant(shape, shape=(len(shape), ), dtype=np.int32)
full_shape = K.concatenate([batch_shape, full_shape])
return K.random_normal(full_shape, 0, 1)
def yolo3_postprocess(args,
anchors,
num_classes,
max_boxes=100,
confidence=0.1,
iou_threshold=0.4,
elim_grid_sense=False):
"""Postprocess for YOLOv3 model on given input and return filtered boxes."""
num_layers = len(anchors) // 3 # default setting
yolo_outputs = args[:num_layers]
image_shape = args[num_layers]
# here we sort the prediction tensor list with grid size (e.g. 19/38/76)
# to make sure it matches with anchors order
yolo_outputs.sort(key=lambda x: x.shape[1])
if num_layers == 3:
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]]
scale_x_y = [1.05, 1.1, 1.2] if elim_grid_sense else [None, None, None]
else:
anchor_mask = [[3, 4, 5], [0, 1, 2]]
scale_x_y = [1.05, 1.05] if elim_grid_sense else [None, None]
input_shape = K.shape(yolo_outputs[0])[1:3] * 32
boxes = []
box_scores = []
for l in range(num_layers):
_boxes, _box_scores = yolo3_boxes_and_scores(yolo_outputs[l],
anchors[anchor_mask[l]],
num_classes,
input_shape,
image_shape,
scale_x_y=scale_x_y[l])
boxes.append(_boxes)
box_scores.append(_box_scores)
boxes = K.concatenate(boxes, axis=0)
box_scores = K.concatenate(box_scores, axis=0)
mask = box_scores >= confidence
max_boxes_tensor = K.constant(max_boxes, dtype='int32')
boxes_ = []
scores_ = []
classes_ = []
for c in range(num_classes):
# TODO: use keras backend instead of tf.
class_boxes = tf.boolean_mask(boxes, mask[:, c])
class_box_scores = tf.boolean_mask(box_scores[:, c], mask[:, c])
nms_index = tf.image.non_max_suppression(class_boxes,
class_box_scores,
max_boxes_tensor,
iou_threshold=iou_threshold)
class_boxes = K.gather(class_boxes, nms_index)
class_box_scores = K.gather(class_box_scores, nms_index)
classes = K.ones_like(class_box_scores, 'int32') * c
boxes_.append(class_boxes)
scores_.append(class_box_scores)
classes_.append(classes)
boxes_ = K.concatenate(boxes_, axis=0)
scores_ = K.concatenate(scores_, axis=0)
classes_ = K.concatenate(classes_, axis=0)
return boxes_, scores_, classes_
