def euclidean_distance(y_true, y_pred):
return K.sqrt(K.sum(K.square(y_pred - y_true), axis=-1))
def normalize(x):
# utility function to normalize a tensor by its L2 norm
return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)
def custom_loss_rmse(y_true, y_pred):
loss = K.sqrt(K.mean(
K.square(y_pred - y_true),
axis=None)) #+K.sum(0*K.abs(penalty)) #can adjust the penalty weight
return loss
help_ = "Load tf model trained weights"
parser.add_argument("-w", "--weights", help=help_)
help_ = "Use mse loss instead of binary cross entropy (default)"
parser.add_argument("-m", "--mse", help=help_, action='store_true')
args = parser.parse_args()
models = (encoder, decoder)
data = (x_test, y_test)
# VAE loss = mse_loss or xent_loss + kl_loss
if args.mse:
reconstruction_loss = mse(inputs, outputs)
else:
reconstruction_loss = binary_crossentropy(inputs, outputs)
reconstruction_loss *= original_dim
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
vae.add_loss(vae_loss)
vae.compile(optimizer='adam')
vae.summary()
plot_model(vae, to_file='vae_mlp.png', show_shapes=True)
save_dir = "vae_mlp_weights"
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
if args.weights:
filepath = os.path.join(save_dir, args.weights)
vae = vae.load_weights(filepath)
else:
def call(self,inputs):
(target,wrt) = inputs
grad = tf.gradients(target,wrt)[0]
return K.sqrt(tf.math.reduce_sum(K.batch_flatten(K.square(grad)),axis =1, keepdims = True))-1
def r2_score(y_true, y_pred):
from tensorflow.keras import backend as K
SS_res = K.sum(K.square(y_true - y_pred))
SS_tot = K.sum(K.square(y_true - K.mean(y_true)))
return (1 - SS_res / (SS_tot + K.epsilon()))
def call(self, y_true, y_pred):
"""
Computation of the YOLO loss.
Loss computation is adopted from
https://github.com/ecaradec/humble-yolo/blob/master/main.py
and modified.
:param y_true: Tensor dim=(batch, grid_rows, grid_cols,
5 + n_categories), ground truth labels for each sample in
the batch.
:param y_pred: Tensor dim=(batch, grid_rows, grid_cols,
5 + n_categories), predicted labels for each sample in the
batch.
:return: Tensor dim=(), total YOLO loss (xy loss + wh loss +
confidence loss + classification loss).
"""
# Extract needed values
pred_boxes = K.reshape(
x=y_pred[..., :5],
shape=(-1, self.grid_rows * self.grid_cols, 5)
)
true_boxes = K.reshape(
x=y_true[..., :5],
shape=(-1, self.grid_rows * self.grid_cols, 5)
)
y_true_conf = true_boxes[..., 0]
y_true_xy = true_boxes[..., 1:3]
y_true_wh = true_boxes[..., 3:] * (self.grid_cols, self.grid_rows)
y_pred_conf = pred_boxes[..., 0]
y_pred_xy = pred_boxes[..., 1:3]
y_pred_wh = pred_boxes[..., 3:] * (self.grid_cols, self.grid_rows)
# XY loss
xy_loss = self.l_coord * K.sum(
K.sum(
K.square(y_pred_xy - y_true_xy),
axis=-1
)
* y_true_conf,
axis=-1
)
# WH loss
wh_loss = self.l_coord * K.sum(
K.sum(
K.square(K.sqrt(y_pred_wh + K.epsilon()) - K.sqrt(y_true_wh + K.epsilon())),
axis=-1
)
* y_true_conf,
axis=-1
)
# Confidence (IoU) loss
intersect_wh = K.maximum(
K.zeros_like(y_pred_wh),
(y_pred_wh + y_true_wh) / 2 - K.abs(y_pred_xy - y_true_xy)
)
intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1]
true_area = y_true_wh[..., 0] * y_true_wh[..., 1]
pred_area = y_pred_wh[..., 0] * y_pred_wh[..., 1]
union_area = pred_area + true_area - intersect_area
iou = tf.math.divide_no_nan(intersect_area, union_area)
# TODO: conf loss could be wrong
# conf_loss1 = K.sum(
# K.square(y_pred_conf - iou) * y_true_conf,
# axis=-1
# )
# aux = tf.cast(iou < 0.5, dtype='float32') * (1 - y_true_conf)
# # tf.print(aux)
# conf_loss2 = self.l_noobj * K.sum(
# K.square(y_pred_conf - iou) * aux,
# # K.square(y_pred_conf - iou) * (1 - y_true_conf),
# axis=-1
# )
conf_loss1 = K.sum(
K.square(y_pred_conf - iou * y_true_conf) * y_true_conf,
axis=-1
)
conf_loss2 = self.l_noobj * K.sum(
K.square(y_pred_conf - iou * y_true_conf) * K.abs(1 - y_true_conf),
axis=-1
)
conf_loss = conf_loss1 + conf_loss2
loss_sum = xy_loss + wh_loss + conf_loss
return tf.math.reduce_mean(loss_sum)
def loss(y_true, y_pred):
return sse(y_true, y_pred) - K.sum(K.square(z), axis=-1)
def MSE_scaled(
y_in,
y_out,
):
return mean(square(y_in - y_out) / square(y_err))
moving_mean_initializer='zeros',
moving_variance_initializer='ones',
beta_regularizer=None,
gamma_regularizer=None,
beta_constraint=None,
gamma_constraint=None)(x)
x = Conv2D(64, (3, 3), activation='relu')(x)
x = AveragePooling2D((2, 2))(x)
x = Conv2D(128, (3, 3), activation='relu')(x)
x = Conv2D(128, (3, 3), activation='relu')(x)
x = Reshape((-1, 128))(x)
x = Capsule(32, 8, 3, True)(x)
x = Capsule(32, 8, 3, True)(x)
capsule = Capsule(5, 16, 3, True)(x)
output = Lambda(lambda z: K.sqrt(K.sum(K.square(z), 2)))(capsule)
model = Model(inputs=[input_image], outputs=[output])
adam = optimizers.Adam(lr=0.001)
model.compile(loss=margin_loss, optimizer=adam, metrics=['accuracy'])
model.summary()
model.fit([x_train], [y_train],
batch_size=batch_size,
epochs=epochs,
shuffle=True)
model.save_weights('pre-train.h5')
def r2(y_true, y_pred):
SS_res = K.sum(K.square(y_true - y_pred))
SS_tot = K.sum(K.square(y_true - K.mean(y_true)))
return (1 - SS_res / (SS_tot + K.epsilon()))
def coeff_determination(y_pred,
y_true): #Order of function inputs is important here
SS_res = K.sum(K.square(y_true - y_pred))
SS_tot = K.sum(K.square(y_true - K.mean(y_true)))
return (1 - SS_res / (SS_tot + K.epsilon()))
def train_step(self,
images,
style,
noise,
perform_gp=True,
perform_pl=False):
with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:
#Get style information
w_space = []
pl_lengths = self.pl_mean
for i in range(len(style)):
w_space.append(self.GAN.S(style[i]))
#Generate images
generated_images = self.GAN.G(w_space + [noise])
#Discriminate
real_output = self.GAN.D(images, training=True)
fake_output = self.GAN.D(generated_images, training=True)
#Hinge loss function
gen_loss = K.mean(fake_output)
divergence = K.mean(
K.relu(1 + real_output) + K.relu(1 - fake_output))
disc_loss = divergence
if perform_gp:
#R1 gradient penalty
disc_loss += gradient_penalty(images, real_output, 10)
if perform_pl:
#Slightly adjust W space
w_space_2 = []
for i in range(len(style)):
std = 0.1 / (K.std(w_space[i], axis=0, keepdims=True) +
1e-8)
w_space_2.append(w_space[i] +
K.random_normal(tf.shape(w_space[i])) /
(std + 1e-8))
#Generate from slightly adjusted W space
pl_images = self.GAN.G(w_space_2 + [noise])
#Get distance after adjustment (path length)
delta_g = K.mean(K.square(pl_images - generated_images),
axis=[1, 2, 3])
pl_lengths = delta_g
if self.pl_mean > 0:
gen_loss += K.mean(K.square(pl_lengths - self.pl_mean))
#Get gradients for respective areas
gradients_of_generator = gen_tape.gradient(
gen_loss, self.GAN.GM.trainable_variables)
gradients_of_discriminator = disc_tape.gradient(
disc_loss, self.GAN.D.trainable_variables)
#Apply gradients
self.GAN.GMO.apply_gradients(
zip(gradients_of_generator, self.GAN.GM.trainable_variables))
self.GAN.DMO.apply_gradients(
zip(gradients_of_discriminator, self.GAN.D.trainable_variables))
return disc_loss, gen_loss, divergence, pl_lengths
def main():
# Create a VideoCapture object
batch_shape = style_img.shape
shape = style_img.shape[1:]
cap = cv2.VideoCapture('./input_video.avi')
#frame_width = int(cap.get(3))
#frame_height = int(cap.get(4))
# Check if camera opened successfully
if (cap.isOpened() == False):
print("Unable to read camera feed")
# Default resolutions of the frame are obtained.The default resolutions are system dependent.
# We convert the resolutions from float to integer.
# Define the codec and create VideoWriter object.The output is stored in 'outpy.avi' file.
count = 0
#fourcc = cv2.VideoWriter_fourcc('M','J','P','G')
#out = cv2.VideoWriter('outpy.avi',fourcc,20.0,(224,224))
while (cap.isOpened()):
ret, frame = cap.read()
if ret is False:
break
print(str(count), 'th frame processing')
frame = cv2.resize(frame, (224, 224))
X = preprocess_img(frame)
vgg = vgg_avg_pooling(shape=shape)
content_model = Model(vgg.input, vgg.layers[13].get_output_at(0))
content_target = content_model.predict(X)
symb_conv_outputs = [layer.get_output_at(1) for layer in \
vgg.layers if layer.name.endswith('conv1')]
multi_output_model = Model(vgg.input, symb_conv_outputs)
symb_layer_out = [
K.variable(y) for y in multi_output_model.predict(style_img)
]
weights = [0.2, 0.4, 0.3, 0.5, 0.2]
loss = K.sum(K.square(content_model.output - content_target))
for symb, actual, w in zip(symb_conv_outputs, symb_layer_out, weights):
loss += 0.03 * w * style_loss(symb[0], actual[0])
loss += 0.1 * total_variation_loss(symb_conv_outputs[-1])
grad = K.gradients(loss, vgg.input)
get_loss_grad = K.function(inputs=[vgg.input], outputs=[loss] + grad)
def get_loss_grad_wrapper(x_vec):
l, g = get_loss_grad([x_vec.reshape(*batch_shape)])
return l.astype(np.float64), g.flatten().astype(np.float64)
final_img = min_loss(fn=get_loss_grad_wrapper,
epochs=100,
batch_shape=batch_shape)
#plt.imshow(scale(final_img))
#plt.show()
filename = "./style_images_test/frame%d.jpg" % count
count += 1
cv2.imwrite(filename, final_img)
cv2.destroyAllWindows()
#Converting images to video clipping
image_folder = './style_images_test'
video_name = 'output_videos/output_video_test.avi'
images = [img for img in os.listdir(image_folder) if img.endswith(".jpg")]
frame = cv2.imread(os.path.join(image_folder, images[0]))
height, width, _ = frame.shape
fourcc = cv2.VideoWriter_fourcc('M', 'J', 'P', 'G')
video = cv2.VideoWriter(video_name, fourcc, count + 1, (width, height))
for image in images:
video.write(cv2.imread(os.path.join(image_folder, image)))
cv2.destroyAllWindows()
video.release()
def yolo_loss(args, anchors, num_classes, ignore_thresh=.5, print_loss=False):
'''Return yolo_loss tensor
Parameters
----------
yolo_outputs: list of tensor, the output of yolo_body or tiny_yolo_body
y_true: list of array, the output of preprocess_true_boxes
anchors: array, shape=(N, 2), wh
num_classes: integer
ignore_thresh: float, the iou threshold whether to ignore object confidence loss
Returns
-------
loss: tensor, shape=(1,)
'''
num_layers = len(anchors) // 3 # default setting
yolo_outputs = args[:num_layers]
y_true = args[num_layers:]
anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]
] if num_layers == 3 else [[3, 4, 5], [1, 2, 3]]
input_shape = K.cast(
K.shape(yolo_outputs[0])[1:3] * 32, K.dtype(y_true[0]))
grid_shapes = [
K.cast(K.shape(yolo_outputs[l])[1:3], K.dtype(y_true[0]))
for l in range(num_layers)
]
loss = 0
m = K.shape(yolo_outputs[0])[0] # batch size, tensor
mf = K.cast(m, K.dtype(yolo_outputs[0]))
for l in range(num_layers):
object_mask = y_true[l][..., 4:5]
true_class_probs = y_true[l][..., 5:]
grid, raw_pred, pred_xy, pred_wh = yolo_head(yolo_outputs[l],
anchors[anchor_mask[l]],
num_classes,
input_shape,
calc_loss=True)
pred_box = K.concatenate([pred_xy, pred_wh])
# Darknet raw box to calculate loss.
raw_true_xy = y_true[l][..., :2] * grid_shapes[l][::-1] - grid
raw_true_wh = K.log(y_true[l][..., 2:4] / anchors[anchor_mask[l]] *
input_shape[::-1])
raw_true_wh = K.switch(object_mask, raw_true_wh,
K.zeros_like(raw_true_wh)) # avoid log(0)=-inf
box_loss_scale = 2 - y_true[l][..., 2:3] * y_true[l][..., 3:4]
# Find ignore mask, iterate over each of batch.
ignore_mask = tf.TensorArray(K.dtype(y_true[0]),
size=1,
dynamic_size=True)
object_mask_bool = K.cast(object_mask, 'bool')
def loop_body(b, ignore_mask):
true_box = tf.boolean_mask(y_true[l][b, ..., 0:4],
object_mask_bool[b, ..., 0])
iou = box_iou(pred_box[b], true_box)
best_iou = K.max(iou, axis=-1)
ignore_mask = ignore_mask.write(
b, K.cast(best_iou < ignore_thresh, K.dtype(true_box)))
return b + 1, ignore_mask
_, ignore_mask = tf.while_loop(lambda b, *args: b < m, loop_body,
[0, ignore_mask])
ignore_mask = ignore_mask.stack()
ignore_mask = K.expand_dims(ignore_mask, -1)
# K.binary_crossentropy is helpful to avoid exp overflow.
xy_loss = object_mask * box_loss_scale * K.binary_crossentropy(
raw_true_xy, raw_pred[..., 0:2], from_logits=True)
wh_loss = object_mask * box_loss_scale * 0.5 * K.square(
raw_true_wh - raw_pred[..., 2:4])
confidence_loss = object_mask * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True)+ \
(1-object_mask) * K.binary_crossentropy(object_mask, raw_pred[...,4:5], from_logits=True) * ignore_mask
class_loss = object_mask * K.binary_crossentropy(
true_class_probs, raw_pred[..., 5:], from_logits=True)
xy_loss = K.sum(xy_loss) / mf
wh_loss = K.sum(wh_loss) / mf
confidence_loss = K.sum(confidence_loss) / mf
class_loss = K.sum(class_loss) / mf
loss += xy_loss + wh_loss + confidence_loss + class_loss
if print_loss:
loss = tf.print(loss, [
loss, xy_loss, wh_loss, confidence_loss, class_loss,
K.sum(ignore_mask)
],
message='loss: ')
return loss
def coeff_determination(y_true, y_pred):
SS_res = K.sum(K.square(y_true - y_pred))
SS_tot = K.sum(K.square(y_true - K.mean(y_true)))
return (1 - SS_res / (SS_tot + K.epsilon()))
def euclidean_distance(self, inputs):
u, v = inputs
return K.sqrt(
K.maximum(K.sum(K.square(u - v), axis=1, keepdims=True),
K.epsilon()))
def sse(y_true, y_pred):
return K.sum(K.square(y_pred - y_true), axis=-1)
def loss(y, d):
positive = (1 - y) * K.square(d)
negative = y * K.square(K.maximum(margin - d, 0.))
# K.print_tensor(positive, message='POSITVE')
# K.print_tensor(negative, message='NEGATIVE')
return K.mean(positive + negative)
def margin_loss(y_true, y_pred):
lamb, margin = 0.5, 0.1
return K.sum(y_true * K.square(K.relu(1 - margin - y_pred)) + lamb *
(1 - y_true) * K.square(K.relu(y_pred - margin)),
axis=-1)
def euclidean_distance(vects):
x, y = vects
sum_square = K.sum(K.square(x - y), axis=1, keepdims=True)
return K.sqrt(K.maximum(sum_square, K.epsilon()))
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True) + K.epsilon()
scale = K.sqrt(s_squared_norm) / (0.5 + s_squared_norm)
return scale * x
def test(tensors):
x, y = tensors
sum_square = K.sum(K.square(x - y), axis=1, keepdims=True)
return K.sqrt(K.maximum(sum_square, K.epsilon()))
def PSNR(y_true, y_pred):
# print(K.mean(K.square(y_pred - y_true)))
# exit()
max_pixel = 1.0
return 10.0 * (1.0 / math.log(10)) * K.log(
(max_pixel**2) / (K.mean(K.square(y_pred - y_true))))
def PSNR(y_true, y_pred):
# 参考：https://ja.wikipedia.org/wiki/%E3%83%94%E3%83%BC%E3%82%AF%E4%BF%A1%E5%8F%B7%E5%AF%BE%E9%9B%91%E9%9F%B3%E6%AF%94
pic_gt = y_true[:, :, :, :3]
pic_pred = y_pred[:, :, :, :3]
return 20 * K.log(2.0) / K.log(10.0) - 10.0 * K.log(
K.mean(K.square(pic_gt - pic_pred), axis=(1, 2, 3))) / K.log(10.0)
def _sum_square(x):
return K.sum(K.square(x), axis=-1)
def normalize(x):
"""Utility function to normalize a tensor by its L2 norm"""
return (x + 1e-10) / (K.sqrt(K.mean(K.square(x))) + 1e-10)
def msle_holes(y_true, y_pred):
idxs = tf.where(tf.math.logical_not(tf.math.is_nan(y_true)))
y_true = tf.gather_nd(y_true, idxs)
y_pred = tf.gather_nd(y_pred, idxs)
return K.mean(K.square(K.log(1 + y_true) - K.log(1 + y_pred)), axis=-1)
def CLOSS(y_true, y_pred):
'''
MAE style content loss
'''
return K.mean(K.square(loss_models[0](y_true) - loss_models[0](y_pred)))+\
K.mean(K.square(y_true - y_pred))
def rmse(y_true, y_pred): import tensorflow.keras.backend as K return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1))
def loss(y_true: tf.Tensor, y_pred: tf.Tensor) \
-> tf.Tensor:
return K.mean(K.square(y_pred - y_true))
