def class_loss_regr_fixed_num(y_true, y_pred):
x = y_true[:, :, 4 * num_classes:] - y_pred
x_abs = K.abs(x)
x_bool = K.cast(K.less_equal(x_abs, 1.0), 'float32')
return lambda_cls_regr * K.sum(
y_true[:, :, :4 * num_classes] *
(x_bool * (0.5 * x * x) + (1 - x_bool) *
(x_abs - 0.5))) / K.sum(epsilon + y_true[:, :, :4 * num_classes])
def rpn_loss_regr_fixed_num(y_true, y_pred):
x = y_true[:, :, :, 4 * num_anchors:] - y_pred
x_abs = K.abs(x)
x_bool = K.cast(K.less_equal(x_abs, 1.0), tf.float32)
return lambda_rpn_regr * K.sum(
y_true[:, :, :, :4 * num_anchors] *
(x_bool * (0.5 * x * x) + (1 - x_bool) *
(x_abs - 0.5))) / K.sum(epsilon +
y_true[:, :, :, :4 * num_anchors])
def get_initial_states(self, x):
input_shape = self.input_spec[0].shape
init_nb_row = input_shape[self.row_axis]
init_nb_col = input_shape[self.column_axis]
base_initial_state = K.zeros_like(
x) # (batch_samples, timesteps) + image_shape
non_channel_axis = -2
for _ in range(2):
base_initial_state = K.sum(base_initial_state,
axis=non_channel_axis)
base_initial_state = K.sum(base_initial_state,
axis=1) # (samples, nb_channels)
initial_states = []
states_to_pass = ['r', 'c', 'e']
nlayers_to_pass = {u: self.nb_layers for u in states_to_pass}
if self.extrap_start_time is not None:
# pass prediction in states so can use as actual for t+1 when extrapolating
states_to_pass.append('ahat')
nlayers_to_pass['ahat'] = 1
for u in states_to_pass: # ['r', 'c', 'e'] is the state
for l in range(
nlayers_to_pass[u]): # initialize all the state with zero
ds_factor = 2**l # why downsampling?
nb_row = init_nb_row // ds_factor
nb_col = init_nb_col // ds_factor
if u in ['r', 'c']:
stack_size = self.R_stack_sizes[l]
elif u == 'e':
stack_size = 2 * self.stack_sizes[l]
elif u == 'ahat':
stack_size = self.stack_sizes[l]
output_size = nb_row * nb_col * stack_size # flattened size
reducer = K.zeros((input_shape[self.channel_axis],
output_size)) # (nb_channels, output_size)
initial_state = K.dot(base_initial_state,
reducer) # (samples, output_size)
output_shp = [-1, nb_row, nb_col, stack_size]
initial_state = K.reshape(initial_state, output_shp)
initial_states += [initial_state]
if self.extrap_start_time is not None:
initial_states += [
K.variable(0, 'int32')
] # the last state will correspond to the current timestep
return initial_states
def total_variation_loss(x, target_size):
img_height, img_width = target_size
a = K.square(
x[:, :img_height - 1, :img_width - 1, :] - x[:, 1:, :img_width - 1, :])
b = K.square(
x[:, :img_height - 1, :img_width - 1, :] - x[:, :img_height - 1, 1:, :])
return K.sum(K.pow(a + b, 1.25))
def style_loss(style, combination):
S = gram_matrix(style)
C = gram_matrix(combination)
channels = 3
size = height * width
return backend.sum(backend.square(S - C)) / (4. * (channels**2) *
(size**2))
def euclidean_distance(vects):
'''
Auxiliary function to compute the Euclidian distance between two vectors
in a Keras layer.
'''
x, y = vects
return K.sqrt(K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True), K.epsilon()))
def build_loss(self):
r"""Implements the N-dim version of function
$$TV^{\beta}(x) = \sum_{whc} \left ( \left ( x(h, w+1, c) - x(h, w, c) \right )^{2} +
\left ( x(h+1, w, c) - x(h, w, c) \right )^{2} \right )^{\frac{\beta}{2}}$$
to return total variation for all images in the batch.
"""
image_dims = K.ndim(self.img) - 2
# Constructing slice [1:] + [:-1] * (image_dims - 1) and [:-1] * (image_dims)
start_slice = [slice(1, None, None)] + [
slice(None, -1, None) for _ in range(image_dims - 1)
]
end_slice = [slice(None, -1, None) for _ in range(image_dims)]
samples_channels_slice = [
slice(None, None, None),
slice(None, None, None)
]
# Compute pixel diffs by rolling slices to the right per image dim.
tv = None
for i in range(image_dims):
ss = tuple(samples_channels_slice + start_slice)
es = tuple(samples_channels_slice + end_slice)
diff_square = K.square(self.img[utils.slicer[ss]] -
self.img[utils.slicer[es]])
tv = diff_square if tv is None else tv + diff_square
# Roll over to next image dim
start_slice = np.roll(start_slice, 1).tolist()
end_slice = np.roll(end_slice, 1).tolist()
tv = K.sum(K.pow(tv, self.beta / 2.))
return normalize(self.img, tv)
def gradient_penalty(y_true, y_pred, interpolate, lamb):
grad = K.gradients(y_pred, interpolate)[0]
norm = K.square(grad)
norm_sum = K.sum(norm,axis=np.arange(1,len(norm.shape)))
l2_norm = K.sqrt(norm_sum)
gp_reg = lamb*K.square(1-l2_norm)
return K.mean(gp_reg)
def style_loss(style, combination, target_size):
s = gram_matrix(style)
c = gram_matrix(combination)
channels = 3
img_height, img_width = target_size
size = img_height * img_width
return K.sum(K.square(s - c)) / (4.0 * (channels ** 2) * (size ** 2))
def disparityregression(input):
shape = K.shape(input)
disparity_values = np.linspace(-4, 4, 9)
x = K.constant(disparity_values, shape=[9])
x = K.expand_dims(K.expand_dims(K.expand_dims(x, 0), 0), 0)
x = tf.tile(x, [shape[0], shape[1], shape[2], 1])
out = K.sum(multiply([input, x]), -1)
return out
def build_loss(self):
# Infinity norm
if np.isinf(self.p):
value = K.max(self.img)
else:
value = K.pow(K.sum(K.pow(K.abs(self.img), self.p)), 1. / self.p)
return normalize(self.img, value)
def jacard_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
y_pred_f = K.flatten(y_pred)
intersection = K.sum(y_true_f * y_pred_f)
return (intersection + 1.0) / (K.sum(y_true_f) + K.sum(y_pred_f) -
intersection + 1.0)
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
y_pred_f = K.flatten(y_pred)
intersection = K.sum(y_true_f * y_pred_f)
return (2.0 * intersection + 1.0) / (K.sum(y_true_f) + K.sum(y_pred_f) +
1.0)
def euclidean_distance(y_true, y_pred):
return K.sqrt(K.sum(K.square(y_true - y_pred), axis=1))
def vae_loss(x, x_decoded_mean):
xent_loss = original_dim * keras.losses.categorical_crossentropy(
x, x_decoded_mean)
kl_loss = -0.5 * K.sum(
1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
return xent_loss + kl_loss
def main(_):
# disable all training specific operations
K.set_learning_phase(0)
model = applications.inception_v3.InceptionV3(weights='imagenet',
include_top=False)
layer_contributions = {
'mixed2': 0.2,
'mixed3': 3.0,
'mixed4': 2.0,
'mixed5': 1.5
}
layer_dict = dict([(layer.name, layer) for layer in model.layers])
loss = K.variable(0.,)
for layer_name in layer_contributions:
coeff = layer_contributions[layer_name]
activation = layer_dict[layer_name].output
scaling = K.prod(K.cast(K.shape(activation), 'float32'))
# avoid artifacts by only involving non-boarder pixels
loss += coeff * K.sum(K.square(activation[:, 2:-2, 2:-2, :])) / scaling
# start the gradient-ascent process
dream = model.input
grads_list = K.gradients(loss, dream)
grads = grads_list[0]
# trick: normalize gradients
grads /= K.maximum(K.mean(K.abs(grads)), 1e-7)
fetch_loss_and_grads = K.function(inputs=[dream],
outputs=[loss, grads])
def gradient_ascent(x, iterations, step_rate, max_loss=None):
for i in range(iterations):
loss_value, grads_value = fetch_loss_and_grads([x])
if max_loss is not None and loss_value > max_loss:
break
print('@{:4d}: {:.4f}'.format(i, loss_value))
x += step_rate * grads_value
return x
img = preprocess_img(FLAGS.img_path)
original_shape = img.shape[1:3]
successive_shapes = [original_shape]
for i in range(1, NUM_OCTAVES):
shape = tuple([int(dim / (OCTAVES_SCLAE ** i))
for dim in original_shape])
successive_shapes.append(shape)
# reverse
successive_shapes = successive_shapes[::-1]
original_img = np.copy(img)
shrunk_original_img = resize_img(img, successive_shapes[0])
for shape in successive_shapes:
print('Preprocess image with shape: {}'.format(shape))
img = resize_img(img, shape)
img = gradient_ascent(img,
iterations=FLAGS.iterations,
step_rate=FLAGS.step_rate,
max_loss=MAX_LOSS)
same_size_original = resize_img(original_img, shape)
if FLAGS.repair_lost_detail:
upscale_shrunk_original_img = resize_img(shrunk_original_img, shape)
lost_detail = same_size_original - upscale_shrunk_original_img
img += lost_detail
shrunk_original_img = same_size_original
save_img(img, filename='dream_at_scale_{}.png'.format(str(shape)))
save_img(img, filename='dream.png')
def content_loss(content, combination):
return backend.sum(backend.square(combination - content))
def total_variation_loss(x):
a = backend.square(x[:, :height - 1, :width - 1, :] -
x[:, 1:, :width - 1, :])
b = backend.square(x[:, :height - 1, :width - 1, :] -
x[:, :height - 1, 1:, :])
return backend.sum(backend.pow(a + b, 1.25))
def rpn_loss_cls_fixed_num(y_true, y_pred):
return lambda_rpn_class * K.sum(
y_true[:, :, :, :num_anchors] * K.binary_crossentropy(
y_pred[:, :, :, :], y_true[:, :, :, num_anchors:])) / K.sum(
epsilon + y_true[:, :, :, :num_anchors])
test_x = []
for row in train_data:
train_x.append(row[0])
for row in test_data:
test_x.append(row[0])
train_x = np.array(train_x).reshape(-1, 128, 128, 3)
test_x = np.array(test_x).reshape(-1, 128, 128, 3)
reconstruction_loss = tf.keras.losses.binary_crossentropy(
K.flatten(input_img), K.flatten(outputs))
reconstruction_loss *= 128 * 128
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
vae.add_loss(vae_loss)
vae.compile(optimizer='rmsprop')
vae.summary()
flag = 1
if flag == 0:
vae.fit(train_x,
epochs=10,
batch_size=64,
validation_data=(test_x, None))
vae.save_weights("vae_cnn.h5")
elif flag == 1:
vae.load_weights('vae_cnn.h5')
def ms_euclidean(y_true, y_pred):
return K.mean(K.sum(K.square(y_true - y_pred), axis=1))
def total_variation_loss(x):
a = K.square(x[:, :height-1, :width-1, :] - x[:, 1:, :width-1, :])
b = K.square(x[:, :height-1, :width-1, :] - x[:, :height-1, 1:, :])
return K.sum(K.pow(a + b, 1.25))
def content_loss(content, combination):
return K.sum(K.square(combination - content))
def content_loss(base, combination):
return K.sum(K.square(combination - base))
