def gradient_penalty(y_true, y_pred, interpolate, lamb):
grad = K.gradients(y_pred, interpolate)[0]
norm = K.square(grad)
norm_sum = K.sum(norm,axis=np.arange(1,len(norm.shape)))
l2_norm = K.sqrt(norm_sum)
gp_reg = lamb*K.square(1-l2_norm)
return K.mean(gp_reg)
def build_model(self):
l2_regularization_kernel = 1e-5
# Input Layer
input = layers.Input(shape=(self.state_size,), name='input_states')
# Hidden Layers
model = layers.Dense(units=300, kernel_regularizer=regularizers.l2(l2_regularization_kernel))(input)
model = layers.BatchNormalization()(model)
model = layers.LeakyReLU(1e-2)(model)
model = layers.Dense(units=400, kernel_regularizer=regularizers.l2(l2_regularization_kernel))(model)
model = layers.BatchNormalization()(model)
model = layers.LeakyReLU(1e-2)(model)
model = layers.Dense(units=200, kernel_regularizer=regularizers.l2(l2_regularization_kernel))(model)
model = layers.BatchNormalization()(model)
model = layers.LeakyReLU(1e-2)(model)
# Our output layer - a fully connected layer
output = layers.Dense(units=self.action_size, activation='tanh', kernel_regularizer=regularizers.l2(l2_regularization_kernel),
kernel_initializer=initializers.RandomUniform(minval=-3e-3, maxval=3e-3), name='output_actions')(model)
# Keras model
self.model = models.Model(inputs=input, outputs=output)
# Define loss and optimizer
action_gradients = layers.Input(shape=(self.action_size,))
loss = K.mean(-action_gradients * output)
optimizer = optimizers.Adam(lr=1e-4)
update_operation = optimizer.get_updates(params=self.model.trainable_weights, loss=loss)
self.train_fn = K.function(inputs=[self.model.input, action_gradients, K.learning_phase()],
outputs=[], updates=update_operation)
def build_loss(self):
layer_output = self.layer.output
# For all other layers it is 4
is_dense = K.ndim(layer_output) == 2
loss = 0.
for idx in self.filter_indices:
if is_dense:
loss += -K.mean(layer_output[:, idx])
else:
# slicer is used to deal with `channels_first` or `channels_last` image data formats
# without the ugly conditional statements.
loss += -K.mean(layer_output[utils.slicer[:, idx, ...]])
return loss
def atari_qnet(input_shape, num_actions, net_name, net_size):
net_name = net_name.lower()
# input state
state = Input(shape=input_shape)
# convolutional layers
conv1_32 = Conv2D(32, (8, 8), strides=(4, 4), activation='relu')
conv2_64 = Conv2D(64, (4, 4), strides=(2, 2), activation='relu')
conv3_64 = Conv2D(64, (3, 3), strides=(1, 1), activation='relu')
# if recurrent net then change input shape
if 'drqn' in net_name:
# recurrent net (drqn)
lambda_perm_state = lambda x: K.permute_dimensions(x, [0, 3, 1, 2])
perm_state = Lambda(lambda_perm_state)(state)
dist_state = Lambda(lambda x: K.stack([x], axis=4))(perm_state)
# extract features with `TimeDistributed` wrapped convolutional layers
dist_conv1 = TimeDistributed(conv1_32)(dist_state)
dist_conv2 = TimeDistributed(conv2_64)(dist_conv1)
dist_convf = TimeDistributed(conv3_64)(dist_conv2)
feature = TimeDistributed(Flatten())(dist_convf)
elif 'dqn' in net_name:
# fully connected net (dqn)
# extract features with convolutional layers
conv1 = conv1_32(state)
conv2 = conv2_64(conv1)
convf = conv3_64(conv2)
feature = Flatten()(convf)
# network type. Dense for dqn; LSTM or GRU for drqn
if 'lstm' in net_name:
net_type = LSTM
elif 'gru' in net_name:
net_type = GRU
else:
net_type = Dense
# dueling or regular dqn/drqn
if 'dueling' in net_name:
value1 = net_type(net_size, activation='relu')(feature)
adv1 = net_type(net_size, activation='relu')(feature)
value2 = Dense(1)(value1)
adv2 = Dense(num_actions)(adv1)
mean_adv2 = Lambda(lambda x: K.mean(x, axis=1))(adv2)
ones = K.ones([1, num_actions])
lambda_exp = lambda x: K.dot(K.expand_dims(x, axis=1), -ones)
exp_mean_adv2 = Lambda(lambda_exp)(mean_adv2)
sum_adv = add([exp_mean_adv2, adv2])
exp_value2 = Lambda(lambda x: K.dot(x, ones))(value2)
q_value = add([exp_value2, sum_adv])
else:
hid = net_type(net_size, activation='relu')(feature)
q_value = Dense(num_actions)(hid)
# build model
return Model(inputs=state, outputs=q_value)
def build_model(self):
"""Build an actor (policy) network that maps states -> actions."""
# Define input layer (states)
states = layers.Input(shape=(self.state_size, ), name='states')
'''# Add hidden layers
net = layers.Dense(units=32, activation='relu')(states)
net = layers.Dense(units=64, activation='relu')(net)
net = layers.Dense(units=32, activation='relu')(net)
# Try different layer sizes, activations, add batch normalization, regularizers, etc.
# Add final output layer with sigmoid activation
raw_actions = layers.Dense(units=self.action_size, activation='sigmoid',
name='raw_actions')(net)
'''
###################################
# Add hidden layers
net = layers.Dense(units=400,
kernel_regularizer=regularizers.l2(1e-6))(states)
net = layers.BatchNormalization()(net)
net = layers.LeakyReLU(1e-2)(net)
net = layers.Dense(units=300,
kernel_regularizer=regularizers.l2(1e-6))(net)
net = layers.BatchNormalization()(net)
net = layers.LeakyReLU(1e-2)(net)
# Add final output layer with sigmoid activation
raw_actions = layers.Dense(
units=self.action_size,
activation='sigmoid',
name='raw_actions',
kernel_initializer=initializers.RandomUniform(minval=-0.003,
maxval=0.003))(net)
#######################################
# Scale [0, 1] output for each action dimension to proper range
actions = layers.Lambda(lambda x:
(x * self.action_range) + self.action_low,
name='actions')(raw_actions)
# Create Keras model
self.model = models.Model(inputs=states, outputs=actions)
# Define loss function using action value (Q value) gradients
action_gradients = layers.Input(shape=(self.action_size, ))
loss = K.mean(-action_gradients * actions)
# Incorporate any additional losses here (e.g. from regularizers)
# Define optimizer and training function
optimizer = optimizers.Adam(lr=1e-6)
updates_op = optimizer.get_updates(params=self.model.trainable_weights,
loss=loss)
self.train_fn = K.function(
inputs=[self.model.input, action_gradients,
K.learning_phase()],
outputs=[],
updates=updates_op)
def build_model(self):
"""Build an actor (policy) network that maps states -> actions."""
# Define input layer (states)
states = layers.Input(shape=(self.state_size, ), name='states')
#--------- copy from DDPG quadcopter -----------
net = layers.Dense(units=400)(states)
# net = layers.BatchNormalization()(net)
net = layers.Activation("relu")(net)
net = layers.Dense(units=200)(net)
# net = layers.BatchNormalization()(net)
net = layers.Activation("relu")(net)
actions = layers.Dense(units=self.action_size,
activation='softmax',
name='actions',
kernel_initializer=initializers.RandomUniform(
minval=-1, maxval=1))(net)
# actions = layers.Dense(units=self.action_size, activation='sigmoid', name='actions',
# kernel_initializer=initializers.RandomUniform(minval=-0.001, maxval=0.001))(net)
# Add hidden layers
# net = layers.Dense(units=16,activation=activations.sigmoid)(states)
# net = layers.BatchNormalization()(net)
# net = layers.Dense(units=16,activation=activations.sigmoid)(net)
# net = layers.BatchNormalization()(net)
# net = layers.Dense(units=128,activation=activations.relu)(net)
# net = layers.BatchNormalization()(net)
# Add final output layer with sigmoid activation
# actions = layers.Dense(units=self.action_size, activation='linear', # sigmoid
# name='raw_actions' )(net)
# Scale [0, 1] output for each action dimension to proper range
# actions = layers.Lambda(lambda x: (x * self.action_range) + self.action_low,
# name='actions')(raw_actions)
# Create Keras model
self.model = models.Model(inputs=states, outputs=actions)
action_gradients = layers.Input(shape=(self.action_size, ))
loss = K.mean(-action_gradients * actions)
# Define optimizer and training function
optimizer = optimizers.Adam(lr=.0001)
updates_op = optimizer.get_updates(params=self.model.trainable_weights,
loss=loss)
self.train_fn = K.function(
inputs=[self.model.input, action_gradients,
K.learning_phase()],
outputs=[],
updates=updates_op)
def contrastive_loss(y_true, y_pred):
'''
Contrastive loss from Hadsell-et-al.'06
http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
@param
y_true : true label 1 for positive pair, 0 for negative pair
y_pred : distance output of the Siamese network
'''
margin = 1
# if positive pair, y_true is 1, penalize for large distance returned by Siamese network
# if negative pair, y_true is 0, penalize for distance smaller than the margin
return K.mean(y_true * K.square(y_pred) +
(1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))
def generate_pattern(model, layer_name, filter_index, steps, learning_rate, size=224):
layer_output = model.get_layer(layer_name).output
loss = K.mean(layer_output[:, :, :, filter_index])
# obtain the gradient of the loss with respect to the model's input image
grads_list = K.gradients(loss, model.input)
grads = grads_list[0]
# gradient normalization trick
grads /= (K.sqrt(K.mean(K.square(grads))) + EPSILON)
# fetch loss and normalized-gradients for a given input
iterate = K.function(inputs=[model.input], outputs=[loss, grads])
# loss maximization via stochastic gradient descent
input_img_data = np.random.random((1, size, size, 3)) * 20 + 128 # start from gray image with random noise
for i in range(steps):
loss_value, grads_value = iterate([input_img_data])
print('@{:-4d}: {:.4f}'.format(i, loss_value))
# gradient ascent: adjust the input image in the direction that maximizes the loss
input_img_data += grads_value * learning_rate
img_tensor = input_img_data[0]
return tensor_to_image(img_tensor)
def contrastive_loss(y_true, y_pred):
'''
Contrastive loss from Hadsell-et-al.'06
http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
@param
y_true : true label 1 for positive pair, 0 for negative pair
y_pred : distance output of the Siamese network
@return
contrastive_loss between two values
'''
margin = 1
# if positive pair, y_true is 1, penalize for large distance returned by Siamese network
# if negative pair, y_true is 0, penalize for distance smaller than the margin
y_positive_true = 1
y_negative_true = 0
return K.mean(y_true * K.square(y_pred) +
(y_positive_true - y_true) * K.square(K.maximum(margin - y_pred, y_negative_true)))
def build_model(self):
states = layers.Input(shape=(self.state_size,), name='inputStates')
# Hidden Layers
model = layers.Dense(units=128, activation='linear')(states)
model = layers.BatchNormalization()(model)
model = layers.LeakyReLU(0.01)(model)
model = layers.Dropout(0.3)(model)
model = layers.Dense(units=256, activation='linear')(model)
model = layers.BatchNormalization()(model)
model = layers.LeakyReLU(0.01)(model)
model = layers.Dropout(0.3)(model)
model = layers.Dense(units=512, activation='linear')(model)
model = layers.BatchNormalization()(model)
model = layers.LeakyReLU(0.01)(model)
model = layers.Dropout(0.3)(model)
model = layers.Dense(units=128, activation='linear')(model)
model = layers.BatchNormalization()(model)
model = layers.LeakyReLU(0.01)(model)
model = layers.Dropout(0.3)(model)
output = layers.Dense(
units=self.action_size,
activation='tanh',
kernel_regularizer=regularizers.l2(0.01),
name='outputActions')(model)
#Keras
self.model = models.Model(inputs=states, outputs=output)
#Definint Optimizer
actionGradients = layers.Input(shape=(self.action_size,))
loss = K.mean(-actionGradients * output)
optimizer = optimizers.Adam()
update_operation = optimizer.get_updates(params=self.model.trainable_weights, loss=loss)
self.train_fn = K.function(
inputs=[self.model.input, actionGradients, K.learning_phase()],
outputs=[],
updates=update_operation)
def __init__(self, input_tensor, losses, input_range=(0, 255), wrt_tensor=None, norm_grads=True):
"""Creates an optimizer that minimizes weighted loss function.
Args:
input_tensor: An input tensor of shape: `(samples, channels, image_dims...)` if `image_data_format=
channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`.
losses: List of ([Loss](vis.losses#Loss), weight) tuples.
input_range: Specifies the input range as a `(min, max)` tuple. This is used to rescale the
final optimized input to the given range. (Default value=(0, 255))
wrt_tensor: Short for, with respect to. This instructs the optimizer that the aggregate loss from `losses`
should be minimized with respect to `wrt_tensor`.
`wrt_tensor` can be any tensor that is part of the model graph. Default value is set to None
which means that loss will simply be minimized with respect to `input_tensor`.
norm_grads: True to normalize gradients. Normalization avoids very small or large gradients and ensures
a smooth gradient gradient descent process. If you want the actual gradient
(for example, visualizing attention), set this to false.
"""
self.input_tensor = input_tensor
self.input_range = input_range
self.loss_names = []
self.loss_functions = []
self.wrt_tensor = self.input_tensor if wrt_tensor is None else wrt_tensor
overall_loss = None
for loss, weight in losses:
# Perf optimization. Don't build loss function with 0 weight.
if weight != 0:
loss_fn = weight * loss.build_loss()
overall_loss = loss_fn if overall_loss is None else overall_loss + loss_fn
self.loss_names.append(loss.name)
self.loss_functions.append(loss_fn)
# Compute gradient of overall with respect to `wrt` tensor.
grads = K.gradients(overall_loss, self.wrt_tensor)[0]
if norm_grads:
grads = grads / (K.sqrt(K.mean(K.square(grads))) + K.epsilon())
# The main function to compute various quantities in optimization loop.
self.compute_fn = K.function([self.input_tensor, K.learning_phase()],
self.loss_functions + [overall_loss, grads, self.wrt_tensor])
def gradCAM(model, dog_x, class_idx):
model_class_output = model.output[:, class_idx]
print('model_class_output shape', model_class_output.shape)
last_conv_layer = model.get_layer('block5_conv3')
grads_list = K.gradients(model_class_output, last_conv_layer.output)
grads = grads_list[0]
# grads shape: (?, 14, 14, 512)
pooled_grads = K.mean(grads, axis=(0, 1, 2))
# pooled_grads shape: (512,)
iterate = K.function(inputs=[model.input],
outputs=[pooled_grads, last_conv_layer.output[0]])
pooled_grads_value, conv_layer_output_value = iterate([dog_x])
# conv_layer_output_value shape: (14, 14, 512)
for i in range(last_conv_layer.filters):
conv_layer_output_value[:, :, i] *= pooled_grads_value[i]
heatmap = np.mean(conv_layer_output_value, axis=-1)
# heatmap shape: (14, 14)
heatmap = normalize_heatmap(heatmap)
return heatmap
def reg_mean_euclidean(y_true, y_pred):
return K.mean(
euclidean_distance(y_true, y_pred) *
(1 + 10 * euclidean_distance(y_true, tf.constant([0.0, 0.0]))))
def ms_euclidean(y_true, y_pred):
return K.mean(K.sum(K.square(y_true - y_pred), axis=1))
def mean_euclidean(y_true, y_pred):
return K.mean(euclidean_distance(y_true, y_pred))
def class_loss_cls(y_true, y_pred):
return lambda_cls_class * K.mean(
categorical_crossentropy(y_true[0, :, :], y_pred[0, :, :]))
def normalize(x):
return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)
# the name of the layer we want to visualize
layer_name = 'forVisual'
# this is the placeholder for the input images
input_img = model.input
# get the symbolic outputs of each "key" layer.
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
kept_filters = []
for filter_index in range(0, 50):
print('start')
# we build a loss function that maximizes the activation
# of the nth filter of the layer considered
layer_output = layer_dict[layer_name].output
loss = K.mean(layer_output[:, :, :, filter_index])
# we compute the gradient of the input picture wrt this loss
grads = K.gradients(loss, input_img)[0]
# normalization trick: we normalize the gradient by its L2 norm
grads = normalize(grads)
# this function returns the loss and grads given the input picture
iterate = K.function([input_img], [loss, grads])
# step size for gradient ascent
step = 1.
# we start from a gray image with some random noise
input_img_data = np.random.random((1, resolution, resolution, 1))
def main(_):
# disable all training specific operations
K.set_learning_phase(0)
model = applications.inception_v3.InceptionV3(weights='imagenet',
include_top=False)
layer_contributions = {
'mixed2': 0.2,
'mixed3': 3.0,
'mixed4': 2.0,
'mixed5': 1.5
}
layer_dict = dict([(layer.name, layer) for layer in model.layers])
loss = K.variable(0.,)
for layer_name in layer_contributions:
coeff = layer_contributions[layer_name]
activation = layer_dict[layer_name].output
scaling = K.prod(K.cast(K.shape(activation), 'float32'))
# avoid artifacts by only involving non-boarder pixels
loss += coeff * K.sum(K.square(activation[:, 2:-2, 2:-2, :])) / scaling
# start the gradient-ascent process
dream = model.input
grads_list = K.gradients(loss, dream)
grads = grads_list[0]
# trick: normalize gradients
grads /= K.maximum(K.mean(K.abs(grads)), 1e-7)
fetch_loss_and_grads = K.function(inputs=[dream],
outputs=[loss, grads])
def gradient_ascent(x, iterations, step_rate, max_loss=None):
for i in range(iterations):
loss_value, grads_value = fetch_loss_and_grads([x])
if max_loss is not None and loss_value > max_loss:
break
print('@{:4d}: {:.4f}'.format(i, loss_value))
x += step_rate * grads_value
return x
img = preprocess_img(FLAGS.img_path)
original_shape = img.shape[1:3]
successive_shapes = [original_shape]
for i in range(1, NUM_OCTAVES):
shape = tuple([int(dim / (OCTAVES_SCLAE ** i))
for dim in original_shape])
successive_shapes.append(shape)
# reverse
successive_shapes = successive_shapes[::-1]
original_img = np.copy(img)
shrunk_original_img = resize_img(img, successive_shapes[0])
for shape in successive_shapes:
print('Preprocess image with shape: {}'.format(shape))
img = resize_img(img, shape)
img = gradient_ascent(img,
iterations=FLAGS.iterations,
step_rate=FLAGS.step_rate,
max_loss=MAX_LOSS)
same_size_original = resize_img(original_img, shape)
if FLAGS.repair_lost_detail:
upscale_shrunk_original_img = resize_img(shrunk_original_img, shape)
lost_detail = same_size_original - upscale_shrunk_original_img
img += lost_detail
shrunk_original_img = same_size_original
save_img(img, filename='dream_at_scale_{}.png'.format(str(shape)))
save_img(img, filename='dream.png')
def build_model(self):
kernel_l2_reg = 1e-5
"""Build an actor (policy) network that maps states -> actions."""
# Define input layer (states)
states = layers.Input(shape=(self.state_size, ), name='states')
# size_repeat = 30
# block_size = size_repeat*self.state_size
# print("Actor block size = {}".format(block_size))
#
# net = layers.concatenate([states]*size_repeat)
# # net = layers.Dense(block_size,
# # # kernel_initializer=initializers.RandomNormal(mean=1.0, stddev=0.1),
# # # bias_initializer=initializers.RandomNormal(mean=0.0, stddev=0.01),
# # activation=None,
# # use_bias=False)(states)
# net = layers.BatchNormalization()(net)
# net = layers.Dropout(0.2)(net)
# # net = layers.LeakyReLU(1e-2)(net)
#
# for _ in range(5):
# net = res_block(net, block_size)
# Add hidden layers
net = layers.Dense(
units=300,
kernel_regularizer=regularizers.l2(kernel_l2_reg))(states)
net = layers.BatchNormalization()(net)
net = layers.LeakyReLU(1e-2)(net)
net = layers.Dense(
units=400, kernel_regularizer=regularizers.l2(kernel_l2_reg))(net)
net = layers.BatchNormalization()(net)
net = layers.LeakyReLU(1e-2)(net)
net = layers.Dense(
units=200, kernel_regularizer=regularizers.l2(kernel_l2_reg))(net)
net = layers.BatchNormalization()(net)
net = layers.LeakyReLU(1e-2)(net)
# Try different layer sizes, activations, add batch normalization, regularizers, etc.
# # Add final output layer with sigmoid activation
# raw_actions = layers.Dense(units=self.action_size,
# activation='sigmoid',
# # kernel_regularizer=regularizers.l2(kernel_l2_reg),
# kernel_initializer=initializers.RandomUniform(minval=-3e-3, maxval=3e-3),
# # bias_initializer=initializers.RandomUniform(minval=-3e-3, maxval=3e-3),
# name='raw_actions')(net)
#
# # Scale [0, 1] output for each action dimension to proper range
# actions = layers.Lambda(lambda x: (x * self.action_range) + self.action_low, name='actions')(raw_actions)
actions = layers.Dense(
units=self.action_size,
activation='tanh',
kernel_regularizer=regularizers.l2(kernel_l2_reg),
kernel_initializer=initializers.RandomUniform(minval=-3e-3,
maxval=3e-3),
name='actions')(net)
# Create Keras model
self.model = models.Model(inputs=states, outputs=actions)
# Define loss function using action value (Q value) gradients
action_gradients = layers.Input(shape=(self.action_size, ))
loss = K.mean(-action_gradients * actions)
# Incorporate any additional losses here (e.g. from regularizers)
# Define optimizer and training function
optimizer = optimizers.Adam(lr=1e-4)
updates_op = optimizer.get_updates(params=self.model.trainable_weights,
loss=loss)
self.train_fn = K.function(
inputs=[self.model.input, action_gradients,
K.learning_phase()],
outputs=[],
updates=updates_op)
def wasserstein_loss(y_true, y_pred):
return K.mean(y_true * y_pred)
for row in train_data:
train_x.append(row[0])
for row in test_data:
test_x.append(row[0])
train_x = np.array(train_x).reshape(-1, 128, 128, 3)
test_x = np.array(test_x).reshape(-1, 128, 128, 3)
reconstruction_loss = tf.keras.losses.binary_crossentropy(
K.flatten(input_img), K.flatten(outputs))
reconstruction_loss *= 128 * 128
kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var)
kl_loss = K.sum(kl_loss, axis=-1)
kl_loss *= -0.5
vae_loss = K.mean(reconstruction_loss + kl_loss)
vae.add_loss(vae_loss)
vae.compile(optimizer='rmsprop')
vae.summary()
flag = 1
if flag == 0:
vae.fit(train_x,
epochs=10,
batch_size=64,
validation_data=(test_x, None))
vae.save_weights("vae_cnn.h5")
elif flag == 1:
vae.load_weights('vae_cnn.h5')
image = cv2.imread("input.jpg")
while True:
def step(self, a, states):
"""
:param a: ground-truth
:param states:
type: list
index[:-2]: r, c, e (#: self.nb_layers)
index[-2:] (if self.extrap_start_time is not None:): [frame_prediction, t+1]
:return:
"""
r_tm1 = states[:self.nb_layers]
c_tm1 = states[self.nb_layers:2 * self.nb_layers]
e_tm1 = states[2 * self.nb_layers:3 * self.nb_layers]
if self.extrap_start_time is not None:
t = states[-1]
# if past self.extrap_start_time, the previous prediction will be treated as the actual
a = K.switch(t >= self.t_extrap, states[-2], a)
c = []
r = []
e = []
for l in reversed(range(self.nb_layers)):
inputs = [r_tm1[l], e_tm1[l]]
if l < self.nb_layers - 1:
inputs.append(r_up)
inputs = K.concatenate(inputs, axis=self.channel_axis)
# print l, inputs.shape
i = self.conv_layers['i'][l].call(inputs)
f = self.conv_layers['f'][l].call(inputs)
o = self.conv_layers['o'][l].call(inputs)
_c = f * c_tm1[l] + i * self.conv_layers['c'][l].call(inputs)
_r = o * self.LSTM_activation(_c)
c.insert(0, _c)
r.insert(0, _r)
if l > 0:
r_up = self.upsample.call(_r) # upsampling
for l in range(self.nb_layers):
ahat = self.conv_layers['ahat'][l].call(r[l])
if l == 0:
ahat = K.minimum(ahat, self.pixel_max)
frame_prediction = ahat
### threshold
where = K.greater_equal(frame_prediction,
K.constant(self.threshold))
frame_prediction = tf.where(
where,
0.5 * tf.ones_like(frame_prediction, dtype=tf.float32),
tf.zeros_like(frame_prediction, dtype=tf.float32))
###
# compute errors
e_up = ahat - a
e_down = a - ahat
# ROI loss
if l == 0 and self.use_roi_loss:
e_up = tf.add(e_up,
tf.multiply(e_up, a, name='multiply_up_err'),
name='add_up_err')
e_down = tf.add(e_down,
tf.multiply(e_down,
a,
name='multiply_down_err'),
name='add_down_err')
#
e_up = self.error_activation(e_up)
e_down = self.error_activation(e_down)
e.append(K.concatenate((e_up, e_down), axis=self.channel_axis))
if self.output_layer_num == l:
if self.output_layer_type == 'A':
output = a
elif self.output_layer_type == 'Ahat':
output = ahat
elif self.output_layer_type == 'R':
output = r[l]
elif self.output_layer_type == 'E':
output = e[l]
if l < self.nb_layers - 1:
a = self.conv_layers['a'][l].call(e[l])
a = self.pool.call(a) # target for next layer (downsampling)
if self.output_layer_type is None:
if self.output_mode == 'prediction':
output = frame_prediction
else:
for l in range(self.nb_layers):
layer_error = K.mean(K.batch_flatten(e[l]),
axis=-1,
keepdims=True)
# TODO: where is all_error ?
all_error = layer_error if l == 0 else K.concatenate(
(all_error, layer_error), axis=-1)
# print l, e[l].shape, layer_error.shape, all_error.shape
if self.output_mode == 'error':
output = all_error
else:
output = K.concatenate(
(K.batch_flatten(frame_prediction), all_error),
axis=-1)
# print output.shape
states = r + c + e
if self.extrap_start_time is not None:
###
'''
sess = tf.get_default_session()
comparison = tf.greater_equal(frame_prediction, tf.constant(0.3))
sess.run(comparison)
conditional_op = tf.assign(frame_prediction, tf.where(comparison, 0.5 * tf.ones_like(frame_prediction), tf.zeros_like(frame_prediction)))
sess.run(conditional_op)
'''
###
states += [frame_prediction, t + 1]
return output, states
