def gradient_penalty(y_true, y_pred, interpolate, lamb): grad = K.gradients(y_pred, interpolate)[0] norm = K.square(grad) norm_sum = K.sum(norm,axis=np.arange(1,len(norm.shape))) l2_norm = K.sqrt(norm_sum) gp_reg = lamb*K.square(1-l2_norm) return K.mean(gp_reg)
def build_model(self): l2_regularization_kernel = 1e-5 # Input Layer input = layers.Input(shape=(self.state_size,), name='input_states') # Hidden Layers model = layers.Dense(units=300, kernel_regularizer=regularizers.l2(l2_regularization_kernel))(input) model = layers.BatchNormalization()(model) model = layers.LeakyReLU(1e-2)(model) model = layers.Dense(units=400, kernel_regularizer=regularizers.l2(l2_regularization_kernel))(model) model = layers.BatchNormalization()(model) model = layers.LeakyReLU(1e-2)(model) model = layers.Dense(units=200, kernel_regularizer=regularizers.l2(l2_regularization_kernel))(model) model = layers.BatchNormalization()(model) model = layers.LeakyReLU(1e-2)(model) # Our output layer - a fully connected layer output = layers.Dense(units=self.action_size, activation='tanh', kernel_regularizer=regularizers.l2(l2_regularization_kernel), kernel_initializer=initializers.RandomUniform(minval=-3e-3, maxval=3e-3), name='output_actions')(model) # Keras model self.model = models.Model(inputs=input, outputs=output) # Define loss and optimizer action_gradients = layers.Input(shape=(self.action_size,)) loss = K.mean(-action_gradients * output) optimizer = optimizers.Adam(lr=1e-4) update_operation = optimizer.get_updates(params=self.model.trainable_weights, loss=loss) self.train_fn = K.function(inputs=[self.model.input, action_gradients, K.learning_phase()], outputs=[], updates=update_operation)
def build_loss(self): layer_output = self.layer.output # For all other layers it is 4 is_dense = K.ndim(layer_output) == 2 loss = 0. for idx in self.filter_indices: if is_dense: loss += -K.mean(layer_output[:, idx]) else: # slicer is used to deal with `channels_first` or `channels_last` image data formats # without the ugly conditional statements. loss += -K.mean(layer_output[utils.slicer[:, idx, ...]]) return loss
def atari_qnet(input_shape, num_actions, net_name, net_size): net_name = net_name.lower() # input state state = Input(shape=input_shape) # convolutional layers conv1_32 = Conv2D(32, (8, 8), strides=(4, 4), activation='relu') conv2_64 = Conv2D(64, (4, 4), strides=(2, 2), activation='relu') conv3_64 = Conv2D(64, (3, 3), strides=(1, 1), activation='relu') # if recurrent net then change input shape if 'drqn' in net_name: # recurrent net (drqn) lambda_perm_state = lambda x: K.permute_dimensions(x, [0, 3, 1, 2]) perm_state = Lambda(lambda_perm_state)(state) dist_state = Lambda(lambda x: K.stack([x], axis=4))(perm_state) # extract features with `TimeDistributed` wrapped convolutional layers dist_conv1 = TimeDistributed(conv1_32)(dist_state) dist_conv2 = TimeDistributed(conv2_64)(dist_conv1) dist_convf = TimeDistributed(conv3_64)(dist_conv2) feature = TimeDistributed(Flatten())(dist_convf) elif 'dqn' in net_name: # fully connected net (dqn) # extract features with convolutional layers conv1 = conv1_32(state) conv2 = conv2_64(conv1) convf = conv3_64(conv2) feature = Flatten()(convf) # network type. Dense for dqn; LSTM or GRU for drqn if 'lstm' in net_name: net_type = LSTM elif 'gru' in net_name: net_type = GRU else: net_type = Dense # dueling or regular dqn/drqn if 'dueling' in net_name: value1 = net_type(net_size, activation='relu')(feature) adv1 = net_type(net_size, activation='relu')(feature) value2 = Dense(1)(value1) adv2 = Dense(num_actions)(adv1) mean_adv2 = Lambda(lambda x: K.mean(x, axis=1))(adv2) ones = K.ones([1, num_actions]) lambda_exp = lambda x: K.dot(K.expand_dims(x, axis=1), -ones) exp_mean_adv2 = Lambda(lambda_exp)(mean_adv2) sum_adv = add([exp_mean_adv2, adv2]) exp_value2 = Lambda(lambda x: K.dot(x, ones))(value2) q_value = add([exp_value2, sum_adv]) else: hid = net_type(net_size, activation='relu')(feature) q_value = Dense(num_actions)(hid) # build model return Model(inputs=state, outputs=q_value)
def build_model(self): """Build an actor (policy) network that maps states -> actions.""" # Define input layer (states) states = layers.Input(shape=(self.state_size, ), name='states') '''# Add hidden layers net = layers.Dense(units=32, activation='relu')(states) net = layers.Dense(units=64, activation='relu')(net) net = layers.Dense(units=32, activation='relu')(net) # Try different layer sizes, activations, add batch normalization, regularizers, etc. # Add final output layer with sigmoid activation raw_actions = layers.Dense(units=self.action_size, activation='sigmoid', name='raw_actions')(net) ''' ################################### # Add hidden layers net = layers.Dense(units=400, kernel_regularizer=regularizers.l2(1e-6))(states) net = layers.BatchNormalization()(net) net = layers.LeakyReLU(1e-2)(net) net = layers.Dense(units=300, kernel_regularizer=regularizers.l2(1e-6))(net) net = layers.BatchNormalization()(net) net = layers.LeakyReLU(1e-2)(net) # Add final output layer with sigmoid activation raw_actions = layers.Dense( units=self.action_size, activation='sigmoid', name='raw_actions', kernel_initializer=initializers.RandomUniform(minval=-0.003, maxval=0.003))(net) ####################################### # Scale [0, 1] output for each action dimension to proper range actions = layers.Lambda(lambda x: (x * self.action_range) + self.action_low, name='actions')(raw_actions) # Create Keras model self.model = models.Model(inputs=states, outputs=actions) # Define loss function using action value (Q value) gradients action_gradients = layers.Input(shape=(self.action_size, )) loss = K.mean(-action_gradients * actions) # Incorporate any additional losses here (e.g. from regularizers) # Define optimizer and training function optimizer = optimizers.Adam(lr=1e-6) updates_op = optimizer.get_updates(params=self.model.trainable_weights, loss=loss) self.train_fn = K.function( inputs=[self.model.input, action_gradients, K.learning_phase()], outputs=[], updates=updates_op)
def build_model(self): """Build an actor (policy) network that maps states -> actions.""" # Define input layer (states) states = layers.Input(shape=(self.state_size, ), name='states') #--------- copy from DDPG quadcopter ----------- net = layers.Dense(units=400)(states) # net = layers.BatchNormalization()(net) net = layers.Activation("relu")(net) net = layers.Dense(units=200)(net) # net = layers.BatchNormalization()(net) net = layers.Activation("relu")(net) actions = layers.Dense(units=self.action_size, activation='softmax', name='actions', kernel_initializer=initializers.RandomUniform( minval=-1, maxval=1))(net) # actions = layers.Dense(units=self.action_size, activation='sigmoid', name='actions', # kernel_initializer=initializers.RandomUniform(minval=-0.001, maxval=0.001))(net) # Add hidden layers # net = layers.Dense(units=16,activation=activations.sigmoid)(states) # net = layers.BatchNormalization()(net) # net = layers.Dense(units=16,activation=activations.sigmoid)(net) # net = layers.BatchNormalization()(net) # net = layers.Dense(units=128,activation=activations.relu)(net) # net = layers.BatchNormalization()(net) # Add final output layer with sigmoid activation # actions = layers.Dense(units=self.action_size, activation='linear', # sigmoid # name='raw_actions' )(net) # Scale [0, 1] output for each action dimension to proper range # actions = layers.Lambda(lambda x: (x * self.action_range) + self.action_low, # name='actions')(raw_actions) # Create Keras model self.model = models.Model(inputs=states, outputs=actions) action_gradients = layers.Input(shape=(self.action_size, )) loss = K.mean(-action_gradients * actions) # Define optimizer and training function optimizer = optimizers.Adam(lr=.0001) updates_op = optimizer.get_updates(params=self.model.trainable_weights, loss=loss) self.train_fn = K.function( inputs=[self.model.input, action_gradients, K.learning_phase()], outputs=[], updates=updates_op)
def contrastive_loss(y_true, y_pred): ''' Contrastive loss from Hadsell-et-al.'06 http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf @param y_true : true label 1 for positive pair, 0 for negative pair y_pred : distance output of the Siamese network ''' margin = 1 # if positive pair, y_true is 1, penalize for large distance returned by Siamese network # if negative pair, y_true is 0, penalize for distance smaller than the margin return K.mean(y_true * K.square(y_pred) + (1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))
def generate_pattern(model, layer_name, filter_index, steps, learning_rate, size=224): layer_output = model.get_layer(layer_name).output loss = K.mean(layer_output[:, :, :, filter_index]) # obtain the gradient of the loss with respect to the model's input image grads_list = K.gradients(loss, model.input) grads = grads_list[0] # gradient normalization trick grads /= (K.sqrt(K.mean(K.square(grads))) + EPSILON) # fetch loss and normalized-gradients for a given input iterate = K.function(inputs=[model.input], outputs=[loss, grads]) # loss maximization via stochastic gradient descent input_img_data = np.random.random((1, size, size, 3)) * 20 + 128 # start from gray image with random noise for i in range(steps): loss_value, grads_value = iterate([input_img_data]) print('@{:-4d}: {:.4f}'.format(i, loss_value)) # gradient ascent: adjust the input image in the direction that maximizes the loss input_img_data += grads_value * learning_rate img_tensor = input_img_data[0] return tensor_to_image(img_tensor)
def contrastive_loss(y_true, y_pred): ''' Contrastive loss from Hadsell-et-al.'06 http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf @param y_true : true label 1 for positive pair, 0 for negative pair y_pred : distance output of the Siamese network @return contrastive_loss between two values ''' margin = 1 # if positive pair, y_true is 1, penalize for large distance returned by Siamese network # if negative pair, y_true is 0, penalize for distance smaller than the margin y_positive_true = 1 y_negative_true = 0 return K.mean(y_true * K.square(y_pred) + (y_positive_true - y_true) * K.square(K.maximum(margin - y_pred, y_negative_true)))
def build_model(self): states = layers.Input(shape=(self.state_size,), name='inputStates') # Hidden Layers model = layers.Dense(units=128, activation='linear')(states) model = layers.BatchNormalization()(model) model = layers.LeakyReLU(0.01)(model) model = layers.Dropout(0.3)(model) model = layers.Dense(units=256, activation='linear')(model) model = layers.BatchNormalization()(model) model = layers.LeakyReLU(0.01)(model) model = layers.Dropout(0.3)(model) model = layers.Dense(units=512, activation='linear')(model) model = layers.BatchNormalization()(model) model = layers.LeakyReLU(0.01)(model) model = layers.Dropout(0.3)(model) model = layers.Dense(units=128, activation='linear')(model) model = layers.BatchNormalization()(model) model = layers.LeakyReLU(0.01)(model) model = layers.Dropout(0.3)(model) output = layers.Dense( units=self.action_size, activation='tanh', kernel_regularizer=regularizers.l2(0.01), name='outputActions')(model) #Keras self.model = models.Model(inputs=states, outputs=output) #Definint Optimizer actionGradients = layers.Input(shape=(self.action_size,)) loss = K.mean(-actionGradients * output) optimizer = optimizers.Adam() update_operation = optimizer.get_updates(params=self.model.trainable_weights, loss=loss) self.train_fn = K.function( inputs=[self.model.input, actionGradients, K.learning_phase()], outputs=[], updates=update_operation)
def __init__(self, input_tensor, losses, input_range=(0, 255), wrt_tensor=None, norm_grads=True): """Creates an optimizer that minimizes weighted loss function. Args: input_tensor: An input tensor of shape: `(samples, channels, image_dims...)` if `image_data_format= channels_first` or `(samples, image_dims..., channels)` if `image_data_format=channels_last`. losses: List of ([Loss](vis.losses#Loss), weight) tuples. input_range: Specifies the input range as a `(min, max)` tuple. This is used to rescale the final optimized input to the given range. (Default value=(0, 255)) wrt_tensor: Short for, with respect to. This instructs the optimizer that the aggregate loss from `losses` should be minimized with respect to `wrt_tensor`. `wrt_tensor` can be any tensor that is part of the model graph. Default value is set to None which means that loss will simply be minimized with respect to `input_tensor`. norm_grads: True to normalize gradients. Normalization avoids very small or large gradients and ensures a smooth gradient gradient descent process. If you want the actual gradient (for example, visualizing attention), set this to false. """ self.input_tensor = input_tensor self.input_range = input_range self.loss_names = [] self.loss_functions = [] self.wrt_tensor = self.input_tensor if wrt_tensor is None else wrt_tensor overall_loss = None for loss, weight in losses: # Perf optimization. Don't build loss function with 0 weight. if weight != 0: loss_fn = weight * loss.build_loss() overall_loss = loss_fn if overall_loss is None else overall_loss + loss_fn self.loss_names.append(loss.name) self.loss_functions.append(loss_fn) # Compute gradient of overall with respect to `wrt` tensor. grads = K.gradients(overall_loss, self.wrt_tensor)[0] if norm_grads: grads = grads / (K.sqrt(K.mean(K.square(grads))) + K.epsilon()) # The main function to compute various quantities in optimization loop. self.compute_fn = K.function([self.input_tensor, K.learning_phase()], self.loss_functions + [overall_loss, grads, self.wrt_tensor])
def gradCAM(model, dog_x, class_idx): model_class_output = model.output[:, class_idx] print('model_class_output shape', model_class_output.shape) last_conv_layer = model.get_layer('block5_conv3') grads_list = K.gradients(model_class_output, last_conv_layer.output) grads = grads_list[0] # grads shape: (?, 14, 14, 512) pooled_grads = K.mean(grads, axis=(0, 1, 2)) # pooled_grads shape: (512,) iterate = K.function(inputs=[model.input], outputs=[pooled_grads, last_conv_layer.output[0]]) pooled_grads_value, conv_layer_output_value = iterate([dog_x]) # conv_layer_output_value shape: (14, 14, 512) for i in range(last_conv_layer.filters): conv_layer_output_value[:, :, i] *= pooled_grads_value[i] heatmap = np.mean(conv_layer_output_value, axis=-1) # heatmap shape: (14, 14) heatmap = normalize_heatmap(heatmap) return heatmap
def reg_mean_euclidean(y_true, y_pred): return K.mean( euclidean_distance(y_true, y_pred) * (1 + 10 * euclidean_distance(y_true, tf.constant([0.0, 0.0]))))
def ms_euclidean(y_true, y_pred): return K.mean(K.sum(K.square(y_true - y_pred), axis=1))
def mean_euclidean(y_true, y_pred): return K.mean(euclidean_distance(y_true, y_pred))
def class_loss_cls(y_true, y_pred): return lambda_cls_class * K.mean( categorical_crossentropy(y_true[0, :, :], y_pred[0, :, :]))
def normalize(x): return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)
# the name of the layer we want to visualize layer_name = 'forVisual' # this is the placeholder for the input images input_img = model.input # get the symbolic outputs of each "key" layer. layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]]) kept_filters = [] for filter_index in range(0, 50): print('start') # we build a loss function that maximizes the activation # of the nth filter of the layer considered layer_output = layer_dict[layer_name].output loss = K.mean(layer_output[:, :, :, filter_index]) # we compute the gradient of the input picture wrt this loss grads = K.gradients(loss, input_img)[0] # normalization trick: we normalize the gradient by its L2 norm grads = normalize(grads) # this function returns the loss and grads given the input picture iterate = K.function([input_img], [loss, grads]) # step size for gradient ascent step = 1. # we start from a gray image with some random noise input_img_data = np.random.random((1, resolution, resolution, 1))
def main(_): # disable all training specific operations K.set_learning_phase(0) model = applications.inception_v3.InceptionV3(weights='imagenet', include_top=False) layer_contributions = { 'mixed2': 0.2, 'mixed3': 3.0, 'mixed4': 2.0, 'mixed5': 1.5 } layer_dict = dict([(layer.name, layer) for layer in model.layers]) loss = K.variable(0.,) for layer_name in layer_contributions: coeff = layer_contributions[layer_name] activation = layer_dict[layer_name].output scaling = K.prod(K.cast(K.shape(activation), 'float32')) # avoid artifacts by only involving non-boarder pixels loss += coeff * K.sum(K.square(activation[:, 2:-2, 2:-2, :])) / scaling # start the gradient-ascent process dream = model.input grads_list = K.gradients(loss, dream) grads = grads_list[0] # trick: normalize gradients grads /= K.maximum(K.mean(K.abs(grads)), 1e-7) fetch_loss_and_grads = K.function(inputs=[dream], outputs=[loss, grads]) def gradient_ascent(x, iterations, step_rate, max_loss=None): for i in range(iterations): loss_value, grads_value = fetch_loss_and_grads([x]) if max_loss is not None and loss_value > max_loss: break print('@{:4d}: {:.4f}'.format(i, loss_value)) x += step_rate * grads_value return x img = preprocess_img(FLAGS.img_path) original_shape = img.shape[1:3] successive_shapes = [original_shape] for i in range(1, NUM_OCTAVES): shape = tuple([int(dim / (OCTAVES_SCLAE ** i)) for dim in original_shape]) successive_shapes.append(shape) # reverse successive_shapes = successive_shapes[::-1] original_img = np.copy(img) shrunk_original_img = resize_img(img, successive_shapes[0]) for shape in successive_shapes: print('Preprocess image with shape: {}'.format(shape)) img = resize_img(img, shape) img = gradient_ascent(img, iterations=FLAGS.iterations, step_rate=FLAGS.step_rate, max_loss=MAX_LOSS) same_size_original = resize_img(original_img, shape) if FLAGS.repair_lost_detail: upscale_shrunk_original_img = resize_img(shrunk_original_img, shape) lost_detail = same_size_original - upscale_shrunk_original_img img += lost_detail shrunk_original_img = same_size_original save_img(img, filename='dream_at_scale_{}.png'.format(str(shape))) save_img(img, filename='dream.png')
def build_model(self): kernel_l2_reg = 1e-5 """Build an actor (policy) network that maps states -> actions.""" # Define input layer (states) states = layers.Input(shape=(self.state_size, ), name='states') # size_repeat = 30 # block_size = size_repeat*self.state_size # print("Actor block size = {}".format(block_size)) # # net = layers.concatenate([states]*size_repeat) # # net = layers.Dense(block_size, # # # kernel_initializer=initializers.RandomNormal(mean=1.0, stddev=0.1), # # # bias_initializer=initializers.RandomNormal(mean=0.0, stddev=0.01), # # activation=None, # # use_bias=False)(states) # net = layers.BatchNormalization()(net) # net = layers.Dropout(0.2)(net) # # net = layers.LeakyReLU(1e-2)(net) # # for _ in range(5): # net = res_block(net, block_size) # Add hidden layers net = layers.Dense( units=300, kernel_regularizer=regularizers.l2(kernel_l2_reg))(states) net = layers.BatchNormalization()(net) net = layers.LeakyReLU(1e-2)(net) net = layers.Dense( units=400, kernel_regularizer=regularizers.l2(kernel_l2_reg))(net) net = layers.BatchNormalization()(net) net = layers.LeakyReLU(1e-2)(net) net = layers.Dense( units=200, kernel_regularizer=regularizers.l2(kernel_l2_reg))(net) net = layers.BatchNormalization()(net) net = layers.LeakyReLU(1e-2)(net) # Try different layer sizes, activations, add batch normalization, regularizers, etc. # # Add final output layer with sigmoid activation # raw_actions = layers.Dense(units=self.action_size, # activation='sigmoid', # # kernel_regularizer=regularizers.l2(kernel_l2_reg), # kernel_initializer=initializers.RandomUniform(minval=-3e-3, maxval=3e-3), # # bias_initializer=initializers.RandomUniform(minval=-3e-3, maxval=3e-3), # name='raw_actions')(net) # # # Scale [0, 1] output for each action dimension to proper range # actions = layers.Lambda(lambda x: (x * self.action_range) + self.action_low, name='actions')(raw_actions) actions = layers.Dense( units=self.action_size, activation='tanh', kernel_regularizer=regularizers.l2(kernel_l2_reg), kernel_initializer=initializers.RandomUniform(minval=-3e-3, maxval=3e-3), name='actions')(net) # Create Keras model self.model = models.Model(inputs=states, outputs=actions) # Define loss function using action value (Q value) gradients action_gradients = layers.Input(shape=(self.action_size, )) loss = K.mean(-action_gradients * actions) # Incorporate any additional losses here (e.g. from regularizers) # Define optimizer and training function optimizer = optimizers.Adam(lr=1e-4) updates_op = optimizer.get_updates(params=self.model.trainable_weights, loss=loss) self.train_fn = K.function( inputs=[self.model.input, action_gradients, K.learning_phase()], outputs=[], updates=updates_op)
def wasserstein_loss(y_true, y_pred): return K.mean(y_true * y_pred)
for row in train_data: train_x.append(row[0]) for row in test_data: test_x.append(row[0]) train_x = np.array(train_x).reshape(-1, 128, 128, 3) test_x = np.array(test_x).reshape(-1, 128, 128, 3) reconstruction_loss = tf.keras.losses.binary_crossentropy( K.flatten(input_img), K.flatten(outputs)) reconstruction_loss *= 128 * 128 kl_loss = 1 + z_log_var - K.square(z_mean) - K.exp(z_log_var) kl_loss = K.sum(kl_loss, axis=-1) kl_loss *= -0.5 vae_loss = K.mean(reconstruction_loss + kl_loss) vae.add_loss(vae_loss) vae.compile(optimizer='rmsprop') vae.summary() flag = 1 if flag == 0: vae.fit(train_x, epochs=10, batch_size=64, validation_data=(test_x, None)) vae.save_weights("vae_cnn.h5") elif flag == 1: vae.load_weights('vae_cnn.h5') image = cv2.imread("input.jpg") while True:
def step(self, a, states): """ :param a: ground-truth :param states: type: list index[:-2]: r, c, e (#: self.nb_layers) index[-2:] (if self.extrap_start_time is not None:): [frame_prediction, t+1] :return: """ r_tm1 = states[:self.nb_layers] c_tm1 = states[self.nb_layers:2 * self.nb_layers] e_tm1 = states[2 * self.nb_layers:3 * self.nb_layers] if self.extrap_start_time is not None: t = states[-1] # if past self.extrap_start_time, the previous prediction will be treated as the actual a = K.switch(t >= self.t_extrap, states[-2], a) c = [] r = [] e = [] for l in reversed(range(self.nb_layers)): inputs = [r_tm1[l], e_tm1[l]] if l < self.nb_layers - 1: inputs.append(r_up) inputs = K.concatenate(inputs, axis=self.channel_axis) # print l, inputs.shape i = self.conv_layers['i'][l].call(inputs) f = self.conv_layers['f'][l].call(inputs) o = self.conv_layers['o'][l].call(inputs) _c = f * c_tm1[l] + i * self.conv_layers['c'][l].call(inputs) _r = o * self.LSTM_activation(_c) c.insert(0, _c) r.insert(0, _r) if l > 0: r_up = self.upsample.call(_r) # upsampling for l in range(self.nb_layers): ahat = self.conv_layers['ahat'][l].call(r[l]) if l == 0: ahat = K.minimum(ahat, self.pixel_max) frame_prediction = ahat ### threshold where = K.greater_equal(frame_prediction, K.constant(self.threshold)) frame_prediction = tf.where( where, 0.5 * tf.ones_like(frame_prediction, dtype=tf.float32), tf.zeros_like(frame_prediction, dtype=tf.float32)) ### # compute errors e_up = ahat - a e_down = a - ahat # ROI loss if l == 0 and self.use_roi_loss: e_up = tf.add(e_up, tf.multiply(e_up, a, name='multiply_up_err'), name='add_up_err') e_down = tf.add(e_down, tf.multiply(e_down, a, name='multiply_down_err'), name='add_down_err') # e_up = self.error_activation(e_up) e_down = self.error_activation(e_down) e.append(K.concatenate((e_up, e_down), axis=self.channel_axis)) if self.output_layer_num == l: if self.output_layer_type == 'A': output = a elif self.output_layer_type == 'Ahat': output = ahat elif self.output_layer_type == 'R': output = r[l] elif self.output_layer_type == 'E': output = e[l] if l < self.nb_layers - 1: a = self.conv_layers['a'][l].call(e[l]) a = self.pool.call(a) # target for next layer (downsampling) if self.output_layer_type is None: if self.output_mode == 'prediction': output = frame_prediction else: for l in range(self.nb_layers): layer_error = K.mean(K.batch_flatten(e[l]), axis=-1, keepdims=True) # TODO: where is all_error ? all_error = layer_error if l == 0 else K.concatenate( (all_error, layer_error), axis=-1) # print l, e[l].shape, layer_error.shape, all_error.shape if self.output_mode == 'error': output = all_error else: output = K.concatenate( (K.batch_flatten(frame_prediction), all_error), axis=-1) # print output.shape states = r + c + e if self.extrap_start_time is not None: ### ''' sess = tf.get_default_session() comparison = tf.greater_equal(frame_prediction, tf.constant(0.3)) sess.run(comparison) conditional_op = tf.assign(frame_prediction, tf.where(comparison, 0.5 * tf.ones_like(frame_prediction), tf.zeros_like(frame_prediction))) sess.run(conditional_op) ''' ### states += [frame_prediction, t + 1] return output, states