def __init__(self, model, layer_name): self.model = model self.layer_name = layer_name dream = model.input # Get the symbolic outputs of each "key" layer (we gave them unique names). layers_all = [layer.name for layer in model.layers] if layer_name not in layers_all: raise ValueError('Layer ' + layer_name + ' not found in model.') # Define the loss. loss = K.variable(0.) for layer_local in model.layers: if layer_local.name == layer_name: x = layer_local.output # We avoid border artifacts by only involving non-border pixels in the loss. if K.image_data_format() == 'channels_first': scaling = K.prod(K.cast(K.shape(x), 'float32')) loss = loss + K.sum(K.square(x[:, :, 2:-2, 2:-2])) / scaling else: scaling = K.prod(K.cast(K.shape(x), 'float32')) loss = loss + K.sum(K.square(x[:, 2:-2, 2:-2, :])) / scaling # Compute the gradients of the dream wrt the loss. grads = K.gradients(loss, dream)[0] # Normalize gradients. grads /= K.maximum(K.mean(K.abs(grads)), K.epsilon()) # Set up function to retrieve the value # of the loss and gradients given an input image. outputs = [loss, grads] self.fetch_loss_and_grads = K.function([dream], outputs)
def compute_output_shape(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape).as_list() if None not in input_shape[1:]: if self.data_format == "channels_first": total = K.prod(input_shape[2:4]) * self.num_anchors else: total = K.prod(input_shape[1:3]) * self.num_anchors total = K.get_value(total) return tensor_shape.TensorShape((input_shape[0], total, 4)) return tensor_shape.TensorShape((input_shape[0], None, 4))
def compute_output_shape(self, input_shape): boxes_shape, other_shape = input_shape if len(boxes_shape) == 3: output_shape = tuple( list(boxes_shape[:2]) + [K.prod([s for s in other_shape[2:]]) + 4]) return tensor_shape.TensorShape(output_shape) elif len(boxes_shape) == 4: output_shape = tuple( list(boxes_shape[:3]) + [K.prod([s for s in other_shape[3:]]) + 4]) return tensor_shape.TensorShape(output_shape)
def compute_output_shape(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape).as_list() if None not in input_shape[1:]: if self.data_format == "channels_first": total = K.prod(input_shape[2:4]) * self.num_anchors else: total = K.prod(input_shape[1:3]) * self.num_anchors # TODO: fails time_distributed tests for RetinaMask # AttributeError: 'Tensor' object has no attribute 'numpy' total = K.get_value(total) return tensor_shape.TensorShape((input_shape[0], total, 4)) return tensor_shape.TensorShape((input_shape[0], None, 4))
def define_deepDream_model_layerBased(model): dream = model.input print('Model loaded.') # Get the symbolic outputs of each "key" layer (we gave them unique names). layer_dict = dict([(layer.name, layer) for layer in model.layers]) # Define the loss. loss = K.variable(0.) for layer_name in settings['features']: # Add the L2 norm of the features of a layer to the loss. if layer_name not in layer_dict: raise ValueError('Layer ' + layer_name + ' not found in model.') coeff = settings['features'][layer_name] x = layer_dict[layer_name].output # We avoid border artifacts by only involving non-border pixels in the loss. scaling = K.prod(K.cast(K.shape(x), 'float32')) if K.image_data_format() == 'channels_first': loss = loss + coeff * K.sum(K.square(x[:, :, 2:-2, 2:-2])) / scaling else: loss = loss + coeff * K.sum(K.square(x[:, 2:-2, 2:-2, :])) / scaling # Compute the gradients of the dream wrt the loss. grads = K.gradients(loss, dream)[0] # Normalize gradients. grads /= K.maximum(K.mean(K.abs(grads)), K.epsilon()) # Set up function to retrieve the value # of the loss and gradients given an input image. outputs = [loss, grads] fetch_loss_and_grads = K.function([dream], outputs)
def training_phase(): mean_batch = K.mean(mean_instance, axis=0, keepdims=True) variance_batch = K.mean(temp, axis=0, keepdims=True) - K.square(mean_batch) mean_batch_reshaped = K.flatten(mean_batch) variance_batch_reshaped = K.flatten(variance_batch) if K.backend() != 'cntk': sample_size = K.prod( [K.shape(inputs)[axis] for axis in reduction_axes]) sample_size = K.cast(sample_size, dtype=K.dtype(inputs)) # sample variance - unbiased estimator of population variance variance_batch_reshaped *= sample_size / (sample_size - (1.0 + self.epsilon)) self.add_update([ K.moving_average_update(self.moving_mean, mean_batch_reshaped, self.momentum), K.moving_average_update(self.moving_variance, variance_batch_reshaped, self.momentum) ], inputs) return normalize_func(mean_batch, variance_batch)
def gram_matrix(self, X): """グラム行列の算出""" X_sw = K.permute_dimensions( X, (0, 3, 2, 1) ) # 軸の入れ替え s = K.shape(X_sw) new_shape = (s[0], s[1], s[2]*s[3]) X_rs = K.reshape(X_sw, new_shape) X_rs_t = K.permute_dimensions( X_rs, (0, 2, 1) ) # 行列の転置 dot = K.batch_dot(X_rs, X_rs_t) # 内積の計算 norm = K.prod(K.cast(s[1:], 'float32')) return dot/norm
def gram_matrix(X): # 軸の入れ替え => batch, channel, height, width axis_replaced_X = K.permute_dimensions(X, (0, 3, 2, 1)) replaced_shape = K.shape(axis_replaced_X) # 特徴マップ(高さと幅を1つの軸に展開)の内積をとるためのshape dot_shape = (replaced_shape[0], replaced_shape[1], replaced_shape[2] * replaced_shape[3]) # 実際に内積を計算する行列 dot_X = K.reshape(axis_replaced_X, dot_shape) # 転置行列 dot_X_t = K.permute_dimensions(dot_X, (0, 2, 1)) # 行列の内積 dot = K.batch_dot(dot_X, dot_X_t) norm = K.prod(K.cast(replaced_shape[1:], 'float32')) return dot / norm
def build(self, input_shape): input_shape = tensor_shape.TensorShape(input_shape) if len(input_shape) != 5: raise ValueError('Inputs should have rank 5, ' 'received input shape: %s' % input_shape) if self.data_format == 'channels_first': channel_axis = 1 else: channel_axis = -1 if input_shape.dims[channel_axis].value is None: raise ValueError('The channel dimension of the inputs ' 'should be defined, found None: %s' % input_shape) input_dim = int(input_shape[channel_axis]) self.input_spec = InputSpec(ndim=5, axes={channel_axis: input_dim}) if self.data_format == 'channels_first': depth = int(input_shape[2]) else: depth = int(input_shape[1]) kernel_shape = (depth, self.filter_size, self.filter_size, input_dim, 1) # self.kernel = self.add_weight( # 'kernel', # shape=kernel_shape, # initializer=self.kernel_initializer, # regularizer=self.kernel_regularizer, # constraint=self.kernel_constraint, # trainable=False, # dtype=self.dtype) W = K.ones(kernel_shape, dtype=K.floatx()) W = W / K.cast(K.prod(K.int_shape(W)), dtype=K.floatx()) self.kernel = W # self.set_weights([W]) if self.use_bias: self.bias = self.add_weight('bias', shape=(depth, self.filter_size, self.filter_size), initializer=self.bias_initializer, regularizer=self.bias_regularizer, constraint=self.bias_constraint, trainable=False, dtype=self.dtype) else: self.bias = None self.built = True
def compute_output_shape(self, input_shape): boxes_shape, other_shape = input_shape n = len(boxes_shape) - 1 output_shape = tuple( list(boxes_shape[:n]) + [K.prod([s for s in other_shape[n:]]) + 4]) return tensor_shape.TensorShape(output_shape)
def contents_feature_loss(y_contents, contents_pred): norm = K.prod(K.cast(K.shape(y_contents)[1:], 'float32')) # 二乗誤差 return K.sum(K.square(contents_pred - y_contents), axis=(1, 2, 3)) / norm
def __init__(self, model, layer_name, index_feature, kind_of_optim='squared'): """ Initialisation function of the Optimization of the feature map Parameters ---------- model : keras model layer_name : string : the layer you want to use index_feature : string : the index of the feature in the given layer kind_of_optim : string The kind of optimisation maded on the given feature. The default is 'squared'. Use 'pos' for the positive feature maximisation and 'neg' for the negative one Returns ------- None """ self.model = model self.layer_name = layer_name self.index_feature = index_feature self.kind_of_optim = kind_of_optim dream = model.input # Get the symbolic outputs of each "key" layer (we gave them unique names). layers_all = [layer.name for layer in model.layers] if layer_name not in layers_all: raise ValueError('Layer ' + layer_name + ' not found in model.') # Define the loss. loss = K.variable(0.) for layer_local in model.layers: if layer_local.name == layer_name: x = layer_local.output # We avoid border artifacts by only involving non-border pixels in the loss. if K.image_data_format() == 'channels_first': raise (NotImplementedError) x_index_feature = x[:, index_feature, :, :] x_index_feature = K.expand_dims(x_index_feature, axis=1) scaling = K.prod( K.cast(K.shape(x_index_feature), 'float32')) loss = loss + K.sum( K.square(x_index_feature[:, :, 2:-2, 2:-2])) / scaling else: x_index_feature = x[:, :, :, index_feature] x_index_feature = K.expand_dims(x_index_feature, axis=-1) scaling = K.prod( K.cast(K.shape(x_index_feature), 'float32')) if self.kind_of_optim == 'squared': loss = loss + K.sum( K.square(x_index_feature[:, 2:-2, 2:-2, :])) / scaling elif self.kind_of_optim == 'pos': loss = loss + K.sum(x_index_feature[:, 2:-2, 2:-2, :]) / scaling elif self.kind_of_optim == 'neg': loss = loss - K.sum(x_index_feature[:, 2:-2, 2:-2, :]) / scaling # Compute the gradients of the dream wrt the loss. grads = K.gradients(loss, dream)[0] # Normalize gradients. grads /= K.maximum(K.mean(K.abs(grads)), K.epsilon()) # Set up function to retrieve the value # of the loss and gradients given an input image. outputs = [loss, grads] self.fetch_loss_and_grads = K.function([dream], outputs)
def feature_loss(self, y_true, y_pred): """コンテンツ特徴量の損失関数""" norm = K.prod(K.cast(K.shape(y_true)[1:], 'float32')) return K.sum( K.square(y_pred - y_true), axis=(1, 2, 3) )/norm
def msssim(self, y_true, y_pred): ''' Compute multiscale ssim according to Zhao 2016. Has only been tested with tensorflow backend (channels last) so far! Uses convolutions to do the calculations in one go. This function takes proper 2D Keras Tensors (NWHC or NCWH) # Arguments y_true: Keras Tensor with Rank 4: Image to compare to y_pred: Keras Tensor with Rank 4: Image to compare ''' # some useful inits channels = self.__int_shape(y_pred)[self.channel_dim] # repeat kernel for each channel kernel = K.tile(self.kernels, [1, 1, channels, 1]) # compute means mu_true = K.depthwise_conv2d(y_true, kernel, padding='same') mu_pred = K.depthwise_conv2d(y_pred, kernel, padding='same') # compute mean squares mu_true_sq = K.square(mu_true) mu_pred_sq = K.square(mu_pred) mu_true_pred = mu_true * mu_pred # compute input square y_true_sq = K.square(y_true) y_pred_sq = K.square(y_pred) y_true_pred = y_true * y_pred # compute variances/covariance sigma_true_sq = K.depthwise_conv2d(y_true_sq, kernel, padding='same') sigma_pred_sq = K.depthwise_conv2d(y_pred_sq, kernel, padding='same') sigma_true_pred = K.depthwise_conv2d(y_true_pred, kernel, padding='same') # centered squares of variances sigma_true_sq -= mu_true_sq sigma_pred_sq -= mu_pred_sq sigma_true_pred -= mu_true_pred # compute luminance term (l), select only maximum kernel for each channel l = (2 * mu_true_pred + self.c1) / (mu_true_sq + mu_pred_sq + self.c1) if self.dim_ordering == 'channels_last': l_max = l[:,:,:,(self.num - 1)::self.num] else: l_max = l[:,(self.num - 1)::self.num,:,:] # compute contrast-structure term (cs) cs = (2 * sigma_true_pred + self.c2) / (sigma_true_sq + sigma_pred_sq + self.c2) # compute product of different scale cs if self.dim_ordering == 'channels_last': pcs = [K.prod(cs[:,:,:,i*self.num:(i+1)*self.num], axis=-1, keepdims=True) for i in range(channels)] else: pcs = [K.prod(cs[:,i*self.num:(i+1)*self.num,:,:], axis=1, keepdims=True) for i in range(channels)] pcs = K.concatenate(pcs, axis=self.channel_dim) # compute msssim map msssim = l_max * pcs # do normalization? return msssim