def _row_kernel(upsampled_region_size, upsample_factor, axis_offsets, data_shape): data_shape_float = tf.cast(data_shape, tf.float32) row_constant = tf.cast(data_shape_float[1] * upsample_factor, tf.complex64) row_constant = (-1j * 2 * np.pi / row_constant) row_kernel_a = tf.range(0, upsampled_region_size, dtype=tf.float32) row_kernel_a = tf.reshape(row_kernel_a, (1, -1)) row_kernel_a = tf.tile(row_kernel_a, (data_shape[0], 1)) row_kernel_a = tf.transpose(row_kernel_a) row_kernel_a = row_kernel_a - axis_offsets[:, 0] row_kernel_b = tf.range(0, data_shape_float[1], dtype=tf.float32) row_kernel_b = fftshift1d(row_kernel_b) row_kernel_b = tf.reshape(row_kernel_b, (1, -1)) row_kernel_b = tf.tile(row_kernel_b, (data_shape[0], 1)) row_kernel_b = row_kernel_b - tf.floor(data_shape_float[1] / 2.) row_kernel_a = tf.expand_dims(row_kernel_a, 1) row_kernel_b = tf.expand_dims(row_kernel_b, -1) row_kernel = tf.transpose(row_kernel_a) * row_kernel_b row_kernel = tf.transpose(row_kernel, perm=(0, 2, 1)) row_kernel = row_constant * tf.cast(row_kernel, tf.complex64) row_kernel = tf.exp(row_kernel) return row_kernel
def call(self, inputs): kernel_shape = K.int_shape(self.kernel) if self.renormalize: w = K.reshape(self.kernel, (-1, kernel_shape[-1])) sigma, u_bar = max_singular_val( w, self.u, fully_differentiable=self.fully_diff_spectral, ip=self.spectral_iterations) else: w = ktf.transpose(self.kernel, (0, 3, 1, 2)) w = K.reshape(w, [-1, kernel_shape[1] * kernel_shape[2]]) w = K.expand_dims(w, axis=-1) sigma, u_bar = max_singular_val( w, self.u, transpose=lambda x: ktf.transpose(x, [0, 2, 1]), fully_differentiable=self.fully_diff_spectral, ip=self.spectral_iterations) sigma = K.reshape(sigma, [kernel_shape[0], 1, 1, kernel_shape[-1]]) self.add_update(K.update(self.u, u_bar)) kernel = self.kernel self.kernel = self.kernel / sigma outputs = super(SNConditionalDepthwiseConv2D, self).call(inputs) self.kernel = kernel return outputs
def _col_kernel(upsampled_region_size, upsample_factor, axis_offsets, data_shape): data_shape_float = tf.cast(data_shape, tf.float32) col_constant = tf.cast(data_shape_float[2] * upsample_factor, tf.complex64) col_constant = (-1j * 2 * np.pi / col_constant) col_kernel_a = tf.range(0, data_shape_float[2], dtype=tf.float32) col_kernel_a = fftshift1d(col_kernel_a) col_kernel_a = tf.reshape(col_kernel_a, (-1, 1)) col_kernel_a -= tf.floor(data_shape_float[2] / 2.) col_kernel_a = tf.reshape(col_kernel_a, (1, -1)) col_kernel_a = tf.tile(col_kernel_a, (data_shape[0], 1)) col_kernel_b = tf.range(0, upsampled_region_size, dtype=tf.float32) col_kernel_b = tf.reshape(col_kernel_b, (1, -1)) col_kernel_b = tf.tile(col_kernel_b, (data_shape[0], 1)) col_kernel_b = tf.transpose(col_kernel_b) col_kernel_b -= tf.transpose(axis_offsets[:, 1]) col_kernel_b = tf.transpose(col_kernel_b) col_kernel_a = tf.expand_dims(col_kernel_a, 1) col_kernel_b = tf.expand_dims(col_kernel_b, -1) col_kernel = col_kernel_a * col_kernel_b col_kernel = tf.transpose(col_kernel, perm=(0, 2, 1)) col_kernel = col_constant * tf.cast(col_kernel, tf.complex64) col_kernel = tf.exp(col_kernel) return col_kernel
def call(self, inputs): print("xxxx",inputs) expanded_tensor = ktf.expand_dims(inputs[0], -1) multiples = [1, self.number_of_transforms, 1, 1, 1] tiled_tensor = ktf.tile(expanded_tensor, multiples=multiples) repeated_tensor = ktf.reshape(tiled_tensor, ktf.shape(inputs[0]) * np.array([self.number_of_transforms, 1, 1, 1])) affine_transforms = inputs[1] / self.affine_mul affine_transforms = ktf.reshape(affine_transforms, (-1, 8)) tranformed = tf_affine_transform(repeated_tensor, affine_transforms) res = ktf.reshape(tranformed, [-1, self.number_of_transforms] + self.image_size) res = ktf.transpose(res, [0, 2, 3, 1, 4]) #Use masks if len(inputs) == 3: mask = ktf.transpose(inputs[2], [0, 2, 3, 1]) mask = ktf.image.resize_images(mask, self.image_size[:2], method=ktf.image.ResizeMethod.NEAREST_NEIGHBOR) res = res * ktf.expand_dims(mask, axis=-1) if self.aggregation_fn == 'none': res = ktf.reshape(res, [-1] + self.image_size[:2] + [self.image_size[2] * self.number_of_transforms]) elif self.aggregation_fn == 'max': res = ktf.reduce_max(res, reduction_indices=[-2]) elif self.aggregation_fn == 'avg': counts = ktf.reduce_sum(mask, reduction_indices=[-1]) counts = ktf.expand_dims(counts, axis=-1) res = ktf.reduce_sum(res, reduction_indices=[-2]) res /= counts res = ktf.where(ktf.is_nan(res), ktf.zeros_like(res), res) return res
def get_image_emotion(image, net): net = load_model(net, compile=True) image = ktf.transpose(image, [0, 2, 3, 1]) image = ktf.image.resize_images(image, (299, 299), ) image = ktf.transpose(image, [0, 3, 1, 2]) score = net(image) return score[:, 1]
def get_image_memorability(image, net='models/mem_internal.h5'): memnet = load_model(net, custom_objects={'LRN': LRN}, compile=True) image *= 255 image = ktf.transpose(image, [0, 2, 3, 1]) image = ktf.image.resize_images(image, (227, 227), ) image = ktf.transpose(image, [0, 3, 1, 2]) image = preprocess_symbolic_input(image, data_format='channels_first', mode='caffe') score = memnet(image) return score
def call(self, input_tensor): """ target: batch x ih x iw (queryL=ihxiw) x idf source: batch x sourceL(seq_len) x idf mask: batch x sourceL -inf or 0 が入っている """ target, source, mask = input_tensor idf = self.output_dim ih = self.ih iw = self.iw queryL = self.queryL sourceL = self.sourceL # --> batch x queryL x idf target = K.reshape(target, (-1, queryL, idf)) # --> batch x idf x sourceL sourceT = ktf.transpose(source, perm=[0, 2, 1]) # Get attention # (batch x queryL x idf)(batch x idf x sourceL) # -->batch x queryL x sourceL attn = ktf.matmul(target, sourceT) addmask = K.switch( self.use_mask, lambda: K.repeat_elements( K.reshape(mask, (-1, 1, sourceL)), rep=queryL, axis=1), lambda: 0.0) attn = attn + addmask attn = K.softmax(attn) # (batch x queryL x sourceL)(batch x sourceL x idf) # --> batch x queryL x idf weightedContext = ktf.matmul(attn, source) weightedContext = K.reshape(weightedContext, (-1, ih, iw, idf)) attn = K.reshape(attn, (-1, ih, iw, sourceL)) #計算ではこの後未使用 return [weightedContext, attn]
def call(self, inputs): kernel_shape = K.int_shape(self.kernel) if not self.renormalize: w = K.reshape(self.kernel, (kernel_shape[0], kernel_shape[1] * kernel_shape[2] * kernel_shape[3], kernel_shape[-1])) sigma, u_bar = max_singular_val( w, self.u, transpose=lambda x: ktf.transpose(x, [0, 2, 1]), fully_differentiable=self.fully_diff_spectral, ip=self.spectral_iterations) sigma = K.reshape(sigma, (self.filters_emb, 1, 1, 1, 1)) else: w = K.reshape(self.kernel, (-1, kernel_shape[-1])) sigma, u_bar = max_singular_val( w, self.u, fully_differentiable=self.fully_diff_spectral, ip=self.spectral_iterations) self.add_update(K.update(self.u, u_bar)) kernel = self.kernel self.kernel = self.kernel / sigma outputs = super(SNFactorizedConv11, self).call(inputs) self.kernel = kernel return outputs
def call(self, x): x_shape = x.get_shape() offsets = super(Pool2DOffset, self).call(x) offsets = tf.transpose(offsets, [0, 3, 1, 2]) offsets = tf.reshape(offsets, (-1, int(x_shape[1]), int(x_shape[2]), 2)) n_batches = tf.shape(offsets)[0] x = tf.transpose(x, [0, 3, 1, 2]) x = tf.reshape(x, (-1, int(x_shape[1]), int(x_shape[2]), 1)) #offsets = tf.resampler(x, offsets) x = batch_map_offsets(x, offsets) x = tf.reshape(x, (-1, int(x_shape[3]), int(x_shape[1]), int(x_shape[2]))) x = tf.transpose(x, [0, 2, 3, 1]) return x
def call(self, inputs): w = self.kernel kernel_shape = K.int_shape(self.kernel) if self.renormalize: w = K.reshape(w, [-1, kernel_shape[-1]]) sigma, u_bar = max_singular_val( w, self.u, fully_differentiable=self.fully_diff_spectral, ip=self.spectral_iterations) else: sigma, u_bar = max_singular_val( w, self.u, transpose=lambda x: ktf.transpose(x, [0, 2, 1]), fully_differentiable=self.fully_diff_spectral, ip=self.spectral_iterations) sigma = K.reshape(sigma, (self.number_of_classes, 1, 1)) self.add_update(K.update(self.u, u_bar)) kernel = self.kernel self.kernel = self.kernel / sigma outputs = super(SNCondtionalDense, self).call(inputs) self.kernel = kernel return outputs
def get_image_aesthetics(image, net='models/ava1.h5', ava1_mean=True): ilgnet = load_model(net, custom_objects={'LRN': LRN}, compile=True) if not ava1_mean: IMAGENET_MEAN = K.constant(-np.array([90.05038554687053, 98.58527740451973, 107.50910979753826])[::-1]) else: IMAGENET_MEAN = K.constant(-np.array([91.00178961467464, 99.46385127608664, 107.8456162486691])[::-1]) image *= 255 image = ktf.transpose(image, [0, 2, 3, 1]) image = ktf.image.resize_images(image, (227, 227), ) image = ktf.transpose(image, [0, 3, 1, 2]) image = preprocess_symbolic_input(image, data_format='channels_first', mode='caffe', IMAGENET_MEAN=IMAGENET_MEAN) score = ilgnet(image) return score[:, 1]
def call(self, inputs): mask = ktf.transpose(inputs[2], [0, 2, 3, 1]) # mask = ktf.image.resize_images(mask, self.image_size[:2], method=ktf.image.ResizeMethod.NEAREST_NEIGHBOR) if self.number_of_transforms == 11: # return inputs[0] * K.tile(ktf.expand_dims(1-mask[:,:,:,0], axis=3),(1,1,1,inputs[0].shape[-1])) return inputs[0] * ktf.expand_dims(1 - mask[:, :, :, 0], axis=3) else: return inputs[0] * ktf.expand_dims( K.clip(1 - K.sum(mask, 3), 0, 1), axis=3)
def upsampling_from_mask(inputs): X_, A_, I_, M_ = inputs S_ = tf.eye(tf.shape(M_)[0]) S_ = tf.boolean_mask(S_, M_) S_t_ = tf.transpose(S_) X_out_ = K.dot(S_t_, X_) A_out_ = K.dot(K.transpose(K.dot(A_, S_)), S_) I_out_ = K.dot(S_t_, K.cast(I_[:, None], tf.float32))[:, 0] I_out_ = K.cast(I_out_, tf.int32) return [X_out_, A_out_, I_out_]
def _postprocess_conv2d_output(x, data_format): """Transpose and cast the output from conv2d if needed. # Arguments x: A tensor. data_format: string, `"channels_last"` or `"channels_first"`. # Returns A tensor. """ if data_format == 'channels_first': x = tf.transpose(x, (0, 3, 1, 2)) if floatx() == 'float64': x = tf.cast(x, 'float64') return x
def _preprocess_conv2d_input(x, data_format): """Transpose and cast the input before the conv2d. # Arguments x: input tensor. data_format: string, `"channels_last"` or `"channels_first"`. # Returns A tensor. """ if dtype(x) == 'float64': x = tf.cast(x, 'float32') if data_format == 'channels_first': # TF uses the last dimension as channel dimension, # instead of the 2nd one. # TH input shape: (samples, input_depth, rows, cols) # TF input shape: (samples, rows, cols, input_depth) x = tf.transpose(x, (0, 2, 3, 1)) return x
def call(self, x): x_shape = x.get_shape() offsets = super(Conv2DOffset, self).call(x) #offsets *= 10 channels = int(offsets.get_shape()[3].value) n_batches = tf.shape(offsets)[0] # Change offset's order from [x1, x2, ..., y1, y2, ...] to [x1, y1, x2, y2, ...] # Codes below are written to make sure same results of MXNet implementation. # You can remove them, and it won't influence the module's performance. ind_shuffle = tf.concat( [tf.range(0, channels, 2), tf.range(1, channels + 1, 2)], axis=0) #ind_shuffle = tf.expand_dims(ind_shuffle, axis=0) #ind_shuffle = tf.expand_dims(ind_shuffle, axis=0) #ind_shuffle = tf.tile(ind_shuffle, [input_w, input_h, 1]) offsets = tf.gather(offsets, ind_shuffle, axis=3) # ------------------------------------------------------------------------ #x = tf.transpose(x, [0, 3, 1, 2]) #x = tf.reshape(x, (-1, int(x_shape[1]), int(x_shape[2]))) #offsets = tf.resampler(x, offsets) offsets = batch_map_offsets(x, offsets) #offsets = tf.reshape(x, (-1, int(x_shape[3]), int(x_shape[1]), int(x_shape[2]))) #offsets = tf.transpose(x, [0, 2, 3, 1]) offset_shape = offsets.get_shape() num_channels = offset_shape[1].value height = offset_shape[2].value width = offset_shape[3].value f_offset = [ tf.reshape(offsets[..., ind:ind + 3], (-1, num_channels, height, width * 3)) for ind in range(0, 9, 3) ] f_offset = tf.concat(f_offset, axis=-1) f_offset = tf.reshape(f_offset, (-1, num_channels, height * 3, width * 3)) f_offset = tf.transpose(f_offset, (0, 2, 3, 1)) return f_offset
def _get_vals_by_coords(input, coords, n_coords): coords_shape = tf.shape(coords) input_shape = input.get_shape() input_w = input_shape[2].value input_h = input_shape[1].value channel_size = input_shape[3].value batch_size = tf.shape(input)[0] input = tf.transpose(input, (0, 3, 1, 2)) input = tf.reshape(input, (-1, channel_size, input_h * input_w)) indices = coords[..., 0] * input_w + coords[..., 1] #indices = tf.expand_dims(indices, axis=1) #indices = tf.tile(indices, [1, channel_size, 1, 1, 1]) #indices = tf.reshape(indices, (-1, channel_size, input_h * input_w * n_coords)) #indices = tf.transpose(indices, (0, 3, 1, 2)) indices = tf.reshape(indices, (-1, input_h * input_w * n_coords)) indices = tf.cast(indices, 'int32') #indices = tf.reshape(indices, [-1]) #input = tf.reshape(input, [-1]) vals = tf.gather(input, indices[0], axis=-1) #vals = tf.map_fn(lambda x: tf.gather(x[0], x[1], axis=-1), (input,indices), dtype=tf.float32) vals = tf.reshape(vals, (-1, channel_size, input_h, input_w, n_coords)) return vals
def call(self, inputs, training=None): input_shape = K.int_shape(inputs) _, w, h, c = input_shape bs = K.shape(inputs)[0] #if c < self.group: # raise ValueError('Input channels should be larger than group size' + # '; Received input channels: ' + str(c) + # '; Group size: ' + str(self.group) # ) #x = K.reshape(inputs, (batch_size, h, w, self.group, c // self.group)) x_t = ktf.transpose(inputs, (3, 0, 1, 2)) #x_t = ktf.transpose(x, (3, 4, 0, 1, 2)) # BxCxHxW -> CxB*H*W x_flat = ktf.reshape(x_t, (c, -1)) # Covariance m = ktf.reduce_mean(x_flat, axis=1, keepdims=True) m = K.in_train_phase(m, self.moving_mean) f = x_flat - m if self.decomposition == 'cholesky': def get_inv_sqrt(ff): sqrt = ktf.cholesky(ff) inv_sqrt = ktf.matrix_triangular_solve(sqrt, ktf.eye(c)) return sqrt, inv_sqrt elif self.decomposition in ['zca', 'zca-cor']: def get_inv_sqrt(ff): with ktf.device('/cpu:0'): S, U, _ = ktf.svd(ff + ktf.eye(c) * self.epsilon, full_matrices=True) D = ktf.diag(ktf.pow(S, -0.5)) inv_sqrt = ktf.matmul(ktf.matmul(U, D), U, transpose_b=True) D = ktf.diag(ktf.pow(S, 0.5)) sqrt = ktf.matmul(ktf.matmul(U, D), U, transpose_b=True) return sqrt, inv_sqrt elif self.decomposition in ['pca', 'pca-cor']: def get_inv_sqrt(ff): with ktf.device('/cpu:0'): S, U, _ = ktf.svd(ff + ktf.eye(c) * self.epsilon, full_matrices=True) D = ktf.diag(ktf.pow(S, -0.5)) inv_sqrt = ktf.matmul(D, U, transpose_b=True) D = ktf.diag(ktf.pow(S, 0.5)) sqrt = ktf.matmul(D, U, transpose_b=True) return sqrt, inv_sqrt else: assert False def train(): ff_apr = ktf.matmul(f, f, transpose_b=True) / ( ktf.cast(bs * w * h, ktf.float32) - 1.) if self.decomposition in ['pca-cor', 'zca-cor']: dinv = ktf.diag(ktf.sqrt(ktf.diag_part(ff_apr))) ff_apr = ktf.matmul(ktf.matmul(dinv, ff_apr), ktf.matrix_inverse(dinv), transpose_b=True) self.add_update([ K.moving_average_update(self.moving_mean, m, self.momentum), K.moving_average_update(self.moving_cov, ff_apr, self.momentum) ], inputs) ff_apr_shrinked = ( 1 - self.epsilon) * ff_apr + ktf.eye(c) * self.epsilon if self.renorm: l, l_inv = get_inv_sqrt(ff_apr_shrinked) ff_mov = (1 - self.epsilon ) * self.moving_cov + ktf.eye(c) * self.epsilon _, l_mov_inverse = get_inv_sqrt(ff_mov) l_ndiff = K.stop_gradient(l) return ktf.matmul(ktf.matmul(l_mov_inverse, l_ndiff), l_inv) return get_inv_sqrt(ff_apr_shrinked)[1] def test(): ff_mov = ( 1 - self.epsilon) * self.moving_cov + ktf.eye(c) * self.epsilon return get_inv_sqrt(ff_mov)[1] inv_sqrt = K.in_train_phase(train, test) f_hat = ktf.matmul(inv_sqrt, f) decorelated = K.reshape(f_hat, [c, bs, w, h]) decorelated = ktf.transpose(decorelated, [1, 2, 3, 0]) broadcast_shape = [1] * len(input_shape) if self.axis is not None: broadcast_shape[self.axis] = input_shape[self.axis] if self.scale: broadcast_gamma = K.reshape(self.gamma, broadcast_shape) decorelated = decorelated * broadcast_gamma if self.center: broadcast_beta = K.reshape(self.beta, broadcast_shape) decorelated = decorelated + broadcast_beta return decorelated