def call(self, inputs, **kwargs): boxes = K.stop_gradient(inputs[0]) fpn = K.stop_gradient(inputs[1]) time_distributed = K.ndim(boxes) == 4 if time_distributed: boxes_shape = K.shape(boxes) fpn_shape = K.shape(fpn) new_boxes_shape = [-1] + [ boxes_shape[i] for i in range(2, K.ndim(boxes)) ] new_fpn_shape = [-1 ] + [fpn_shape[i] for i in range(2, K.ndim(fpn))] boxes = K.reshape(boxes, new_boxes_shape) fpn = K.reshape(fpn, new_fpn_shape) image_shape = K.cast(K.shape(fpn), K.floatx()) def _roi_align(args): boxes = args[0] fpn = args[1] # process the feature map x1 = boxes[:, 0] y1 = boxes[:, 1] x2 = boxes[:, 2] y2 = boxes[:, 3] fpn_shape = K.cast(K.shape(fpn), dtype=K.floatx()) norm_boxes = K.stack([ (y1 / image_shape[1] * fpn_shape[0]) / (fpn_shape[0] - 1), (x1 / image_shape[2] * fpn_shape[1]) / (fpn_shape[1] - 1), (y2 / image_shape[1] * fpn_shape[0] - 1) / (fpn_shape[0] - 1), (x2 / image_shape[2] * fpn_shape[1] - 1) / (fpn_shape[1] - 1) ], axis=1) rois = tf.image.crop_and_resize( K.expand_dims(fpn, axis=0), norm_boxes, tf.zeros((K.shape(norm_boxes)[0], ), dtype='int32'), self.crop_size) return rois roi_batch = tf.map_fn(_roi_align, elems=[boxes, fpn], dtype=K.floatx(), parallel_iterations=self.parallel_iterations) if time_distributed: roi_shape = tf.shape(roi_batch) new_roi_shape = [boxes_shape[0], boxes_shape[1]] + \ [roi_shape[i] for i in range(1, K.ndim(roi_batch))] roi_batch = tf.reshape(roi_batch, new_roi_shape) return roi_batch
def call(self, inputs): _, kernel_b = xnorize(self.kernel, self.H) _, inputs_b = xnorize(inputs) outputs = K.conv2d(inputs_b, kernel_b, strides=self.strides, padding=self.padding, data_format=self.data_format, dilation_rate=self.dilation_rate) # calculate Wa and xa # kernel_a mask = K.reshape( self.kernel, (-1, self.filters)) # self.nb_row * self.nb_col * channels, filters kernel_a = K.stop_gradient(K.mean(K.abs(mask), axis=0)) # filters # inputs_a if self.data_format == 'channels_first': channel_axis = 1 else: channel_axis = -1 mask = K.mean(K.abs(inputs), axis=channel_axis, keepdims=True) ones = K.ones(self.kernel_size + (1, 1)) inputs_a = K.conv2d(mask, ones, strides=self.strides, padding=self.padding, data_format=self.data_format, dilation_rate=self.dilation_rate ) # nb_sample, 1, new_nb_row, new_nb_col if self.data_format == 'channels_first': outputs = outputs * K.stop_gradient(inputs_a) * K.expand_dims( K.expand_dims(K.expand_dims(kernel_a, 0), -1), -1) else: outputs = outputs * K.stop_gradient(inputs_a) * K.expand_dims( K.expand_dims(K.expand_dims(kernel_a, 0), 0), 0) if self.use_bias: outputs = K.bias_add(outputs, self.bias, data_format=self.data_format) if self.activation is not None: return self.activation(outputs) return outputs
def train(): ff_aprs = utils.get_group_cov(f, self.group, self.m_per_group, self.instance_norm, bs, w, h, c) if self.instance_norm: ff_aprs = tf.transpose(ff_aprs, (1, 0, 2, 3)) ff_aprs = (1 - self.epsilon) * ff_aprs + tf.expand_dims( tf.expand_dims(tf.eye(self.m_per_group) * self.epsilon, 0), 0) else: ff_aprs = (1 - self.epsilon) * ff_aprs + tf.expand_dims( tf.eye(self.m_per_group) * self.epsilon, 0) whitten_matrix = get_inv_sqrt(ff_aprs, self.m_per_group)[1] self.add_update([ K.moving_average_update(self.moving_mean, m, self.momentum), K.moving_average_update( self.moving_matrix, whitten_matrix if '_wm' in self.decomposition else ff_aprs, self.momentum) ], inputs) if self.renorm: l, l_inv = get_inv_sqrt(ff_aprs, self.m_per_group) ff_mov = (1 - self.epsilon) * self.moving_matrix + tf.eye( self.m_per_group) * self.epsilon _, l_mov_inverse = get_inv_sqrt(ff_mov, self.m_per_group) l_ndiff = K.stop_gradient(l) return tf.matmul(tf.matmul(l_mov_inverse, l_ndiff), l_inv) return whitten_matrix
def get_updates(self, loss, params): grads = self.get_gradients(loss, params) self.updates = [K.update_add(self.iterations, 1)] lr = self.lr if self.initial_decay > 0: lr = lr * (1. / (1. + self.decay * K.cast(self.iterations, K.dtype(self.decay)))) t = K.cast(self.iterations, K.floatx()) + 1 # Applies bounds on actual learning rate step_size = lr * (K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t))) final_lr = self.final_lr * lr / self.base_lr lower_bound = final_lr * (1. - 1. / (self.gamma * t + 1.)) upper_bound = final_lr * (1. + 1. / (self.gamma * t)) ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] if self.amsbound: vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] else: vhats = [K.zeros(1) for _ in params] self.weights = [self.iterations] + ms + vs + vhats for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats): # apply weight decay if self.weight_decay != 0.: g += self.weight_decay * K.stop_gradient(p) m_t = (self.beta_1 * m) + (1. - self.beta_1) * g v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g) if self.amsbound: vhat_t = K.maximum(vhat, v_t) denom = (K.sqrt(vhat_t) + self.epsilon) self.updates.append(K.update(vhat, vhat_t)) else: denom = (K.sqrt(v_t) + self.epsilon) # Compute the bounds step_size_p = step_size * K.ones_like(denom) step_size_p_bound = step_size_p / denom bounded_lr_t = m_t * K.minimum( K.maximum(step_size_p_bound, lower_bound), upper_bound) p_t = p - bounded_lr_t self.updates.append(K.update(m, m_t)) self.updates.append(K.update(v, v_t)) new_p = p_t # Apply constraints. if getattr(p, 'constraint', None) is not None: new_p = p.constraint(new_p) self.updates.append(K.update(p, new_p)) return self.updates
def call(self, inputs, training=None, mask=None): q = inputs[0] k = inputs[1] v = inputs[2] qkTensor = K.math_ops.matmul(q,k,transpose_b=True) scaleTensor = K.math_ops.multiply(K.stop_gradient(1. / K.math_ops.sqrt(self.dk)) , qkTensor) softMaxTensor = K.softmax(scaleTensor) drT = self.dropout(softMaxTensor,training = training) vTensor = K.math_ops.matmul(drT,v) return vTensor
def style_loss(style, combination, mask_path=None, nb_channels=None): assert K.ndim(style) == 3 assert K.ndim(combination) == 3 if content_mask_path is not None: content_mask = K.variable(load_mask(content_mask_path, nb_channels)) combination = combination * K.stop_gradient(content_mask) del content_mask if mask_path is not None: style_mask = K.variable(load_mask(mask_path, nb_channels)) style = style * K.stop_gradient(style_mask) if content_mask_path is None: combination = combination * K.stop_gradient(style_mask) del style_mask S = gram_matrix(style) C = gram_matrix(combination) channels = 3 size = img_width * img_height return K.sum(K.square(S - C)) / (4. * (channels ** 2) * (size ** 2))
def call(self, inputs, training=None, mask=None): sentence = inputs[0] position = inputs[1] batchTensor = tf.stop_gradient(tf.nn.embedding_lookup(params=self.iniEmbeddingMatrix, ids=sentence)) batchTensor = K.stop_gradient(K.math_ops.multiply(batchTensor,K.math_ops.sqrt(self.dModel))) positionTensor = self.positionEmbeddingMatrix(position) positionTensor = K.math_ops.multiply(positionTensor,K.stop_gradient(K.math_ops.sqrt(self.dModel))) # print(batchTensor.shape) # print(positionTensor.shape) eDropTensor = self.embeddingDrop(K.math_ops.multiply(K.math_ops.add(batchTensor,positionTensor), K.stop_gradient(tf.convert_to_tensor(1. / 2.,dtype=tf.float32))), training = training) denseTrans = self.denseTrans(eDropTensor) thisTransformer = K.identity(denseTrans) for i in range(self._transformerLayers): thisTransformer = self.transformerList[i](thisTransformer,training=training) flattenTensor = self.flat(thisTransformer) dense1Tensor = self.dense1(flattenTensor) return tf.nn.softmax(dense1Tensor,axis=-1)
def max_singular_val(w, u, fully_differentiable=False, ip=1): if not fully_differentiable: w_ = K.stop_gradient(w) else: w_ = w u = K.expand_dims(u, axis=-1) u_bar = u for _ in range(ip): v_bar = tf.matmul(w_, u_bar, transpose_a=True) v_bar = K.l2_normalize(v_bar, axis=(-1, -2)) u_bar_raw = tf.matmul(w_, v_bar) u_bar = K.l2_normalize(u_bar_raw, axis=(-1, -2)) sigma = tf.matmul(u_bar, tf.matmul(w, v_bar), transpose_a=True) sigma = K.squeeze(sigma, axis=-1) sigma = K.squeeze(sigma, axis=-1) u_bar = K.squeeze(u_bar, axis=-1) return sigma, u_bar
def max_singular_val_for_convolution(w, u, fully_differentiable=False, ip=1, padding='same', strides=(1, 1), data_format='channels_last'): assert ip >= 1 if not fully_differentiable: w_ = K.stop_gradient(w) else: w_ = w u_bar = u for _ in range(ip): v_bar = K.conv2d(u_bar, w_, strides=strides, data_format=data_format, padding=padding) v_bar = K.l2_normalize(v_bar) u_bar_raw = K.conv2d_transpose(v_bar, w_, output_shape=K.int_shape(u), strides=strides, data_format=data_format, padding=padding) u_bar = K.l2_normalize(u_bar_raw) u_bar_raw_diff = K.conv2d_transpose(v_bar, w, output_shape=K.int_shape(u), strides=strides, data_format=data_format, padding=padding) sigma = K.sum(u_bar * u_bar_raw_diff) return sigma, u_bar
def call(self, inputs, **kwargs): image_shape = K.cast(inputs[0], K.floatx()) boxes = K.stop_gradient(inputs[1]) scores = K.stop_gradient(inputs[2]) fpn = [K.stop_gradient(i) for i in inputs[3:]] time_distributed = K.ndim(boxes) == 4 if time_distributed: image_shape = image_shape[1:] boxes_shape = tf.shape(boxes) scores_shape = tf.shape(scores) fpn_shape = [tf.shape(f) for f in fpn] new_boxes_shape = [-1] + [ boxes_shape[i] for i in range(2, K.ndim(boxes)) ] new_scores_shape = [-1] + [ scores_shape[i] for i in range(2, K.ndim(scores)) ] new_fpn_shape = [[-1] + [f_s[i] for i in range(2, K.ndim(f))] for f, f_s in zip(fpn, fpn_shape)] boxes = tf.reshape(boxes, new_boxes_shape) scores = tf.reshape(scores, new_scores_shape) fpn = [tf.reshape(f, f_s) for f, f_s in zip(fpn, new_fpn_shape)] def _roi_align(args): boxes = args[0] scores = args[1] fpn = args[2] # compute from which level to get features from target_levels = self.map_to_level(boxes) # process each pyramid independently rois, ordered_indices = [], [] for i in range(len(fpn)): # select the boxes and classification from this pyramid level indices = tf.where(K.equal(target_levels, i)) ordered_indices.append(indices) level_boxes = tf.gather_nd(boxes, indices) fpn_shape = K.cast(K.shape(fpn[i]), dtype=K.floatx()) # convert to expected format for crop_and_resize x1 = level_boxes[:, 0] y1 = level_boxes[:, 1] x2 = level_boxes[:, 2] y2 = level_boxes[:, 3] level_boxes = K.stack([ (y1 / image_shape[1] * fpn_shape[0]) / (fpn_shape[0] - 1), (x1 / image_shape[2] * fpn_shape[1]) / (fpn_shape[1] - 1), (y2 / image_shape[1] * fpn_shape[0] - 1) / (fpn_shape[0] - 1), (x2 / image_shape[2] * fpn_shape[1] - 1) / (fpn_shape[1] - 1), ], axis=1) # append the rois to the list of rois rois.append( tf.image.crop_and_resize( K.expand_dims(fpn[i], axis=0), level_boxes, tf.zeros((K.shape(level_boxes)[0], ), dtype='int32'), self.crop_size)) # concatenate rois to one blob rois = K.concatenate(rois, axis=0) # reorder rois back to original order indices = K.concatenate(ordered_indices, axis=0) rois = tf.scatter_nd(indices, rois, K.cast(K.shape(rois), 'int64')) return rois roi_batch = tf.map_fn(_roi_align, elems=[boxes, scores, fpn], dtype=K.floatx(), parallel_iterations=self.parallel_iterations) if time_distributed: roi_shape = tf.shape(roi_batch) new_roi_shape = [boxes_shape[0], boxes_shape[1]] + \ [roi_shape[i] for i in range(1, K.ndim(roi_batch))] roi_batch = tf.reshape(roi_batch, new_roi_shape) return roi_batch
def call(self, inputs, **kwargs): image_shape = K.cast(inputs[0], K.floatx()) boxes = K.stop_gradient(inputs[1]) scores = K.stop_gradient(inputs[2]) fpn = [K.stop_gradient(i) for i in inputs[3:]] def _roi_align(args): boxes = args[0] scores = args[1] fpn = args[2] # compute from which level to get features from target_levels = self.map_to_level(boxes) # process each pyramid independently rois, ordered_indices = [], [] for i in range(len(fpn)): # select the boxes and classification from this pyramid level indices = tf.where(K.equal(target_levels, i)) ordered_indices.append(indices) level_boxes = tf.gather_nd(boxes, indices) fpn_shape = K.cast(K.shape(fpn[i]), dtype=K.floatx()) # convert to expected format for crop_and_resize x1 = level_boxes[:, 0] y1 = level_boxes[:, 1] x2 = level_boxes[:, 2] y2 = level_boxes[:, 3] level_boxes = K.stack([ (y1 / image_shape[1] * fpn_shape[0]) / (fpn_shape[0] - 1), (x1 / image_shape[2] * fpn_shape[1]) / (fpn_shape[1] - 1), (y2 / image_shape[1] * fpn_shape[0] - 1) / (fpn_shape[0] - 1), (x2 / image_shape[2] * fpn_shape[1] - 1) / (fpn_shape[1] - 1), ], axis=1) if(len(fpn[i].get_shape()) >=4): unstack = tf.unstack(fpn[i], axis=3) temp_stack=[] for j in unstack: temp = tf.image.crop_and_resize( K.expand_dims(j, axis=3), level_boxes, tf.zeros((K.shape(level_boxes)[0],), dtype='int32'), (self.crop_size[0], self.crop_size[1])) temp_stack.append(temp) rois.append(temp_stack) else: rois.append(tf.image.crop_and_resize( K.expand_dims(fpn[i], axis=0), level_boxes, tf.zeros((K.shape(level_boxes)[0],), dtype='int32'), self.crop_size )) # concatenate rois to one blob rois = K.concatenate(rois, axis=0) # reorder rois back to original order indices = K.concatenate(ordered_indices, axis=0) rois = tf.scatter_nd(indices, rois, K.cast(K.shape(rois), 'int64')) return rois roi_batch = tf.map_fn( _roi_align, elems=[boxes, scores, fpn], dtype=K.floatx(), parallel_iterations=self.parallel_iterations ) return roi_batch
def _mean_abs(x, axis=None, keepdims=False): return K.stop_gradient(K.mean(K.abs(x), axis=axis, keepdims=keepdims))
def round_through(x): '''Element-wise rounding to the closest integer with full gradient propagation. A trick from [Sergey Ioffe](http://stackoverflow.com/a/36480182) ''' rounded = K.round(x) return x + K.stop_gradient(rounded - x)