def test_map(self): x = np.random.rand(10, 3).astype(np.float32) for K in [KTF, KTH]: kx = K.eval(K.map_fn(K.sum, x)) assert (10,) == kx.shape assert_allclose(x.sum(axis=1), kx, atol=1e-05)
def test_map(self): x = np.random.rand(10, 3).astype(np.float32) for K in [KTF, KTH]: vx = K.variable(x) kx = K.eval(K.map_fn(K.sum, vx)) # make sure we can also walk the indexes in tensorflow which we # can't without specifying dtype kx2 = K.eval(K.map_fn( lambda i: K.sum(vx[i]), K.arange(10), dtype=K.floatx() )) assert (10,) == kx.shape assert (10,) == kx2.shape assert_allclose(x.sum(axis=1), kx, atol=1e-05) assert_allclose(kx, kx2, atol=1e-05)
def model_discriminator(global_shape=(256, 256, 3), local_shape=(128, 128, 3)): def crop_image(img, crop): return tf.image.crop_to_bounding_box(img, crop[1], crop[0], crop[3] - crop[1], crop[2] - crop[0]) in_pts = Input(shape=(4,), dtype='int32') cropping = Lambda(lambda x: K.map_fn(lambda y: crop_image(y[0], y[1]), elems=x, dtype=tf.float32), output_shape=local_shape) g_img = Input(shape=global_shape) l_img = cropping([g_img, in_pts]) # Local Discriminator x_l = Conv2D(64, kernel_size=5, strides=2, padding='same')(l_img) x_l = BatchNormalization()(x_l) x_l = Activation('relu')(x_l) x_l = Conv2D(128, kernel_size=5, strides=2, padding='same')(x_l) x_l = BatchNormalization()(x_l) x_l = Activation('relu')(x_l) x_l = Conv2D(256, kernel_size=5, strides=2, padding='same')(x_l) x_l = BatchNormalization()(x_l) x_l = Activation('relu')(x_l) x_l = Conv2D(512, kernel_size=5, strides=2, padding='same')(x_l) x_l = BatchNormalization()(x_l) x_l = Activation('relu')(x_l) x_l = Conv2D(512, kernel_size=5, strides=2, padding='same')(x_l) x_l = BatchNormalization()(x_l) x_l = Activation('relu')(x_l) x_l = Flatten()(x_l) x_l = Dense(1024, activation='relu')(x_l) # Global Discriminator x_g = Conv2D(64, kernel_size=5, strides=2, padding='same')(g_img) x_g = BatchNormalization()(x_g) x_g = Activation('relu')(x_g) x_g = Conv2D(128, kernel_size=5, strides=2, padding='same')(x_g) x_g = BatchNormalization()(x_g) x_g = Activation('relu')(x_g) x_g = Conv2D(256, kernel_size=5, strides=2, padding='same')(x_g) x_g = BatchNormalization()(x_g) x_g = Activation('relu')(x_g) x_g = Conv2D(512, kernel_size=5, strides=2, padding='same')(x_g) x_g = BatchNormalization()(x_g) x_g = Activation('relu')(x_g) x_g = Conv2D(512, kernel_size=5, strides=2, padding='same')(x_g) x_g = BatchNormalization()(x_g) x_g = Activation('relu')(x_g) x_g = Conv2D(512, kernel_size=5, strides=2, padding='same')(x_g) x_g = BatchNormalization()(x_g) x_g = Activation('relu')(x_g) x_g = Flatten()(x_g) x_g = Dense(1024, activation='relu')(x_g) x = concatenate([x_l, x_g]) x = Dense(1, activation='sigmoid')(x) return Model(inputs=[g_img, in_pts], outputs=x)
def call(self, inputs, training=None): # inputs.shape=[None, input_num_capsule, input_dim_capsule] # inputs_expand.shape=[None, 1, input_num_capsule, input_dim_capsule] inputs_expand = K.expand_dims(inputs, 1) # Replicate num_capsule dimension to prepare being multiplied by W # inputs_tiled.shape=[None, num_capsule, input_num_capsule, input_dim_capsule] inputs_tiled = K.tile(inputs_expand, [1, self.num_capsule, 1, 1]) # Compute `inputs * W` by scanning inputs_tiled on dimension 0. # x.shape=[num_capsule, input_num_capsule, input_dim_capsule] # W.shape=[num_capsule, input_num_capsule, dim_capsule, input_dim_capsule] # Regard the first two dimensions as `batch` dimension, # then matmul: [input_dim_capsule] x [dim_capsule, input_dim_capsule]^T -> [dim_capsule]. # inputs_hat.shape = [None, num_capsule, input_num_capsule, dim_capsule] inputs_hat = K.map_fn(lambda x: K.batch_dot(x, self.W, [2, 3]), elems=inputs_tiled) # Begin: Routing algorithm ---------------------------------------------------------------------# # The prior for coupling coefficient, initialized as zeros. # b.shape = [None, self.num_capsule, self.input_num_capsule]. b = tf.zeros(shape=[K.shape(inputs_hat)[0], self.num_capsule, self.input_num_capsule]) assert self.routings > 0, 'The routings should be > 0.' for i in range(self.routings): # c.shape=[batch_size, num_capsule, input_num_capsule] c = tf.nn.softmax(b, dim=1) # c.shape = [batch_size, num_capsule, input_num_capsule] # inputs_hat.shape=[None, num_capsule, input_num_capsule, dim_capsule] # The first two dimensions as `batch` dimension, # then matmal: [input_num_capsule] x [input_num_capsule, dim_capsule] -> [dim_capsule]. # outputs.shape=[None, num_capsule, dim_capsule] outputs = squash(K.batch_dot(c, inputs_hat, [2, 2])) # [None, 10, 16] if i < self.routings - 1: # outputs.shape = [None, num_capsule, dim_capsule] # inputs_hat.shape=[None, num_capsule, input_num_capsule, dim_capsule] # The first two dimensions as `batch` dimension, # then matmal: [dim_capsule] x [input_num_capsule, dim_capsule]^T -> [input_num_capsule]. # b.shape=[batch_size, num_capsule, input_num_capsule] b += K.batch_dot(outputs, inputs_hat, [2, 3]) # End: Routing algorithm -----------------------------------------------------------------------# return outputs
def call(self, inputs, training=None): # inputs.shape=[None, input_num_capsule, input_num_instance, input_dim_capsule] # inputs_hat.shape=[None,num_instance*num_capsule,num_parts*input_num_capsule*input_num_instance,dim_capsule] inputs_hat = K.map_fn(lambda x: self._part_to_whole_predictions(x), elems=inputs) # Begin: Routing algorithm ---------------------------------------------------------------------# # The prior for coupling coefficient, initialized as zeros. # b.shape = [None, self.num_capsule, self.num_parts, self.input_num_capsule]. b = K.tf.zeros(shape=[ K.shape(inputs_hat)[0], self.num_instance * self.num_capsule, self.num_part * self.input_num_capsule * self.input_num_instance ]) assert self.routings > 0, 'The routings should be > 0.' for i in range(self.routings): # c.shape=[batch_size, num_instance*num_capsule, input_num_capsule] tmpb = K.reshape(b, [ -1, self.num_capsule * self.num_instance * self.num_part, self.input_num_capsule * self.input_num_instance ]) # softmax for all outputs of each input_capsule*input_instance tmpc = K.tf.nn.softmax(tmpb, dim=1) c = K.reshape(tmpc, [ -1, self.num_capsule * self.num_instance, self.num_part * self.input_num_capsule * self.input_num_instance ]) #outputs.shape=[None,num_instance * num_capsule,dim_capsule] outputs = self._best_guess(c, inputs_hat) if i < self.routings - 1: # b += self._agreement(outputs, inputs_hat) # End: Routing algorithm -----------------------------------------------------------------------# outputs = K.reshape( outputs, [-1, self.num_instance, self.num_capsule, self.dim_capsule]) outputs = K.permute_dimensions(outputs, [0, 2, 1, 3]) return outputs
def create(self, global_input_tensor, bbox_tensor, local_shape=(128, 128, 3)): dis_builder = _DisBuilder(activation=self.activation, loss='binary_crossentropy', metrics=['acc']) global_net = dis_builder.create( global_input_tensor, conv_n_filters=[64, 128, 256, 512, 512, 512], conv_kernel_sizes=[5, 5, 5, 5, 5, 5], conv_strides=[2, 2, 2, 2, 2, 2], out_n_filter=1024, tag='glcic_global_discriminator' ) cropping_layer = Lambda( lambda x: K.map_fn( lambda z: self.cropping(z[0], z[1]), elems=x, dtype=tf.float32 ), output_shape=local_shape, name='cropping' ) cropped_tensor = cropping_layer([global_input_tensor, bbox_tensor]) local_input_tensor = Input(shape=K.int_shape(cropped_tensor)[1:], name='local_roi') local_net = dis_builder.create( local_input_tensor, conv_n_filters=[64, 128, 256, 512, 512], conv_kernel_sizes=[5, 5, 5, 5, 5], conv_strides=[2, 2, 2, 2, 2], out_n_filter=1024, tag='glcic_local_discriminator' ) global_output_tensor = global_net(global_input_tensor) local_output_tensor = local_net(cropped_tensor) y = concatenate([global_output_tensor, local_output_tensor]) y = Dense(1, activation='sigmoid')(y) self.local_net = local_net self.global_net = global_net return Graph([global_input_tensor, bbox_tensor], y, name=self.__tag__)
def _ctdet_decode(hm, reg, wh, k=100, output_stride=4): hm = K.sigmoid(hm) hm = _nms(hm) hm_shape = K.shape(hm) reg_shape = K.shape(reg) wh_shape = K.shape(wh) batch, width, cat = hm_shape[0], hm_shape[2], hm_shape[3] hm_flat = K.reshape(hm, (batch, -1)) reg_flat = K.reshape(reg, (reg_shape[0], -1, reg_shape[-1])) wh_flat = K.reshape(wh, (wh_shape[0], -1, wh_shape[-1])) def _process_sample(args): _hm, _reg, _wh = args _scores, _inds = tf.math.top_k(_hm, k=k, sorted=True) _classes = K.cast(_inds % cat, 'float32') _inds = K.cast(_inds / cat, 'int32') _xs = K.cast(_inds % width, 'float32') _ys = K.cast(K.cast(_inds / width, 'int32'), 'float32') _wh = K.gather(_wh, _inds) _reg = K.gather(_reg, _inds) _xs = _xs + _reg[..., 0] _ys = _ys + _reg[..., 1] _x1 = _xs - _wh[..., 0] / 2 _y1 = _ys - _wh[..., 1] / 2 _x2 = _xs + _wh[..., 0] / 2 _y2 = _ys + _wh[..., 1] / 2 # rescale to image coordinates _x1 = output_stride * _x1 _y1 = output_stride * _y1 _x2 = output_stride * _x2 _y2 = output_stride * _y2 _detection = K.stack([_x1, _y1, _x2, _y2, _scores, _classes], -1) return _detection detections = K.map_fn(_process_sample, [hm_flat, reg_flat, wh_flat], dtype=K.floatx()) return detections
def __call__(self, w): w_shape = w.shape if w_shape.rank is None or w_shape.rank != 4: raise ValueError( "The weight tensor must have rank 4. " f"Received weight tensor with shape: {w_shape}" ) height, width, channels, kernels = w_shape w = backend.reshape(w, (height, width, channels * kernels)) # TODO(cpeter): Switch map_fn for a faster tf.vectorized_map once # backend.switch is supported. w = backend.map_fn( self._kernel_constraint, backend.stack(tf.unstack(w, axis=-1), axis=0), ) return backend.reshape( backend.stack(tf.unstack(w, axis=0), axis=-1), (height, width, channels, kernels), )
def __call__(self, x): #x=tf.slice(z,[0,0],[200,50]) x = ((x - K.min(x, axis=0)) / (K.max(x, axis=0) - K.min(x, axis=0))) + 0.000001 top = K.map_fn(lambda x: top_half(x), x) median = K.min(top, axis=1) mean = K.mean(x, axis=1) loss_mean = K.foldl(lambda acc, x: acc + kl_divergence(0.05, x), mean) loss_mean = 1.0 * loss_mean loss_median = K.foldl(lambda acc, x: acc + kl_divergence(0.03, x), median) loss_median = 1.0 * loss_median sparse = K.mean(x, axis=0) loss_se = K.foldl(lambda acc, x: acc + kl_divergence(0.05, x), sparse) loss_se = 1.0 * loss_se return loss_mean + loss_median + loss_se
def depth_estimate( S, ray_voxel_indices, ray_voxel_count, ray_to_occupancy_accumulated_pon, ray_to_occupancy_pon ): N, M = K.int_shape(S) return K.map_fn( lambda i: single_ray_depth_estimate( S[i, :ray_voxel_count[i]], ray_voxel_indices[i, :ray_voxel_count[i]], ray_to_occupancy_accumulated_pon, ray_to_occupancy_pon[i, :ray_voxel_count[i]], M ), K.tf.range(0, N, dtype="int32"), dtype="float32" )
def fn_cor(x): landmarks = x[0] landmarks = K.permute_dimensions(landmarks, (1,0,2)) #print('int_shape(landmarks) >>>', K.int_shape(landmarks)) negs = x[1] negs = K.permute_dimensions(negs, (1,0,3,2)) #print('int_shape(negs) >>>', K.int_shape(negs)) def fn(ii): lm = K.gather(landmarks, ii) lm = K.l2_normalize(lm, 1) #print('int_shape(lm) >>>', K.int_shape(lm)) x = K.gather(negs, ii) x = K.l2_normalize(x, 1) #print('int_shape(x) >>>', K.int_shape(x)) res = K.batch_dot(lm, x, axes=1) return res res = K.map_fn(fn, np.arange(max_num_prod), dtype='float32') res = K.permute_dimensions(res, (1,0,2)) #print('int_shape(res) >>>', K.int_shape(res)) return res
def call(self, inputs, training=None): # inputs.shape = (None, in_num_capsule, in_dim_capsule) # inputs_tiled.shape = (None, in_num_capsule, out_num_capsule, in_dim_capsule) inputs_tiled = K.repeat_elements(K.expand_dims(inputs, axis=2), self.out_num_capsule, 2) # inputs * W # x.shape = (in_num_capsule, out_num_capsule, in_dim_capsule) # W.shape = (in_num_capsule, out_num_capsule, in_dim_capsule, out_dim_capsule) # inputs_hat.shape = (None, in_num_capsule, out_num_capsule, out_dim_capsule) inputs_hat = K.map_fn(lambda x: K.batch_dot(x, self.W), elems=inputs_tiled) # Routing algorithm -----------------------------------------------------------------------# # b.shape = (None, in_num_capsule, out_num_capsule) b = K.zeros(shape=(K.shape(inputs)[0], self.in_num_capsule, self.out_num_capsule)) assert self.routings > 0, 'The routings should be > 0.' for i in range(self.routings): # c.shape = (None, in_num_capsule, out_num_capsule) c = K.softmax(b, axis=-1) # c.shape = (None, in_num_capsule, out_num_capsule) # inputs_hat.shape = (None, in_num_capsule, out_num_capsule, out_dim_capsule) # outputs.shape = (None, out_num_capsule, out_dim_capsule) outputs = squash(K.batch_dot( K.permute_dimensions(c, (0, 2, 1)), K.permute_dimensions(inputs_hat, (0, 2, 1, 3))), axis=-1) if i < self.routings - 1: # outputs.shape = (None, out_num_capsule, out_dim_capsule) # inputs_hat.shape = (None, in_num_capsule, out_num_capsule, out_dim_capsule) # b.shape = (None, in_num_capsule, out_num_capsule) b += K.permute_dimensions( K.batch_dot(outputs, K.permute_dimensions(inputs_hat, (0, 2, 3, 1))), (0, 2, 1)) # -----------------------------------------------------------------------------------------# return outputs
def call(self, inputs, training=None): # inputs.shape=[None, input_num_capsule, input_dim_capsule] # inputs_expand.shape=[None, 1, input_num_capsule, input_dim_capsule] inputs_expand = K.expand_dims(inputs, 1) # 重みWにかけられる準備をするためにカプセルを複製する. # inputs_tiled.shape=[None, num_capsule, input_num_capsule, input_dim_capsule] inputs_tiled = K.tile(inputs_expand, [1, self.num_capsule, 1, 1]) # `inputs * W`を計算 # x.shape=[num_capsule, input_num_capsule, input_dim_capsule] # W.shape=[num_capsule, input_num_capsule, dim_capsule, input_dim_capsule] # はじめの2次元をバッチ次元数としてみなす. # そして [input_dim_capsule] x [dim_capsule, input_dim_capsule]^T -> [dim_capsule]. # inputs_hat.shape = [None, num_capsule, input_num_capsule, dim_capsule] inputs_hat = K.map_fn(lambda x: K.batch_dot(x, self.W, [2, 3]), elems=inputs_tiled) # 開始: Routingアルゴリズム ---------------------------------------------------------------------# # 前処理として係数を1に初期化 # c.shape = [None, self.num_capsule, self.input_num_capsule]. c = tf.ones(shape=[K.shape(inputs_hat)[0], self.num_capsule, self.input_num_capsule]) assert self.routings > 0, 'The routings should be > 0.' for i in range(self.routings): # c.shape = [batch_size, num_capsule, input_num_capsule] # inputs_hat.shape=[None, num_capsule, input_num_capsule, dim_capsule] # The first two dimensions as `batch` dimension, # then matmal: [input_num_capsule] x [input_num_capsule, dim_capsule] -> [dim_capsule]. # outputs.shape=[None, num_capsule, dim_capsule] outputs = step(K.batch_dot(c, inputs_hat, [2, 2])) # [None, 10, 16] if i < self.routings - 1: # outputs.shape = [None, num_capsule, dim_capsule] # inputs_hat.shape=[None, num_capsule, input_num_capsule, dim_capsule] # The first two dimensions as `batch` dimension, # then matmal: [dim_capsule] x [input_num_capsule, dim_capsule]^T -> [input_num_capsule]. # b.shape=[batch_size, num_capsule, input_num_capsule] c += K.batch_dot(outputs, inputs_hat, [2, 3]) # 終了: Routingアルゴリズム -----------------------------------------------------------------------# return outputs
def call(self, inputs, mask=None): output = 0 for _ in range(self.samples): tmpmax = self.noise_ratio * inputs tmpmin = self.noise_ratio * inputs for i in range(K.ndim(inputs) - 1): tmpmax = K.max(tmpmax, axis=-1) tmpmin = K.min(tmpmin, axis=-1) tmpdiff = tmpmax - tmpmin batchshape = K.shape(inputs)[1:] batchsize = K.shape(inputs)[0] noise = K.map_fn( lambda i: K.random_normal(batchshape, stddev=tmpdiff[i]), K.arange(0, batchsize), dtype=K.dtype(inputs)) output += self._call(inputs + noise, mask=mask) return self._finalize(inputs, output / self.samples)
def call(self, inputs, mask=None): batch_size = K.shape(inputs)[0] tensor_1 = inputs[:, :-10] tensor_2 = inputs[:, -10:] pos = K.gradients(K.max(tensor_2, axis=-1), tensor_2)[0] neg = 1 - pos ## obtain W embeddings for rpos and rneg Wrpos = K.sum(K.reshape(pos, (batch_size, 10, 1)) * self.W, axis=1) Wrneg = K.reshape(neg, (batch_size, 10, 1)) * self.W ## obtain score value rU = K.dot(tensor_1, self.U) rUrpos = K.reshape(K.sum(rU * Wrpos, axis=-1), (batch_size, 1)) rUrneg = K.map_fn(lambda x: K.sum(x[1] * x[0], axis=-1), (rU, Wrneg), dtype=K.tf.float32) return K.reshape((rUrpos - rUrneg), (batch_size, 10)) #+self.bias?
def call(self, inputs, training=None): inputs_expand = K.expand_dims(inputs, 1) inputs_tiled = K.tile(inputs_expand, [1, self.num_capsule, 1, 1]) inputs_hat = K.map_fn(lambda x: K.batch_dot(x, self.W, [2, 3]), elems=inputs_tiled) inputs_hat_stopped = K.stop_gradient(inputs_hat) b = K.stop_gradient(K.sum(K.zeros_like(inputs_hat), -1)) assert self.num_routing > 0, 'The num_routing should be > 0.' for i in range(self.num_routing): c = tf.nn.softmax(b, dim=1) if i == self.num_routing - 1: outputs = squash(K.batch_dot(c, inputs_hat, [2, 2])) else: outputs = squash(K.batch_dot(c, inputs_hat_stopped, [2, 2])) b += K.batch_dot(outputs, inputs_hat_stopped, [2, 3]) return outputs
def call(self, inputs, training=None): # inputs.shape=[None, input_num_capsule, input_dim_capsule] # inputs_expand.shape=[None, 1, input_num_capsule, input_dim_capsule] inputs_expand = K.expand_dims(inputs, 1) # Replicate num_capsule dimension to prepare being multiplied by W # inputs_tiled.shape=[None, num_capsule, input_num_capsule, input_dim_capsule] inputs_tiled = K.tile(inputs_expand, [1, self.num_capsule, 1, 1]) # Compute `inputs * W` by scanning inputs_tiled on dimension 0. # x.shape=[num_capsule, input_num_capsule, input_dim_capsule] # W.shape=[num_capsule, input_num_capsule, dim_capsule, input_dim_capsule] # Regard the first two dimensions as `batch` dimension, # then matmul: [input_dim_capsule] x [dim_capsule, input_dim_capsule]^T -> [dim_capsule]. # inputs_hat.shape = [None, num_capsule, input_num_capsule, dim_capsule] inputs_hat = K.map_fn(lambda x: K.batch_dot(x, self.W, [2, 3]), elems=inputs_tiled) # In forward pass, `inputs_hat_stopped` = `inputs_hat`; # In backward, no gradient can flow from `inputs_hat_stopped` back to `inputs_hat`. # inputs_hat_stopped = K.stop_gradient(inputs_hat) # The prior for coupling coefficient, initialized as zeros. # b.shape = [None, self.num_capsule, self.input_num_capsule]. # b = tf.zeros(shape=[K.shape(inputs_hat)[0], self.num_capsule, self.input_num_capsule]) b = K.stop_gradient(K.sum(K.zeros_like(inputs_hat), -1)) assert self.routings > 0, 'The routings should be > 0.' for i in range(self.routings): # c.shape=[batch_size, num_capsule, input_num_capsule] c = tf.nn.softmax(b, dim=1) # outputs.shape=[None, num_capsule, dim_capsule] outputs = squash(K.batch_dot(c, inputs_hat, [2, 2])) # [None, 10, 16] # b.shape=[batch_size, num_capsule, input_num_capsule] b += K.batch_dot(outputs, inputs_hat, [2, 3]) return outputs
def lam(paras): with tf.name_scope('Channel_Simple'): lam_user_id=paras[0] lam_item_id=paras[1] lam_x=paras[2] # 每个样本的数据进行相同的处理 def fn(elements): # ***为什么搭建时是int32,使用Lambda层传入数据后自动变为float32?*** a_user_id = tf.squeeze(tf.cast(elements[0],tf.int32)) # scalar shape:(1,) 要作为索引要转化为() a_item_id = tf.squeeze(tf.cast(elements[1],tf.int32)) # scalar a_mashup_feature = elements[2] # 1D tensor tf.assign(features[a_user_id], a_mashup_feature) # 何时更新??? indexes = [] sims = [] for index in range(num_users): indexes.append(index) # 包括自身,不易判断值再剔除 sims.append(tensor_sim(a_mashup_feature, features[index])) # list of scalar topK_prod = [] tensor_sims = [K.expand_dims(sim) for sim in sims] tensor_sims = K.concatenate(tensor_sims) # shape=(n,) # print(tensor_sims.shape) max_indexes = tf.nn.top_k(tensor_sims, max_k + 1)[1] for i in range(1, max_k + 1): index = max_indexes[i] temp_sim = tensor_sims[index] u_i = U_I[index][a_item_id] topK_prod.append(temp_sim * tf.cast(u_i, tf.float32)) topk_sim_features = [K.expand_dims(sum(topK_prod[:topK])) for topK in max_ks] # 各个topK下计算的sim积 tensor CF_feature = K.concatenate(topk_sim_features) # 整合的tensor 形状? final_feature =tf.concat([CF_feature,a_mashup_feature],0) return a_user_id, a_item_id, final_feature # 同时返回user_id是为了保证输入和输出形状相同,user_id无实质意义 _1, _2, final_feature = K.map_fn(fn, (lam_user_id, lam_item_id, lam_x)) return final_feature
def call(self, inputs, **kwargs): # (batch_size, 1, input_num_capsule, input_dim_capsule) expand_inputs = K.expand_dims(inputs, axis=1) # (batch_size, num_capsule, input_num_capsule, input_dim_capsule) expand_inputs = K.tile(expand_inputs, (1, self.num_capsule, 1, 1)) # (batch_size, num_capsule, input_num_capsule, dim_capsule) u_hat = K.map_fn(lambda x: K.batch_dot(x, self.W, axes=[2, 3]), expand_inputs) if self.num_routing <= 0: self.num_routing = 3 # (batch_size, num_capsule, input_num_capsule) b = K.zeros((K.shape(u_hat)[0], self.num_capsule, self.input_num_capsule)) for i in xrange(self.num_routing): # (batch_size, num_capsule, input_num_capsule) c = softmax(b, axis=1) # (batch_size, num_capsule, dim_capsule) s = K.batch_dot(c, u_hat, axes=[2, 2]) squashed_s = squash(s) if i < self.num_routing - 1: # (batch_size, num_capsule, input_num_capsule) b += K.batch_dot(squashed_s, u_hat, axes=[2, 3]) return squashed_s
def bayesian_categorical_crossentropy_internal_elu(true, pred_var): # shape: (N,) std = K.sqrt(pred_var[:, num_classes]) # shape: (N,) variance = pred_var[:, num_classes] variance_depressor = K.exp(variance) - K.ones_like(variance) # shape: (N, C) pred = pred_var[:, 0:num_classes] # shape: (N,) undistorted_loss = K.categorical_crossentropy(pred, true, from_logits=True) # shape: (T,) iterable = K.variable(np.ones(T)) dist = noise_var(pred, std, num_classes) monte_carlo_results = K.map_fn(gaussian_categorical_crossentropy_elu( true, pred, dist, undistorted_loss, num_classes), iterable, name='monte_carlo_results') variance_loss = K.mean(monte_carlo_results, axis=0) * undistorted_loss return variance_loss + undistorted_loss + variance_depressor
def call(self, x): def operation_over_sample(sample): buffer = [] for idx in range(self.filters): oper = K.dot(sample,self.w1[idx])+self.b1[idx] # sum w*x+b oper = K.relu(oper) # activation if self.hidden_dim!=None: # insert hidden dense layer oper = K.dot(oper,self.w2[idx])+self.b2[idx] # sum w*x+b oper = K.relu(oper) # activation # since the first input is pt, which is always positive => convert it as a mask mask = K.expand_dims(K.greater(sample[:,0],0.),axis=1) mask = K.cast(mask,K.floatx()) oper *= mask # apply masking top_indices = tf.nn.top_k(oper[:,0], k=self.toppooling_dim, sorted=True).indices top = K.gather(oper,top_indices) # top-k pooling buffer.append(top) return K.concatenate(buffer) return K.map_fn(operation_over_sample,x)
def lovasz_loss(truth, logit, margin=[1, 5]): def compute_lovasz_gradient(truth): #sorted truth_sum = K.sum(truth) intersection = truth_sum - K.cumsum(truth, 0) union = truth_sum + K.cumsum(1 - truth, 0) jaccard = 1. - intersection / union jaccard = K.concatenate([jaccard[0:1], jaccard[1:] - jaccard[:-1]], axis=0) gradient = jaccard return gradient def lovasz_hinge_one(truth, logit): m = tf.where(K.equal(truth, 1), margin[1] * K.ones_like(truth), margin[0] * K.ones_like(truth)) truth = K.cast(truth, dtype=logit.dtype) sign = 2. * truth - 1. hinge = (m - logit * K.stop_gradient(sign)) hinge, permutation = tf.nn.top_k(hinge, k=K.shape(hinge)[0]) hinge = K.relu(hinge) truth = K.gather(truth, permutation) gradient = compute_lovasz_gradient(truth) loss = K.dot(K.expand_dims(hinge, 0), K.stop_gradient(K.expand_dims(gradient, -1))) return loss #---- batch_size = K.shape(logit)[0] loss = K.map_fn(lambda x: lovasz_hinge_one(truth[x], logit[x]), K.arange(batch_size), dtype='float32') return loss
def evaluate_actions(self, x, actions): ''' 状態xから状態価値、実際の行動actionsのlog確率とエントロピーを求めます # https://github.com/germain-hug/Deep-RL-Keras/blob/master/A2C/actor.py#L29 ''' value, actor_output = self.actor_critic.output log_probs = K.log(actor_output) # log_probs: (80,4) # transpose: (4, 80) # gather: () # expect: (80, 1) # genjitsu: (80, 1, 80) # actions; (80, 1) # a[i][j] = log_probs[i][actions[i][j]] # transpose: [[1,3,2,4], # [6,3,2,1], # ... # [1,2,3,4]] # actions: [2, 3, ..., 1] # => [2, 1, ..., 2] #action_log_probs = K.gather(K.transpose(log_probs), K.reshape(actions, [-1])) #action_log_probs = K.gather(K.reshape(log_probs, [-1]), [i*(K.int_shape(log_probs)[1]+e) for i, e in enumerate(K.eval(K.reshape(actions, [-1])))]) action_log_probs = K.map_fn( lambda x: K.gather(x[0], x[1]), [log_probs, K.reshape(actions, [-1])], dtype="float") # dim=1で行動の種類方向に計算 dist_entropy = -K.mean(K.sum(log_probs * actor_output, axis=1)) # 軸やばい # meanやばい-> size 1? return value, action_log_probs, dist_entropy
def call(self, x): frames_range = np.arange(self.output_size) def create_waves(triples): waves = K.map_fn(create_wave, triples) wave = K.sum(waves, axis=0) / self.oscillators return wave def create_wave(triple): if self.min_frequency == None and self.max_frequency == None: frequency = triple[0] else: frequency = triple[0] * (self.max_frequency - self. min_frequency) + self.min_frequency amplitude = triple[1] phase = triple[2] samples = phase + frames_range * 2 * math.pi * frequency / self.sample_rate wave = amplitude * K.sin(samples) return wave waves = K.map_fn(create_waves, x) return waves
def bayesian_categorical_crossentropy_internal(true, pred_var): # shape: (N,) std = K.sqrt(pred_var[:, num_classes:]) # shape: (N,) #variance = pred_var[:, num_classes] #variance_depressor = K.exp(variance) - K.ones_like(variance) # shape: (N, C) pred = pred_var[:, 0:num_classes] # shape: (N,) #undistorted_loss = K.categorical_crossentropy(true, pred, from_logits=True) # shape: (T,) iterable = K.variable(np.ones(T)) dist = distributions.Normal(loc=K.zeros_like(std), scale=std) monte_carlo_results = K.map_fn(gaussian_categorical_crossentropy( true, pred, dist, num_classes), iterable, name='monte_carlo_results') variance_loss = K.mean(monte_carlo_results, axis=0) # * undistorted_loss return variance_loss #+ undistorted_loss # + variance_depressor
def call(self, inputs, **kwargs): def mapper(inp, n_segments): image = inp[0] slic_output = inp[1] slic_transposed = slic_output - 1 cycles_list = [] for cycle in range(n_segments): # get segment mask = tf.cast(slic_transposed == 0, "float32") segment = image * mask # get average avg = K.sum(segment, axis=[0, 1]) / tf.math.count_nonzero( segment, axis=[0, 1], dtype="float32") # insert into matrix cycles_list.append(avg) slic_transposed -= 1 return K.stack(cycles_list) r = K.map_fn(lambda x: mapper(x, self.n_segments), (inputs[0], inputs[1]), dtype=tf.float32) # tf.logging.log(tf.logging.ERROR, r) return K.max(r, axis=-1, keepdims=True)
def call(self, x, mask=None): v_att = K.dot(x, self.W) if self.bias: v_att += self.b v_att = K.tanh(v_att) mask = K.cast(mask, K.floatx()) mask = K.expand_dims(mask, axis=1) masks = self.masks * mask masks = K.permute_dimensions(masks, [1, 0, 2]) alpha = self.dot_product(v_att, self.u) # > (batch, seq_len) alphas = K.map_fn( lambda mask: K.expand_dims(self.masked_softmax(alpha, mask)), masks) alphas = K.permute_dimensions(alphas, [1, 0, 2, 3]) x = K.expand_dims(x, axis=1) attended_x = x * alphas alphas = K.squeeze(alphas, axis=3) cs = K.sum(attended_x, axis=2) if self.return_sequences: alpha = alphas c = cs else: alpha = tf.gather(alphas, -1, axis=1) c = tf.gather(cs, -1, axis=1) if self.return_coefficients: return [c, alpha] else: return c
def SpaceDetector(x): print("x-sh", x.shape) # print("input: ", K.eval(x)) sp_idx = 0 #ctable.char_indices[' '] sp = np.zeros((INPUT_VOCAB_SIZE)) print(sp) sp[sp_idx] = 1 filtered = x * sp # print("filtered:", K.eval(filtered)) sp_positions = K.tf.where(K.tf.equal(filtered, 1)) # row indices print(sp_positions.shape) # print("sp-p:", K.eval(sp_positions)) starts = sp_positions[:-1] + [0, 1, 0] stops = sp_positions[1:] + [0, 0, INPUT_VOCAB_SIZE] sizes = stops - starts + [1, 0, 0] where = K.tf.equal(sizes[:, 0], 1) starts = K.tf.boolean_mask(starts, where) # Remove multi-sample rows sizes = K.tf.boolean_mask(sizes, where) # Same where = K.tf.greater(sizes[:, 1], 0) starts = K.tf.boolean_mask( starts, where) # Remove words with 0 length (consecutive spaces) sizes = K.tf.boolean_mask(sizes, where) # Same print("starts:", starts, "sh:", starts.shape) print("stops:", stops) print("sizes:", sizes, "sh:", sizes.shape) slices = K.map_fn( lambda info: K.tf.pad(K.squeeze(K.slice(x, info[0], info[ 1]), 0), [[0, MAX_WORD_SIZE - info[1][1]], [0, 0]], "CONSTANT"), [starts, sizes], dtype=float) return slices
def call(self, x): x_shape = x.get_shape().as_list() b = K.zeros([self.batch_size, self.num_capsules, x_shape[1]]) b = Input(tensor=b) x = K.repeat_elements(K.expand_dims(x, axis=1), self.num_capsules, axis=1) x_hat = K.map_fn(lambda i: K.batch_dot(i, self.W, axes=0), elems=x) for i in range(self.routing_iter): c = K.expand_dims(softmax(b, axis=-1)) s = Multiply()([x_hat, c]) s = K.sum(s, axis=2) v = Squash()(s) v_expanded = K.repeat_elements(K.expand_dims(v, 2), x_shape[1], axis=2) update = Multiply()([v_expanded, x_hat]) update = K.sum(update, axis=-1) b += update return v
def call(self, x, mask=None): # Rotate image according to orientation if self.mask_orientation == 'tb': pass elif self.mask_orientation == 'bt': x = K.map_fn(lambda l: tf.image.rot90(l, k=2), x) elif self.mask_orientation == 'lr': x = K.map_fn(lambda l: tf.image.rot90(l, k=3), x) elif self.mask_orientation == 'rl': x = K.map_fn(lambda l: tf.image.rot90(l, k=1), x) # Convolve outputs = K.conv2d( x, self.kernel * self.mask, # masked kernel strides=self.strides, padding=self.padding, data_format=self.data_format, dilation_rate=self.dilation_rate) # Restore image rotation according to orientation if self.mask_orientation == 'tb': pass elif self.mask_orientation == 'bt': outputs = K.map_fn(lambda l: tf.image.rot90(l, k=2), outputs) elif self.mask_orientation == 'lr': outputs = K.map_fn(lambda l: tf.image.rot90(l, k=1), outputs) elif self.mask_orientation == 'rl': outputs = K.map_fn(lambda l: tf.image.rot90(l, k=3), outputs) # Add bias if self.use_bias: outputs = K.bias_add(outputs, self.bias, data_format=self.data_format) # Add activation if self.activation is not None: return self.activation(outputs) return outputs
def call(self, x): """Build the actual logic.""" # Our input has the shape # (batch, features, particles) # Short: bfp if self.debug: x= K.variable(np.array([[[ 229.46118164, 132.46817017, 26.43243217, 13.2313776, 5.75571156], [-195.08522034, -113.19028473, -22.73009872, -10.31623554, -4.25184822], [-114.19178772, -65.08143616, -12.34527397, -8.04754353, -2.59461427], [ -39.42618179, -22.36474037, -5.44153976, -1.97019398, -2.88409066]], [[ 129., 135., 26., 15., 7.], [-105., -114., -20., -10., -6.], [-100., -60., -10., -8., -1.], [ -32., -20., -5., -1., -2.]]])) if self.debug: print ("x:") print (K.eval(x.shape)) print (K.eval(x)) # magic1: 111 000 000 # 000 111 000 # 000 000 111 # # magic2: 100 100 100 # 010 010 010 # 001 001 001 magic1 = K.repeat_elements(K.eye(self.n_particles),self.n_particles,1) magic2 = K.tile(K.eye(self.n_particles),[1,self.n_particles]) # dimension: p p' if self.diff: magic = magic1-magic2 else: magic = magic1+magic2 if self.debug: print ("magic:") print (K.eval(magic.shape)) print (K.eval(magic)) # b f p p' # x * magic gives b f p^2, reshape to b f p p' diff_ij = K.reshape(K.expand_dims(K.dot(x, magic), -1), (x.shape[0], x.shape[1], x.shape[2], x.shape[2])) if self.debug: print ("diff_ij:") print (K.eval(diff_ij.shape)) print (K.eval(diff_ij)) # fold with the metric # b f p p' * f f' = b p p' f prod = theano.tensor.tensordot(diff_ij, self.metric, axes=[1,0]) if self.debug: print ("prod:") print (K.eval(prod.shape)) print (K.eval(prod)) # multiply out the feature dimension # b p p' f * b p'' p ''' f' # -> b p p' p'' p''' prod = theano.tensor.batched_tensordot(prod, diff_ij, axes=[3,1]) if self.debug: print ("prod:") print (K.eval(prod.shape)) print (K.eval(prod)) # build the diagonal we care about # b p p' p'' p''' -> b p p' # TODO: understand why we need these axes for correct diagonal? dij_square = prod.diagonal(axis1=1,axis2=4).diagonal(axis1=1,axis2=2) if self.debug: print ("dij_square:") print (K.eval(dij_square.shape)) print (K.eval(dij_square)) dij_square = K.clip(K.abs(K.pow(dij_square,-1)),0,1) # if self.debug: # print ("dijminus:") # print (K.eval(dijminus.shape)) # print (K.eval(dijminus)) # b lead_pxs = x[:,1,0] lead_pys = x[:,2,0] lead_pts2 = K.expand_dims(K.expand_dims(K.pow(lead_pxs+lead_pys,2),-1),-1) lead_pts2 = K.tile(lead_pts2, [1,self.n_particles, self.n_particles]) if self.debug: print ("lead_pts2:") print (K.eval(lead_pts2.shape)) print (K.eval(lead_pts2)) denom = K.clip(lead_pts2, 0.001, 100000.) if self.debug: print ("self.w_Aij:") print (K.eval(self.w_Aij)) print (K.eval(self.w_Aij)) term1 = dij_square if self.debug: print ("term1:") print (K.eval(term1.shape)) print (K.eval(term1)) poly = self.w_Aij # + self.w[0] * term1 # + self.w[1] * pow(dij_square/denom,2) if self.debug: print ("poly:") print (K.eval(poly.shape)) print (K.eval(poly)) if self.regularize: poly = K.map_fn( lambda x:K.switch(K.less(x,self.w_reg[0]), self.w_reg[1], self.w_reg[2]), poly) li = [] if self.add_total: li.append( K.ones(shape=(self.n_particles, 1))) if self.add_eye: li.append( K.eye(self.n_particles)) li.append(poly) poly = K.concatenate(li,axis = 1) # And multiply with the initial particle positions #ret = K.batch_dot(x,poly) ret = K.dot(x,poly) if self.debug: print ("ret:") print (K.eval(ret.shape)) print (K.eval(ret)) if self.debug: sys.exit() return ret
def call(self, inputs, training=None): # inputs.shape=[None, input_num_capsule, input_dim_capsule] # inputs_expand.shape=[None, 1, input_num_capsule, input_dim_capsule] inputs_expand = K.expand_dims(inputs, 1) # Replicate num_capsule dimension to prepare being multiplied by W # inputs_tiled.shape=[None, num_capsule, input_num_capsule, input_dim_capsule] inputs_tiled = K.tile(inputs_expand, [1, self.num_capsule, 1, 1]) # Compute `inputs * W` by scanning inputs_tiled on dimension 0. # x.shape=[num_capsule, input_num_capsule, input_dim_capsule] # W.shape=[num_capsule, input_num_capsule, dim_capsule, input_dim_capsule] # Regard the first two dimensions as `batch` dimension, # then matmul: [input_dim_capsule] x [dim_capsule, input_dim_capsule]^T -> [dim_capsule]. # inputs_hat.shape = [None, num_capsule, input_num_capsule, dim_capsule] inputs_hat = K.map_fn(lambda x: K.batch_dot(x, self.W, [2, 3]), elems=inputs_tiled) """ # Begin: routing algorithm V1, dynamic ------------------------------------------------------------# # The prior for coupling coefficient, initialized as zeros. b = K.zeros(shape=[self.batch_size, self.num_capsule, self.input_num_capsule]) def body(i, b, outputs): c = tf.nn.softmax(b, dim=1) # dim=2 is the num_capsule dimension outputs = squash(K.batch_dot(c, inputs_hat, [2, 2])) if i != 1: b = b + K.batch_dot(outputs, inputs_hat, [2, 3]) return [i-1, b, outputs] cond = lambda i, b, inputs_hat: i > 0 loop_vars = [K.constant(self.num_routing), b, K.sum(inputs_hat, 2, keepdims=False)] shape_invariants = [tf.TensorShape([]), tf.TensorShape([None, self.num_capsule, self.input_num_capsule]), tf.TensorShape([None, self.num_capsule, self.dim_capsule])] _, _, outputs = tf.while_loop(cond, body, loop_vars, shape_invariants) # End: routing algorithm V1, dynamic ------------------------------------------------------------# """ # Begin: Routing algorithm ---------------------------------------------------------------------# # In forward pass, `inputs_hat_stopped` = `inputs_hat`; # In backward, no gradient can flow from `inputs_hat_stopped` back to `inputs_hat`. inputs_hat_stopped = K.stop_gradient(inputs_hat) # The prior for coupling coefficient, initialized as zeros. # b.shape = [None, self.num_capsule, self.input_num_capsule]. b = tf.zeros(shape=[ K.shape(inputs_hat)[0], self.num_capsule, self.input_num_capsule ]) #assert self.num_routing > 0, 'The num_routing should be > 0.' for i in range(self.num_routing): # c.shape=[batch_size, num_capsule, input_num_capsule] c = tf.nn.softmax(b, dim=1) # At last iteration, use `inputs_hat` to compute `outputs` in order to backpropagate gradient if i == self.num_routing - 1: # c.shape = [batch_size, num_capsule, input_num_capsule] # inputs_hat.shape=[None, num_capsule, input_num_capsule, dim_capsule] # The first two dimensions as `batch` dimension, # then matmal: [input_num_capsule] x [input_num_capsule, dim_capsule] -> [dim_capsule]. # outputs.shape=[None, num_capsule, dim_capsule] outputs = squash(K.batch_dot(c, inputs_hat, [2, 2])) # [None, 10, 16] else: # Otherwise, use `inputs_hat_stopped` to update `b`. No gradients flow on this path. outputs = squash(K.batch_dot(c, inputs_hat_stopped, [2, 2])) # outputs.shape = [None, num_capsule, dim_capsule] # inputs_hat.shape=[None, num_capsule, input_num_capsule, dim_capsule] # The first two dimensions as `batch` dimension, # then matmal: [dim_capsule] x [input_num_capsule, dim_capsule]^T -> [input_num_capsule]. # b.shape=[batch_size, num_capsule, input_num_capsule] b += K.batch_dot(outputs, inputs_hat_stopped, [2, 3]) # End: Routing algorithm -----------------------------------------------------------------------# all = K.concatenate([outputs, c]) return all
def _process_sample(args): _hm, _reg, _wh, _kps, _hm_hp, _hp_offset = args _scores, _inds = tf.math.top_k(_hm, k=k, sorted=True) _classes = K.cast(_inds % cat, 'float32') _inds = K.cast(_inds / cat, 'int32') _xs = K.cast(_inds % width, 'float32') _ys = K.cast(K.cast(_inds / width, 'int32'), 'float32') _wh = K.gather(_wh, _inds) _reg = K.gather(_reg, _inds) _kps = K.gather(_kps, _inds) # shift keypoints by their center _kps_x = _kps[:, ::2] _kps_y = _kps[:, 1::2] _kps_x = _kps_x + K.expand_dims(_xs, -1) # k x J _kps_y = _kps_y + K.expand_dims(_ys, -1) # k x J _kps = K.stack([_kps_x, _kps_y], -1) # k x J x 2 _xs = _xs + _reg[..., 0] _ys = _ys + _reg[..., 1] _x1 = _xs - _wh[..., 0] / 2 _y1 = _ys - _wh[..., 1] / 2 _x2 = _xs + _wh[..., 0] / 2 _y2 = _ys + _wh[..., 1] / 2 # snap center keypoints to the closest heatmap keypoint def _process_channel(args): __kps, __hm_hp = args thresh = 0.1 __hm_scores, __hm_inds = tf.math.top_k(__hm_hp, k=k, sorted=True) __hm_xs = K.cast(__hm_inds % width, 'float32') __hm_ys = K.cast(K.cast(__hm_inds / width, 'int32'), 'float32') __hp_offset = K.gather(_hp_offset, __hm_inds) __hm_xs = __hm_xs + __hp_offset[..., 0] __hm_ys = __hm_ys + __hp_offset[..., 1] mask = K.cast(__hm_scores > thresh, 'float32') __hm_scores = (1. - mask) * -1. + mask * __hm_scores __hm_xs = (1. - mask) * -10000. + mask * __hm_xs __hm_ys = (1. - mask) * -10000. + mask * __hm_ys __hm_kps = K.stack([__hm_xs, __hm_ys], -1) # k x 2 __broadcast_hm_kps = K.expand_dims(__hm_kps, 1) # k x 1 x 2 __broadcast_kps = K.expand_dims(__kps, 0) # 1 x k x 2 dist = K.sqrt( K.sum(K.pow(__broadcast_kps - __broadcast_hm_kps, 2), 2)) # k, k min_dist = K.min(dist, 0) min_ind = K.argmin(dist, 0) __hm_scores = K.gather(__hm_scores, min_ind) __hm_kps = K.gather(__hm_kps, min_ind) mask = (K.cast(__hm_kps[..., 0] < _x1, 'float32') + K.cast(__hm_kps[..., 0] > _x2, 'float32') + K.cast(__hm_kps[..., 1] < _y1, 'float32') + K.cast(__hm_kps[..., 1] > _y2, 'float32') + K.cast(__hm_scores < thresh, 'float32') + K.cast( min_dist > 0.3 * (K.maximum(_wh[..., 0], _wh[..., 1])), 'float32')) mask = K.expand_dims(mask, -1) mask = K.cast(mask > 0, 'float32') __kps = (1. - mask) * __hm_kps + mask * __kps return __kps _kps = K.permute_dimensions(_kps, (1, 0, 2)) # J x k x 2 _hm_hp = K.permute_dimensions(_hm_hp, (1, 0)) # J x -1 _kps = K.map_fn(_process_channel, [_kps, _hm_hp], dtype='float32') _kps = K.reshape(K.permute_dimensions(_kps, (1, 2, 0)), (k, -1)) # k x J * 2 # rescale to image coordinates _x1 = output_stride * _x1 _y1 = output_stride * _y1 _x2 = output_stride * _x2 _y2 = output_stride * _y2 _kps = output_stride * _kps _boxes = K.stack([_x1, _y1, _x2, _y2], -1) _scores = K.expand_dims(_scores, -1) _classes = K.expand_dims(_classes, -1) _detection = K.concatenate([_boxes, _scores, _kps, _classes], -1) return _detection
def repulsion_loss(y_true, y_pred): with K.tf.device('/cpu:0'): return K.map_fn(lambda x: regression_loss_one(x[0], x[1]), (y_true, y_pred), dtype=K.floatx())