def spatial_attention(cost_volume): feature = 4 * 9 k = 9 label = 9 dres0 = convbn_3d(cost_volume, feature / 2, 3, 1) dres0 = Activation('relu')(dres0) dres0 = convbn_3d(dres0, 1, 3, 1) cost0 = Activation('relu')(dres0) cost0 = Lambda(lambda x: K.permute_dimensions(K.squeeze(x, -1), (0, 2, 3, 1)))(cost0) cost1 = convbn(cost0, label // 2, (1, k), 1, 1) cost1 = Activation('relu')(cost1) cost1 = convbn(cost1, 1, (k, 1), 1, 1) cost1 = Activation('relu')(cost1) cost2 = convbn(cost0, label // 2, (k, 1), 1, 1) cost2 = Activation('relu')(cost2) cost2 = convbn(cost2, 1, (1, k), 1, 1) cost2 = Activation('relu')(cost2) cost = add([cost1, cost2]) cost = Activation('sigmoid')(cost) cost = Lambda(lambda y: K.repeat_elements(K.expand_dims(y, 1), 9, 1))(cost) cost = Lambda(lambda y: K.repeat_elements(y, feature, 4))(cost) return multiply([cost, cost_volume])
def __init__(self, restore=None, session=None, use_log=False): self.num_channels = 1 self.image_size = 28 self.num_labels = 10 model = Sequential() model.add(Flatten(input_shape=(28, 28, 1))) model.add(Dense(1024)) model.add(Lambda(lambda x: x * 10)) model.add(Activation('softplus')) model.add(Lambda(lambda x: x * 0.1)) model.add(Dense(10)) # output log probability, used for black-box attack if use_log: model.add(Activation('softmax')) if restore: model.load_weights(restore) layer_outputs = [] for layer in model.layers: if isinstance(layer, Conv2D) or isinstance(layer, Dense): layer_outputs.append( K.function([model.layers[0].input], [layer.output])) self.layer_outputs = layer_outputs self.model = model
def create_model(activation, bn, data, filters, init, kernels, use_padding_same): model = Sequential() if use_padding_same: model.add( Conv2D(filters[0], kernels[0], input_shape=data.train_data.shape[1:], padding="same")) else: model.add( Conv2D(filters[0], kernels[0], input_shape=data.train_data.shape[1:])) if bn: apply_bn(data, model) # model.add(Activation(activation)) model.add(Lambda(activation)) for f, k in zip(filters[1:], kernels[1:]): if use_padding_same: model.add(Conv2D(f, k, padding="same")) else: model.add(Conv2D(f, k)) if bn: apply_bn(data, model) # model.add(Activation(activation)) # ReLU activation model.add(Lambda(activation)) # the output layer model.add(Flatten()) model.add(Dense(data.train_labels.shape[1])) # load initial weights when given if init != None: model.load_weights(init) return model
def rotate_start_in(x, body_members): scope = Scoping.get_global_scope() with scope.name_scope('rotate_start'): left_shoulder = body_members['left_arm']['joints'][1] right_shoulder = body_members['right_arm']['joints'][1] hip = body_members['torso']['joints'][0] head_top = body_members['head']['joints'][-1] base_shape = [int(d) for d in x.shape] base_shape[1] = 1 base_shape[2] = 1 def _get_rotation(arg): coords_list = tf.unstack(arg[:, :, 0, :], axis=1) torso_rot = tf.cross( coords_list[left_shoulder] - coords_list[hip], coords_list[right_shoulder] - coords_list[hip]) side_rot = K.reshape( tf.cross(coords_list[head_top] - coords_list[hip], torso_rot), base_shape) theta_diff = ( (np.pi / 2) - tf.atan2(side_rot[..., 1], side_rot[..., 0])) / 2 cos_theta_diff = tf.cos(theta_diff) sin_theta_diff = tf.sin(theta_diff) zeros_theta = K.zeros_like(sin_theta_diff) return K.stack( [cos_theta_diff, zeros_theta, zeros_theta, sin_theta_diff], axis=-1) start_rotation = Lambda(_get_rotation, name=scope + 'start_rotation')(x) x = Lambda(lambda args: rotate_vector_by_quaternion(args[1], args[0]), name=scope + 'rotate_start_in')([x, start_rotation]) return x, start_rotation
def train(data, file_name, params, num_epochs=50, batch_size=128, train_temp=1, init=None, lr=0.01, decay=1e-5, momentum=0.9): """ Train a 2-layer simple network for MNIST and CIFAR """ # create a Keras sequential model model = Sequential() # reshape the input (28*28*1) or (32*32*3) to 1-D model.add(Flatten(input_shape=data.train_data.shape[1:])) # first dense layer (the hidden layer) model.add(Dense(params[0])) # \alpha = 10 in softplus, multiply input by 10 model.add(Lambda(lambda x: x * 10)) # in Keras the softplus activation cannot set \alpha model.add(Activation('softplus')) # so manually add \alpha to the network model.add(Lambda(lambda x: x * 0.1)) # the output layer, with 10 classes model.add(Dense(10)) # load initial weights when given if init != None: model.load_weights(init) # define the loss function which is the cross entropy between prediction and true label def fn(correct, predicted): return tf.nn.softmax_cross_entropy_with_logits(labels=correct, logits=predicted / train_temp) # initiate the SGD optimizer with given hyper parameters sgd = SGD(lr=lr, decay=decay, momentum=momentum, nesterov=True) # compile the Keras model, given the specified loss and optimizer model.compile(loss=fn, optimizer=sgd, metrics=['accuracy']) # run training with given dataset, and print progress model.fit(data.train_data, data.train_labels, batch_size=batch_size, validation_data=(data.validation_data, data.validation_labels), epochs=num_epochs, shuffle=True) # save model to a file if file_name != None: model.save(file_name) return model
def train(data, file_name, params, num_epochs=50, batch_size=128, train_temp=1, init=None, pool=True): """ Standard neural network training procedure. Trains LeNet-5 style model with pooling optional. """ model = Sequential() print(data.train_data.shape) model.add( Conv2D(params[0], (5, 5), input_shape=data.train_data.shape[1:], padding='same')) model.add(Lambda(tf.nn.relu)) if pool: model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(params[1], (5, 5))) model.add(Lambda(tf.nn.relu)) if pool: model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dense(params[2])) model.add(Lambda(tf.nn.relu)) model.add(Dense(10)) if init != None: model.load_weights(init) def fn(correct, predicted): return tf.nn.softmax_cross_entropy_with_logits(labels=correct, logits=predicted / train_temp) sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss=fn, optimizer=sgd, metrics=['accuracy']) model.fit(data.train_data, data.train_labels, batch_size=batch_size, validation_data=(data.validation_data, data.validation_labels), epochs=num_epochs, shuffle=True) if file_name != None: model.save(file_name) return model
def channel_attention_free(cost_volume): x = GlobalAveragePooling3D()(cost_volume) x = Lambda( lambda y: K.expand_dims(K.expand_dims(K.expand_dims(y, 1), 1), 1))(x) x = Conv3D(170, 1, 1, 'same')(x) x = Activation('relu')(x) x = Conv3D(81, 1, 1, 'same')(x) x = Activation('sigmoid')(x) attention = Lambda(lambda y: K.reshape(y, (K.shape(y)[0], 1, 1, 1, 81)))(x) x = Lambda(lambda y: K.repeat_elements(y, 4, -1))(attention) return multiply([x, cost_volume]), attention
def remove_hip_out(x, hip_info, data_set): scope = Scoping.get_global_scope() with scope.name_scope('remove_hip'): if 'expmaps' in data_set: x = Lambda(lambda args: K.concatenate([args[1], args[0]], axis=1), name=scope + 'remove_hip_out')([x, hip_info]) else: x = Lambda(lambda args: K.concatenate([args[1], args[0] + args[1]], axis=1), name=scope + 'remove_hip_out')([x, hip_info]) return x
def translate_start_in(x): scope = Scoping.get_global_scope() with scope.name_scope('translate_start'): def _get_start(arg): return K.reshape(arg[:, 0, 0, :], (arg.shape[0], 1, 1, 3)) start_coords = Lambda(_get_start, name=scope + 'start_coords')(x) x = Lambda(lambda args: args[0] - args[1], name=scope + 'translate_start_in')([x, start_coords]) return x, start_coords
def seq_to_diff_in(x, x_mask=None): scope = Scoping.get_global_scope() with scope.name_scope('seq_to_diff'): start_pose = Lambda(lambda arg: arg[:, :, 0, :], name=scope + 'start_pose')(x) x = Lambda(lambda arg: arg[:, :, 1:, :] - arg[:, :, :-1, :], name=scope + 'seq_to_diff_in')(x) if x_mask is not None: x_mask = Lambda(lambda arg: arg[:, :, 1:, :] * arg[:, :, :-1, :], name=scope + 'seq_mask_to_diff_in')(x_mask) return x, x_mask, start_pose
def channel_attention_mirror(cost_volume): x = GlobalAveragePooling3D()(cost_volume) x = Lambda( lambda y: K.expand_dims(K.expand_dims(K.expand_dims(y, 1), 1), 1))(x) x = Conv3D(170, 1, 1, 'same')(x) x = Activation('relu')(x) x = Conv3D(25, 1, 1, 'same')(x) x = Activation('sigmoid')(x) x = Lambda(lambda y: K.reshape(y, (K.shape(y)[0], 5, 5)))(x) x = Lambda(lambda y: tf.pad(y, [[0, 0], [0, 4], [0, 4]], 'REFLECT'))(x) attention = Lambda(lambda y: K.reshape(y, (K.shape(y)[0], 1, 1, 1, 81)))(x) x = Lambda(lambda y: K.repeat_elements(y, 4, -1))(attention) return multiply([x, cost_volume]), attention
def branch_attention(cost_volume_3d, cost_volume_h, cost_volume_v, cost_volume_45, cost_volume_135): feature = 4 * 9 k = 9 label = 9 cost1 = convbn(cost_volume_3d, 6, 3, 1, 1) cost1 = Activation('relu')(cost1) cost1 = convbn(cost1, 4, 3, 1, 1) cost1 = Activation('sigmoid')(cost1) cost_h = Lambda(lambda y: K.repeat_elements( K.expand_dims(y[:, :, :, :1], 1), 9, 1))(cost1) cost_h = Lambda(lambda y: K.repeat_elements(y, feature, 4))(cost_h) cost_v = Lambda(lambda y: K.repeat_elements( K.expand_dims(y[:, :, :, 1:2], 1), 9, 1))(cost1) cost_v = Lambda(lambda y: K.repeat_elements(y, feature, 4))(cost_v) cost_45 = Lambda(lambda y: K.repeat_elements( K.expand_dims(y[:, :, :, 2:3], 1), 9, 1))(cost1) cost_45 = Lambda(lambda y: K.repeat_elements(y, feature, 4))(cost_45) cost_135 = Lambda(lambda y: K.repeat_elements( K.expand_dims(y[:, :, :, 3:4], 1), 9, 1))(cost1) cost_135 = Lambda(lambda y: K.repeat_elements(y, feature, 4))(cost_135) return concatenate([ multiply([cost_h, cost_volume_h]), multiply([cost_v, cost_volume_v]), multiply([cost_45, cost_volume_45]), multiply([cost_135, cost_volume_135]) ], axis=4), cost1
def rotate_start_out(x, start_rotation): scope = Scoping.get_global_scope() with scope.name_scope('rotate_start'): x = Lambda(lambda args: rotate_vector_by_quaternion( quaternion_conjugate(args[1]), args[0]), name=scope + 'rotate_start_out')([x, start_rotation]) return x
def res(x): x = ResidualStart()(x) x1 = Conv2D(f, 3, strides=1, padding='same')(x) x1 = BatchNormalization()(x1) x1 = Lambda(activation)(x1) x1 = Conv2D(f, 3, strides=1, padding='same')(x1) x1 = BatchNormalization()(x1) return Add()([x1, x])
def train(data, file_name, params, num_epochs=50, batch_size=256, train_temp=1, init=None, lr=0.01, decay=1e-5, momentum=0.9, activation="relu", optimizer_name="sgd"): """ Train a n-layer simple network for MNIST and CIFAR """ # create a Keras sequential model model = Sequential() # reshape the input (28*28*1) or (32*32*3) to 1-D model.add(Flatten(input_shape=data.train_data.shape[1:])) # dense layers (the hidden layer) n = 0 for param in params: n += 1 model.add(Dense(param, kernel_initializer='he_uniform')) # ReLU activation if activation == "arctan": model.add(Lambda(lambda x: tf.atan(x), name=activation+"_"+str(n))) else: model.add(Activation(activation, name=activation+"_"+str(n))) # the output layer, with 10 classes model.add(Dense(10, kernel_initializer='he_uniform')) # load initial weights when given if init != None: model.load_weights(init) # define the loss function which is the cross entropy between prediction and true label def fn(correct, predicted): return tf.nn.softmax_cross_entropy_with_logits(labels=correct, logits=predicted/train_temp) if optimizer_name == "sgd": # initiate the SGD optimizer with given hyper parameters optimizer = SGD(lr=lr, decay=decay, momentum=momentum, nesterov=True) elif optimizer_name == "adam": optimizer = Adam(lr=lr, beta_1 = 0.9, beta_2 = 0.999, epsilon = None, decay=decay, amsgrad=False) # compile the Keras model, given the specified loss and optimizer model.compile(loss=fn, optimizer=optimizer, metrics=['accuracy']) model.summary() print("Traing a {} layer model, saving to {}".format(len(params) + 1, file_name)) # run training with given dataset, and print progress history = model.fit(data.train_data, data.train_labels, batch_size=batch_size, validation_data=(data.validation_data, data.validation_labels), epochs=num_epochs, shuffle=True) # save model to a file if file_name != None: model.save(file_name) print('model saved to ', file_name) return {'model':model, 'history':history}
def UpSampling3DBilinear(size): def UpSampling3DBilinear_(x, size): shape = K.shape(x) x = K.reshape(x, (shape[0] * shape[1], shape[2], shape[3], shape[4])) x = tf.image.resize_bilinear(x, size, align_corners=True) x = K.reshape(x, (shape[0], shape[1], size[0], size[1], shape[4])) return x return Lambda(lambda x: UpSampling3DBilinear_(x, size))
def rescale_body_in(x, body_members): scope = Scoping.get_global_scope() with scope.name_scope('rescale'): members_from, members_to, _ = get_body_graph(body_members) def _get_avg_bone_len(arg): bone_list = tf.unstack(arg[:, :, 0, :], axis=1) bones = [ bone_list[j] - bone_list[i] for i, j in zip(members_from, members_to) ] bones = K.expand_dims(K.stack(bones, axis=1), axis=2) bone_len = K.sqrt( K.sum(K.square(bones), axis=-1, keepdims=True) + K.epsilon()) return K.mean(bone_len, axis=1, keepdims=True) bone_len = Lambda(_get_avg_bone_len, name=scope + 'bone_len')(x) x = Lambda(lambda args: args[0] / args[1], name=scope + 'rescale_body_in')([x, bone_len]) return x, bone_len
def remove_hip_in(x, x_mask, data_set): scope = Scoping.get_global_scope() with scope.name_scope('remove_hip'): if 'expmaps' in data_set: hip_info = Lambda(lambda arg: arg[:, :2, :, :], name=scope + 'hip_expmaps')(x) x = Lambda(lambda arg: arg[:, 2:, ...], name=scope + 'remove_hip_in')(x) x_mask = Lambda(lambda arg: arg[:, 2:, ...], name=scope + 'remove_hip_mask_in')(x_mask) else: def _get_hips(arg): return K.reshape(arg[:, 0, :, :], (arg.shape[0], 1, arg.shape[2], 3)) hip_info = Lambda(_get_hips, name=scope + 'hip_coords')(x) x = Lambda(lambda args: (args[0] - args[1])[:, 1:, ...], name=scope + 'remove_hip_in')([x, hip_info]) x_mask = Lambda(lambda arg: arg[:, 1:, ...], name=scope + 'remove_hip_mask_in')(x_mask) return x, x_mask, hip_info
def classifier(self, x): scope = Scoping.get_global_scope() with scope.name_scope('classifier'): if self.data_set == 'NTURGBD': blocks = [{'size': 128, 'bneck': 32, 'groups': 16, 'strides': 1}, {'size': 256, 'bneck': 64, 'groups': 16, 'strides': 2}, {'size': 512, 'bneck': 128, 'groups': 16, 'strides': 2}] n_reps = 3 else: blocks = [{'size': 64, 'bneck': 32, 'groups': 8, 'strides': 3}, {'size': 128, 'bneck': 64, 'groups': 8, 'strides': 3}] n_reps = 3 def _data_augmentation(x): return K.in_train_phase(_sim_occlusions(_jitter_height(x)), x) x = Lambda(_data_augmentation, name=scope+"data_augmentation")(x) x = CombMatrix(self.njoints, name=scope+'comb_matrix')(x) x = EDM(name=scope+'edms')(x) x = Reshape((self.njoints * self.njoints, self.seq_len, 1), name=scope+'resh_in')(x) x = BatchNormalization(axis=-1, name=scope+'bn_in')(x) x = Conv2D(blocks[0]['bneck'], 1, 1, name=scope+'conv_in', **CONV2D_ARGS)(x) for i in range(len(blocks)): for j in range(n_reps): with scope.name_scope('block_%d_%d' % (i, j)): x = _conv_block(x, blocks[i]['size'], blocks[i]['bneck'], blocks[i]['groups'], 3, blocks[i]['strides'] if j == 0 else 1) x = Lambda(lambda args: K.mean(args, axis=(1, 2)), name=scope+'mean_pool')(x) x = BatchNormalization(axis=-1, name=scope + 'bn_out')(x) x = Activation('relu', name=scope + 'relu_out')(x) x = Dropout(self.dropout, name=scope+'dropout')(x) x = Dense(self.num_actions, activation='softmax', name=scope+'label')(x) return x
def to_3d_135(cost_volume_135): feature = 4 * 9 channel_135 = GlobalAveragePooling3D( data_format='channels_last')(cost_volume_135) channel_135 = Lambda(lambda y: K.expand_dims( K.expand_dims(K.expand_dims(y, 1), 1), 1))(channel_135) channel_135 = Conv3D(feature / 2, 1, 1, 'same', data_format='channels_last')(channel_135) channel_135 = Activation('relu')(channel_135) channel_135 = Conv3D(3, 1, 1, 'same', data_format='channels_last')(channel_135) channel_135 = Activation('sigmoid')(channel_135) channel_135 = Lambda(lambda y: K.concatenate([ y[:, :, :, :, 0:1], y[:, :, :, :, 0:1], y[:, :, :, :, 0:1], y[:, :, :, :, 0:1], y[:, :, :, :, 1:2], y[:, :, :, :, 2:3], y[:, :, :, :, 2:3], y[:, :, :, :, 2:3], y[:, :, :, :, 2:3] ], axis=-1))(channel_135) channel_135 = Lambda(lambda y: K.reshape(y, (K.shape(y)[0], 1, 1, 1, 9)))( channel_135) channel_135 = Lambda(lambda y: K.repeat_elements(y, 4, -1))(channel_135) cv_135_tmp = multiply([channel_135, cost_volume_135]) cv_135_tmp = Conv3D(feature / 2, 1, 1, 'same', data_format='channels_last')(cv_135_tmp) cv_135_tmp = Activation('relu')(cv_135_tmp) cv_135_tmp = Conv3D(3, 1, 1, 'same', data_format='channels_last')(cv_135_tmp) cv_135_tmp = Activation('sigmoid')(cv_135_tmp) attention_135 = Lambda(lambda y: K.concatenate([ y[:, :, :, :, 0:1], y[:, :, :, :, 0:1], y[:, :, :, :, 0:1], y[:, :, :, :, 0:1], y[:, :, :, :, 1:2], y[:, :, :, :, 2:3], y[:, :, :, :, 2:3], y[:, :, :, :, 2:3], y[:, :, :, :, 2:3] ], axis=-1))(cv_135_tmp) attention_135 = Lambda(lambda y: K.repeat_elements(y, 4, -1))( attention_135) cv_135_multi = multiply([attention_135, cost_volume_135]) dres3 = convbn_3d(cv_135_multi, feature, 3, 1) dres3 = Activation('relu')(dres3) dres3 = convbn_3d(cv_135_multi, feature / 2, 3, 1) dres3 = Activation('relu')(dres3) dres3 = convbn_3d(cv_135_multi, feature / 2, 3, 1) dres3 = Activation('relu')(dres3) dres3 = convbn_3d(cv_135_multi, feature / 4, 3, 1) dres3 = Activation('relu')(dres3) dres3 = convbn_3d(dres3, 1, 3, 1) cost3 = Activation('relu')(dres3) cost3 = Lambda(lambda x: K.permute_dimensions(K.squeeze(x, -1), (0, 2, 3, 1)))(cost3) return cost3, cv_135_multi
def define_LFattNet(sz_input, sz_input2, view_n, learning_rate): """ 81 inputs""" input_list = [] for i in range(len(view_n) * len(view_n)): print('input ' + str(i)) input_list.append(Input(shape=(sz_input, sz_input2, 1))) """ 81 features""" feature_extraction_layer = feature_extraction(sz_input, sz_input2) feature_list = [] for i in range(len(view_n) * len(view_n)): print('feature ' + str(i)) feature_list.append(feature_extraction_layer(input_list[i])) """ cost volume """ cv = Lambda(_getCostVolume_)(feature_list) """ channel attention """ cv, attention = channel_attention(cv) """ cost volume regression """ cost = basic(cv) cost = Lambda(lambda x: K.permute_dimensions(K.squeeze(x, -1), (0, 2, 3, 1)))(cost) pred = Activation('softmax')(cost) pred = Lambda(disparityregression)(pred) # when training use below # model = Model(inputs=input_list, outputs=[pred]) # when evaluation use below model = Model(inputs=input_list, outputs=[pred, attention]) model.summary() opt = Adam(lr=learning_rate) model.compile(optimizer=opt, loss='mae') return model
def _preact_conv(x, out_filters, kernel_size, strides, groups=1): scope = Scoping.get_global_scope() x = BatchNormalization(axis=-1, name=scope + 'bn')(x) x = Activation('relu', name=scope + 'relu')(x) if groups > 1: branches = [] group_size = int(x.shape[-1]) // groups for j in range(groups): with scope.name_scope('branch_%d' % j): x_group = Lambda(lambda arg: arg[:, :, :, j * group_size: (j + 1) * group_size], name=scope+'split')(x) branches.append(Conv2D(filters=out_filters // groups, kernel_size=kernel_size, strides=strides, name=scope+'conv', **CONV2D_ARGS)(x_group)) x = Concatenate(name=scope+'cat')(branches) else: x = Conv2D(filters=out_filters, kernel_size=kernel_size, strides=strides, name=scope+'conv', **CONV2D_ARGS)(x) return x
def seq_to_diff_out(x, start_pose): scope = Scoping.get_global_scope() with scope.name_scope('seq_to_diff'): def _diff_to_seq(args): diffs, start_pose = args diffs_list = tf.unstack(diffs, axis=2) poses = [start_pose] for p in range(diffs.shape[2]): poses.append(poses[p] + diffs_list[p]) return K.stack(poses, axis=2) x = Lambda(_diff_to_seq, name=scope + 'seq_to_diff_out')([x, start_pose]) return x
def define_AttMLFNet(sz_input, sz_input2, view_n, learning_rate): """ 4 branches inputs""" input_list = [] for i in range(len(view_n) * 4): input_list.append(Input(shape=(sz_input, sz_input2, 1))) """ 4 branches features""" feature_extraction_layer = feature_extraction(sz_input, sz_input2) feature_list = [] for i in range(len(view_n) * 4): feature_list.append(feature_extraction_layer(input_list[i])) feature_v_list = [] feature_h_list = [] feature_45_list = [] feature_135_list = [] for i in range(9): feature_h_list.append(feature_list[i]) for i in range(9, 18): feature_v_list.append(feature_list[i]) for i in range(18, 27): feature_45_list.append(feature_list[i]) for i in range(27, len(feature_list)): feature_135_list.append(feature_list[i]) """ cost volume """ cv_h = Lambda(_get_h_CostVolume_)(feature_h_list) cv_v = Lambda(_get_v_CostVolume_)(feature_v_list) cv_45 = Lambda(_get_45_CostVolume_)(feature_45_list) cv_135 = Lambda(_get_135_CostVolume_)(feature_135_list) """ intra branch """ cv_h_3d, cv_h_ca = to_3d_h(cv_h) cv_v_3d, cv_v_ca = to_3d_v(cv_v) cv_45_3d, cv_45_ca = to_3d_45(cv_45) cv_135_3d, cv_135_ca = to_3d_135(cv_135) """ inter branch """ cv, attention_4 = branch_attention( multiply([cv_h_3d, cv_v_3d, cv_45_3d, cv_135_3d]), cv_h_ca, cv_v_ca, cv_45_ca, cv_135_ca) """ cost volume regression """ cost = basic(cv) cost = Lambda(lambda x: K.permute_dimensions(K.squeeze(x, -1), (0, 2, 3, 1)))(cost) pred = Activation('softmax')(cost) pred = Lambda(disparityregression)(pred) model = Model(inputs=input_list, outputs=[pred]) model.summary() opt = Adam(lr=learning_rate) model.compile(optimizer=opt, loss='mae') return model
def __init__(self, params, restore=None, session=None, use_log=False, image_size=28, image_channel=1, activation='relu'): self.image_size = image_size self.num_channels = image_channel self.num_labels = 10 model = Sequential() model.add(Flatten(input_shape=(image_size, image_size, image_channel))) # list of all hidden units weights self.U = [] for param in params: # add each dense layer, and save a reference to list U self.U.append(Dense(param)) model.add(self.U[-1]) # ReLU activation # model.add(Activation(activation)) if activation == "arctan": model.add(Lambda(lambda x: tf.atan(x))) else: model.add(Activation(activation)) self.W = Dense(10) model.add(self.W) # output log probability, used for black-box attack if use_log: model.add(Activation('softmax')) if restore: model.load_weights(restore) layer_outputs = [] for layer in model.layers: if isinstance(layer, Conv2D) or isinstance(layer, Dense): layer_outputs.append( K.function([model.layers[0].input], [layer.output])) self.layer_outputs = layer_outputs self.model = model
def convert(file_name, new_name, cifar=False): if not cifar: eq_weights, new_params = get_weights(file_name) data = MNIST() else: eq_weights, new_params = get_weights(file_name, inp_shape=(32, 32, 3)) data = CIFAR() model = Sequential() model.add(Flatten(input_shape=data.train_data.shape[1:])) for param in new_params: model.add(Dense(param)) model.add(Lambda(lambda x: tf.nn.relu(x))) model.add(Dense(10)) for i in range(len(eq_weights)): try: print(eq_weights[i][0].shape) except: pass model.layers[i].set_weights(eq_weights[i]) sgd = SGD(lr=0.01, decay=1e-5, momentum=0.9, nesterov=True) model.compile(loss=fn, optimizer=sgd, metrics=['accuracy']) model.save(new_name) acc = model.evaluate(data.validation_data, data.validation_labels)[1] printlog("Converting CNN to MLP") nlayer = file_name.split('_')[-3][0] filters = file_name.split('_')[-2] kernel_size = file_name.split('_')[-1] printlog( "model name = {0}, numlayer = {1}, filters = {2}, kernel size = {3}". format(file_name, nlayer, filters, kernel_size)) printlog("Model accuracy: {:.3f}".format(acc)) printlog("-----------------------------------") return acc
def channel_attention(cost_volume): x = GlobalAveragePooling3D()(cost_volume) x = Lambda( lambda y: K.expand_dims(K.expand_dims(K.expand_dims(y, 1), 1), 1))(x) x = Conv3D(170, 1, 1, 'same')(x) x = Activation('relu')(x) x = Conv3D(15, 1, 1, 'same')(x) # [B, 1, 1, 1, 15] x = Activation('sigmoid')(x) # 15 -> 25 # 0 1 2 3 4 # 5 6 7 8 # 9 10 11 # 12 13 # 14 # # 0 1 2 3 4 # 1 5 6 7 8 # 2 6 9 10 11 # 3 7 10 12 13 # 4 8 11 13 14 x = Lambda(lambda y: K.concatenate([ y[:, :, :, :, 0:5], y[:, :, :, :, 1:2], y[:, :, :, :, 5:9], y[:, :, :, :, 2:3], y[:, :, :, :, 6:7], y[:, :, :, :, 9:12], y[:, :, :, :, 3:4], y[:, :, :, :, 7:8], y[:, :, :, :, 10:11], y[:, :, :, :, 12:14], y[:, :, :, :, 4:5], y[:, :, :, :, 8:9], y[:, :, :, :, 11:12], y[:, :, :, :, 13:15] ], axis=-1))(x) x = Lambda(lambda y: K.reshape(y, (K.shape(y)[0], 5, 5)))(x) x = Lambda(lambda y: tf.pad(y, [[0, 0], [0, 4], [0, 4]], 'REFLECT'))(x) attention = Lambda(lambda y: K.reshape(y, (K.shape(y)[0], 1, 1, 1, 81)))(x) x = Lambda(lambda y: K.repeat_elements(y, 4, -1))(attention) return multiply([x, cost_volume]), attention
def translate_start_out(x, start_coords): scope = Scoping.get_global_scope() with scope.name_scope('translate_start'): x = Lambda(lambda args: args[0] + args[1], name=scope + 'translate_start_out')([x, start_coords]) return x
def seq_to_angles_out(x, body_members, hip_coords, bone_len, fixed_angles): scope = Scoping.get_global_scope() with scope.name_scope('seq_to_angles'): members_from, members_to, body_graph = get_body_graph(body_members) x = Lambda(lambda args: K.concatenate(args, axis=1), name=scope + 'concat_angles')([fixed_angles, x]) x = Lambda(lambda arg: expmap_to_rotmat(arg), name=scope + 'rotmat')(x) # euler_out = Lambda(lambda arg: rotmat_to_euler(arg), name=scope+'euler')(x) def _get_coords(args): rotmat, bone_len = args rotmat_list = tf.unstack(rotmat, axis=1) bone_len_list = tf.unstack(bone_len, axis=1) base_shape = [int(d) for d in rotmat.shape] base_shape.pop(1) base_shape[-2] = 1 base_shape[-1] = 1 bone_idcs = { idx_tup: i for i, idx_tup in enumerate( [idx_tup for idx_tup in zip(members_from, members_to)]) } def _get_coords_for_joint(joint_idx, parent_idx, child_angle_idx, coords): if parent_idx is None: # joint_idx should be 0 coords[joint_idx] = K.zeros(base_shape[:-2] + [3, 1]) parent_bone = K.constant( np.concatenate([ np.ones(base_shape), np.zeros(base_shape), np.zeros(base_shape) ], axis=-2)) else: parent_bone = coords[parent_idx] - coords[joint_idx] parent_bone_norm = K.sqrt( K.sum(K.square(parent_bone), axis=-2, keepdims=True) + K.epsilon()) parent_bone = parent_bone / parent_bone_norm for child_idx in body_graph[joint_idx]: child_bone = tf.matmul(rotmat_list[child_angle_idx], parent_bone) child_bone_idx = bone_idcs[(joint_idx, child_idx)] child_bone = child_bone * K.reshape( bone_len_list[child_bone_idx], (child_bone.shape[0], 1, 1, 1)) coords[child_idx] = child_bone + coords[joint_idx] child_angle_idx += 1 for child_idx in body_graph[joint_idx]: child_angle_idx, coords = _get_coords_for_joint( child_idx, joint_idx, child_angle_idx, coords) return child_angle_idx, coords child_angle_idx, coords = _get_coords_for_joint(0, None, 0, {}) coords = K.stack([t for i, t in sorted(coords.iteritems())], axis=1) coords = K.squeeze(coords, axis=-1) return coords x = Lambda(_get_coords, name=scope + 'coords')([x, bone_len]) x = Lambda(lambda args: args[0] + args[1], name=scope + 'add_hip_coords')([x, hip_coords]) return x
def UpSampling2DBilinear(size): return Lambda( lambda x: tf.image.resize_bilinear(x, size, align_corners=True))