def rmac(input_shape, num_rois): print('input_shape: %s' % str(input_shape)) # Load VGG16 vgg16_model = VGG16(weights='imagenet', include_top=False, input_shape=input_shape) # Freeze the layers for layer in vgg16_model.layers: layer.trainable = False # Check the trainable status of the individual layers for layer in vgg16_model.layers: print(layer, layer.trainable) in_roi = Input(shape=(num_rois, 4), name='input_roi') # ROI pooling # every matrix use roi pooling, get? (1,14) vector, sum them, get a (1, 14) vector x = RoiPooling([1], num_rois)([vgg16_model.layers[-5].output, in_roi]) # Normalization #every vecter(1, 14) normalization x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='norm1')(x) model = Model([vgg16_model.input, in_roi], x) return model
def rmac(model, num_rois): # Regions as input in_roi = Input(shape=(num_rois, 4), name='input_roi') # ROI pooling x = RoiPooling([1], num_rois)([model.layers[-5].output, in_roi]) # Normalization x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='norm1')(x) # PCA x = TimeDistributed( Dense(512, name='pca', kernel_initializer='identity', bias_initializer='zeros'))(x) # Normalization x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='pca_norm')(x) # Addition rmac = Lambda(addition, output_shape=(512, ), name='rmac')(x) # # Normalization rmac_norm = Lambda(lambda x: K.l2_normalize(x, axis=1), name='rmac_norm')(rmac) # Define model model = Model(input=[model.input, in_roi], output=rmac_norm) ## Load PCA weights #mat = scipy.io.loadmat('data/PCAmatrices.mat') #b = np.squeeze(mat['bias'], axis=1) #w = np.transpose(mat['weights']) #model.layers[-4].set_weights([w, b]) return model
def Rmac_net(image_a,roi_a,num_rois): #create image_a model # Block 1 # image_a = get_source_inputs(image_a) # roi_a = get_source_inputs(roi_a) img_a = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1')(image_a) img_a = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2')(img_a) img_a = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(img_a) # Block 2 img_a = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1')(img_a) img_a = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2')(img_a) img_a = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(img_a) # Block 3 img_a = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1')(img_a) img_a = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2')(img_a) img_a = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3')(img_a) img_a = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(img_a) # Block 4 img_a = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1')(img_a) img_a = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2')(img_a) img_a = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3')(img_a) img_a = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(img_a) # Block 5 img_a = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(img_a) img_a = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(img_a) img_a = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3')(img_a) img_a = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(img_a) # Get rmac regions with a RoiPooling layer. # If batch size was 1, we end up with N_regions x D x pooled_h x pooled_w # x = keras.layers.concatenate([x, rois_input]) # num_rois = len(roi_input) x = RoiPooling([1], num_rois)([img_a, roi_a]) # out_roi_pool_a = RoiPoolingConv(pool_size=1, num_rois=num_rios)([img_a, roi_a]) # Normalization x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='norm1')(x) # PCA x = TimeDistributed(Dense(512, name='pca', kernel_initializer='identity', bias_initializer='zeros'))(x) # Normalization x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='pca_norm')(x) # Addition rmac = Lambda(addition, output_shape=(512,), name='rmac')(x) # # Normalization embeding = Lambda(lambda x: K.l2_normalize(x, axis=1), name='rmac_norm')(rmac) return embeding
def rmac(input_shape, num_rois): #load ResNet101 resnet101_model = resnet.ResNet101(include_top=True, weights='imagenet', input_tensor=None, input_shape=(3, 224, 224), pooling=None, classes=1000, backend=keras.backend, layers=keras.layers, models=keras.models, utils=keras.utils) # Load VGG16 #vgg16_model = VGG16('', input_shape) # vgg16_model = VGG16(include_top=True, weights='imagenet', input_tensor=None, input_shape=(3, 224, 224), pooling=None, classes=1000) # Regions as input in_roi = Input(shape=(num_rois, 4), name='input_roi') #reshape # xxx = K.permute_dimensions(vgg16_model.layers[-5].output, (0, 3, 1, 2)) # ROI pooling layer_name = resnet101_model.layers[-4].name layer_output = resnet101_model.layers[-4].output print("layer name : " + layer_name) print(layer_output) # print('layer name : ' + vgg16_model.layers[-5].name) # print(vgg16_model.layers[-5].output) x = RoiPooling([1], num_rois)([layer_output, in_roi]) # Normalization x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='norm1')(x) # PCA x = TimeDistributed(Dense(2048, name='pca', kernel_initializer='identity', bias_initializer='zeros'))(x) # Normalization x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='pca_norm')(x) # Addition rmac = Lambda(addition, output_shape=(2048,), name='rmac')(x) # # Normalization rmac_norm = Lambda(lambda x: K.l2_normalize(x, axis=1), name='rmac_norm')(rmac) # Define model model = Model([resnet101_model.input, in_roi], rmac_norm) # Load PCA weights #todo pca layer is trained by data ??? mat = scipy.io.loadmat(utils.DATA_DIR + utils.PCA_FILE) b = np.squeeze(mat['bias'], axis=1) w = np.transpose(mat['weights']) model.layers[-4].set_weights([w, b]) return model
def rmac(input_shape, num_rois): # Load VGG16 vgg_conv = VGG16(weights='imagenet', include_top=False, input_shape=input_shape) #vgg16_model = VGG16(utils.DATA_DIR + utils.WEIGHTS_FILE, input_shape) # Freeze the layers except the last 4 layers for layer in vgg_conv.layers[:-4]: layer.trainable = False # Check the trainable status of the individual layers #for layer in vgg_conv.layers: #print(layer, layer.trainable) # Regions as input in_roi = Input(shape=(num_rois, 4), name='input_roi') # ROI pooling x = RoiPooling([1], num_rois)([vgg_conv.layers[-5].output, in_roi]) # Normalization x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='norm1')(x) # PCA x = TimeDistributed( Dense(512, name='pca', kernel_initializer='identity', bias_initializer='zeros'))(x) # Normalization x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='pca_norm')(x) # Addition rmac = Lambda(addition, output_shape=(512, ), name='rmac')(x) # # Normalization rmac_norm = Lambda(lambda x: K.l2_normalize(x, axis=1), name='rmac_norm')(rmac) # Define model model = Model([vgg_conv.input, in_roi], rmac_norm) # Load PCA weights mat = scipy.io.loadmat(utils.DATA_DIR + utils.PCA_FILE) b = np.squeeze(mat['bias'], axis=1) w = np.transpose(mat['weights']) model.layers[-4].set_weights([w, b]) return model
def rmac(input_shape, num_rois): # Load ResNet50 # vgg16_model = ResNet50(weights='imagenet') vgg16_model = keras.models.load_model( '../triplet_loss_resnet50.h5', custom_objects={'triplet_loss': triplet_loss}) vgg16_model.summary() # Regions as input in_roi = Input(shape=(num_rois, 4), name='input_roi') # Must check dimension when using finetuned model resnet_model = vgg16_model.layers[-4] # Second to last conv2 layer x = RoiPooling([1], num_rois)([resnet_model.layers[-12].output, in_roi]) # Normalization x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='norm1')(x) # PCA x = TimeDistributed( Dense(512, name='pca', kernel_initializer='identity', bias_initializer='zeros'))(x) # Normalization x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='pca_norm')(x) # Addition rmac = Lambda(addition, output_shape=(512, ), name='rmac')(x) # # Normalization rmac_norm = Lambda(lambda x: K.l2_normalize(x, axis=1), name='rmac_norm')(rmac) # Define model model = Model([resnet_model.get_input_at(0), in_roi], rmac_norm) # Load PCA weights mat = scipy.io.loadmat(utils.DATA_DIR + utils.PCA_FILE) b = np.squeeze(mat['bias'], axis=1) w = np.transpose(mat['weights']) model.layers[-4].set_weights([w, b]) # save model into .h5 return model
def rmac(input_shape, num_rois): # Load VGG16 # vgg16_model = VGG16(utils.DATA_DIR + utils.WEIGHTS_FILE, input_shape) vgg16_model = VGG16(input_shape=input_shape, include_top=False) # Regions as input in_roi = Input(shape=(num_rois, 4), name='input_roi') # ROI pooling x = RoiPooling([1], num_rois)([vgg16_model.layers[-1].output, in_roi]) # print('ROI pooling layer name:', vgg16_model.layers[-1].output) # Normalization x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='norm1')(x) # PCA x = TimeDistributed( Dense(512, name='pca', kernel_initializer='identity', bias_initializer='zeros'))(x) # Normalization x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='pca_norm')(x) # Addition rmac = Lambda(addition, output_shape=(512, ), name='rmac')(x) # # Normalization rmac_norm = Lambda(lambda x: K.l2_normalize(x, axis=1), name='rmac_norm')(rmac) # Define model model = Model(inputs=[vgg16_model.input, in_roi], outputs=rmac_norm) # Load PCA weights mat = scipy.io.loadmat(utils.DATA_DIR + utils.PCA_FILE) b = np.squeeze(mat['bias'], axis=1) w = np.transpose(mat['weights']) model.layers[-4].set_weights([w, b]) # print('layer name:', model.layers[-4].name) # for idx, layer in enumerate(model.layers): # print(idx, layer.name) return model
def classifer(base_input, input_rois, num_rois, nb_classes=2, trainable=False): pooling_regions = 14 input_shape = (num_rois, 14, 14, 1024) out_roi_pooling = RoiPooling(pooling_regions, num_rois)([base_input, input_rois]) out = classifer_layer(out_roi_pooling, input_shape=input_shape, trainable=True) out = TimeDistributed(Flatten())(out) out_class = TimeDistributed(Dense(nb_classes, activation='softmax', kernel_initializer='zero'), name='dense_class_{}'.format(nb_classes))(out) # note: no regression target for bg class out_regr = TimeDistributed(Dense(4 * (nb_classes - 1), activation='linear', kernel_initializer='zero'), name='dense_regress_{}'.format(nb_classes))(out) return [out_class, out_regr]
def rmac(input_shape, num_rois, rank): # Load VGG16 vgg16_model = VGG16( utils.DATA_DIR + "vgg16_weights_th_dim_ordering_th_kernels_" + str(rank) + ".h5", input_shape) # Regions as input in_roi = Input(shape=(num_rois, 4), name='input_roi') # ROI pooling x = RoiPooling([1], num_rois)([vgg16_model.layers[-5].output, in_roi]) # Normalization x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='norm1')(x) # PCA x = TimeDistributed( Dense(512, name='pca', kernel_initializer='identity', bias_initializer='zeros'))(x) # Normalization x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='pca_norm')(x) # Addition rmac = Lambda(addition, output_shape=(512, ), name='rmac')(x) # # Normalization rmac_norm = Lambda(lambda x: K.l2_normalize(x, axis=1), name='rmac_norm')(rmac) # Define model model = Model([vgg16_model.input, in_roi], rmac_norm) # Load PCA weights mat = scipy.io.loadmat(utils.DATA_DIR + utils.PCA_FILE) b = np.squeeze(mat['bias'], axis=1) w = np.transpose(mat['weights']) model.layers[-4].set_weights([w, b]) del vgg16_model return model
def classcify(self, base_net, rois, num_classes=20): num_classes += 1 pooling_layer = RoiPooling(pool_size=14, num_rois=10)( base_net, rois) #[num_rois, pool_size, pool_size, channels] pool_net = Conv2D(256, (3, 3), padding='same', strides=[1, 1], kernel_initializer='glorot_normal', activation='relu')(pooling_layer) pool_net = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(pool_net) pool_net = Conv2D(256, (3, 3), padding='same', strides=[1, 1], kernel_initializer='glorot_normal', activation='relu')(pool_net) pool_net = AveragePooling2D((7, 7), strides=(1, 1), padding='same')(pool_net) classes = Conv2D(num_classes * 4, (1, 1), padding='same', strides=[1, 1], kernel_initializer='unifotm', activation='sigmoid')(pool_net) classes = Conv2D(num_classes, (1, 1), padding='same', strides=[1, 1], kernel_initializer='unifotm', activation='sigmoid')(classes) classes = tf.nn.softmax(classes) offset = Conv2D(24, (1, 1), padding='same', strides=[1, 1], kernel_initializer='zeros', activation='tanh')(pool_net) offset = Conv2D(4, (1, 1), padding='same', strides=[1, 1], kernel_initializer='zeros', activation='tanh')(offset) return classes, offset
def body_model(include_top=True, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000, w_decay=None): pooling_regions = [1] num_rois = 1 num_channels = 3 #num_rois = 2 if input_tensor is None: #num_channels = 3 img_input = Input(shape=input_shape) # else: # if not K.is_keras_tensor(input_tensor): # img_input = Input(tensor=input_tensor, shape=input_shape) # else: # img_input = input_tensor # roi in_img = Input(shape=(96, num_channels)) in_roi = Input(shape=(num_rois, 4)) if K.image_data_format() == 'channels_first': channel_axis = 1 else: channel_axis = 3 input = Input(shape=(299, 299, 3)) # Conv1 ~ 6 conv_1 = conv2d_bn(input, 32, 3, 3, strides=(2, 2), padding="same") conv_2 = conv2d_bn(conv_1, 32, 3, 3, strides=(2, 2), padding="same") conv_3 = conv2d_bn(conv_2, 64, 3, 3) pool_4 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), border_mode="same")(conv_3) # inception module incep_1a = inception_1a(pool_4) incep_1b = inception_1b(incep_1a) # TODO : ROI layer roi = RoiPooling(pooling_regions, num_rois)([incep_1b, in_roi]) incep_2a = inception_2a(roi) incep_2b = inception_2b(incep_2a) incep_3a = inception_3a(incep_2b) incep_3b = inception_3b(incep_3a) glob_pool = GlobalAveragePooling2D(pool_size=(6, 6), strides=(1, 1))(incep_3b) fc7 = Dense(256, activation='relu', kernel_initializer='glorot_normal')(glob_pool) fc7 = BatchNormalization(scale=True)(fc7) fc7 = Dropout(0.7)(fc7) predictions = Dense(16161, kernel_initializer='glorot_normal', activation='softmax')(fc7) model = Model(input, predictions) return model
dim_ordering = K.image_dim_ordering() assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}' pooling_regions = [1, 2, 4] num_rois = 2 num_channels = 3 if dim_ordering == 'tf': in_img = Input(shape=(None, None, num_channels)) elif dim_ordering == 'th': in_img = Input(shape=(num_channels, None, None)) in_roi = Input(shape=(num_rois, 4)) out_roi_pool = RoiPooling(pooling_regions, num_rois)([in_img, in_roi]) model = Model([in_img, in_roi], out_roi_pool) model.summary() model.compile(loss='mse', optimizer='sgd') for img_size in [8, 16, 32]: if dim_ordering == 'th': X_img = np.random.rand(1, num_channels, img_size, img_size) row_length = [float(X_img.shape[2]) / i for i in pooling_regions] col_length = [float(X_img.shape[3]) / i for i in pooling_regions] elif dim_ordering == 'tf': X_img = np.random.rand(1, img_size, img_size, num_channels) row_length = [float(X_img.shape[1]) / i for i in pooling_regions]
if mode == 'tf': feature_map = np.zeros((h, w, 512)) elif mode == 'th': feature_map = np.zeros((512, h, w)) for i in range(h): for j in range(w): if np.random.rand() < 0.1: if mode == 'tf': feature_map[i, j, :] = np.random.rand() elif mode == 'th': feature_map[:, i, j] = np.random.rand() roi_batch = np.array([[0, 0, 10, 10], [2, 2, 5, 5]]) print(feature_map.shape) roi_pooled = RoiPooling(mode=mode).get_pooled_rois(feature_map, roi_batch) print(roi_pooled.shape) # region = RoiPooling(mode=mode).get_region(feature_map, roi_batch[0]) if mode == 'tf': _, ax = plt.subplots(2) ax[0].imshow(feature_map[..., 0]) xmin, ymin, xmax, ymax = roi_batch[0] ax[0].add_patch( patches.Rectangle((xmin, ymin), xmax - xmin, ymax - ymin, edgecolor='r', facecolor='none', linewidth=1))