def rmac(input_shape, num_rois):
print('input_shape: %s' % str(input_shape))
# Load VGG16
vgg16_model = VGG16(weights='imagenet',
include_top=False,
input_shape=input_shape)
# Freeze the layers
for layer in vgg16_model.layers:
layer.trainable = False
# Check the trainable status of the individual layers
for layer in vgg16_model.layers:
print(layer, layer.trainable)
in_roi = Input(shape=(num_rois, 4), name='input_roi')
# ROI pooling
# every matrix use roi pooling, get? (1,14) vector, sum them, get a (1, 14) vector
x = RoiPooling([1], num_rois)([vgg16_model.layers[-5].output, in_roi])
# Normalization
#every vecter(1, 14) normalization
x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='norm1')(x)
model = Model([vgg16_model.input, in_roi], x)
return model
def rmac(model, num_rois):
# Regions as input
in_roi = Input(shape=(num_rois, 4), name='input_roi')
# ROI pooling
x = RoiPooling([1], num_rois)([model.layers[-5].output, in_roi])
# Normalization
x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='norm1')(x)
# PCA
x = TimeDistributed(
Dense(512,
name='pca',
kernel_initializer='identity',
bias_initializer='zeros'))(x)
# Normalization
x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='pca_norm')(x)
# Addition
rmac = Lambda(addition, output_shape=(512, ), name='rmac')(x)
# # Normalization
rmac_norm = Lambda(lambda x: K.l2_normalize(x, axis=1),
name='rmac_norm')(rmac)
# Define model
model = Model(input=[model.input, in_roi], output=rmac_norm)
## Load PCA weights
#mat = scipy.io.loadmat('data/PCAmatrices.mat')
#b = np.squeeze(mat['bias'], axis=1)
#w = np.transpose(mat['weights'])
#model.layers[-4].set_weights([w, b])
return model
def Rmac_net(image_a,roi_a,num_rois):
#create image_a model
# Block 1
# image_a = get_source_inputs(image_a)
# roi_a = get_source_inputs(roi_a)
img_a = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1')(image_a)
img_a = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2')(img_a)
img_a = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(img_a)
# Block 2
img_a = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1')(img_a)
img_a = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2')(img_a)
img_a = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(img_a)
# Block 3
img_a = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1')(img_a)
img_a = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2')(img_a)
img_a = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3')(img_a)
img_a = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(img_a)
# Block 4
img_a = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1')(img_a)
img_a = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2')(img_a)
img_a = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3')(img_a)
img_a = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(img_a)
# Block 5
img_a = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(img_a)
img_a = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(img_a)
img_a = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3')(img_a)
img_a = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(img_a)
# Get rmac regions with a RoiPooling layer.
# If batch size was 1, we end up with N_regions x D x pooled_h x pooled_w
# x = keras.layers.concatenate([x, rois_input])
# num_rois = len(roi_input)
x = RoiPooling([1], num_rois)([img_a, roi_a])
# out_roi_pool_a = RoiPoolingConv(pool_size=1, num_rois=num_rios)([img_a, roi_a])
# Normalization
x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='norm1')(x)
# PCA
x = TimeDistributed(Dense(512, name='pca',
kernel_initializer='identity',
bias_initializer='zeros'))(x)
# Normalization
x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='pca_norm')(x)
# Addition
rmac = Lambda(addition, output_shape=(512,), name='rmac')(x)
# # Normalization
embeding = Lambda(lambda x: K.l2_normalize(x, axis=1), name='rmac_norm')(rmac)
return embeding
def rmac(input_shape, num_rois):
#load ResNet101
resnet101_model = resnet.ResNet101(include_top=True, weights='imagenet', input_tensor=None, input_shape=(3, 224, 224),
pooling=None, classes=1000, backend=keras.backend, layers=keras.layers, models=keras.models, utils=keras.utils)
# Load VGG16
#vgg16_model = VGG16('', input_shape)
# vgg16_model = VGG16(include_top=True, weights='imagenet', input_tensor=None, input_shape=(3, 224, 224), pooling=None, classes=1000)
# Regions as input
in_roi = Input(shape=(num_rois, 4), name='input_roi')
#reshape
# xxx = K.permute_dimensions(vgg16_model.layers[-5].output, (0, 3, 1, 2))
# ROI pooling
layer_name = resnet101_model.layers[-4].name
layer_output = resnet101_model.layers[-4].output
print("layer name : " + layer_name)
print(layer_output)
# print('layer name : ' + vgg16_model.layers[-5].name)
# print(vgg16_model.layers[-5].output)
x = RoiPooling([1], num_rois)([layer_output, in_roi])
# Normalization
x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='norm1')(x)
# PCA
x = TimeDistributed(Dense(2048, name='pca',
kernel_initializer='identity',
bias_initializer='zeros'))(x)
# Normalization
x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='pca_norm')(x)
# Addition
rmac = Lambda(addition, output_shape=(2048,), name='rmac')(x)
# # Normalization
rmac_norm = Lambda(lambda x: K.l2_normalize(x, axis=1), name='rmac_norm')(rmac)
# Define model
model = Model([resnet101_model.input, in_roi], rmac_norm)
# Load PCA weights
#todo pca layer is trained by data ???
mat = scipy.io.loadmat(utils.DATA_DIR + utils.PCA_FILE)
b = np.squeeze(mat['bias'], axis=1)
w = np.transpose(mat['weights'])
model.layers[-4].set_weights([w, b])
return model
def rmac(input_shape, num_rois):
# Load VGG16
vgg_conv = VGG16(weights='imagenet',
include_top=False,
input_shape=input_shape)
#vgg16_model = VGG16(utils.DATA_DIR + utils.WEIGHTS_FILE, input_shape)
# Freeze the layers except the last 4 layers
for layer in vgg_conv.layers[:-4]:
layer.trainable = False
# Check the trainable status of the individual layers
#for layer in vgg_conv.layers:
#print(layer, layer.trainable)
# Regions as input
in_roi = Input(shape=(num_rois, 4), name='input_roi')
# ROI pooling
x = RoiPooling([1], num_rois)([vgg_conv.layers[-5].output, in_roi])
# Normalization
x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='norm1')(x)
# PCA
x = TimeDistributed(
Dense(512,
name='pca',
kernel_initializer='identity',
bias_initializer='zeros'))(x)
# Normalization
x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='pca_norm')(x)
# Addition
rmac = Lambda(addition, output_shape=(512, ), name='rmac')(x)
# # Normalization
rmac_norm = Lambda(lambda x: K.l2_normalize(x, axis=1),
name='rmac_norm')(rmac)
# Define model
model = Model([vgg_conv.input, in_roi], rmac_norm)
# Load PCA weights
mat = scipy.io.loadmat(utils.DATA_DIR + utils.PCA_FILE)
b = np.squeeze(mat['bias'], axis=1)
w = np.transpose(mat['weights'])
model.layers[-4].set_weights([w, b])
return model
def rmac(input_shape, num_rois):
# Load ResNet50
# vgg16_model = ResNet50(weights='imagenet')
vgg16_model = keras.models.load_model(
'../triplet_loss_resnet50.h5',
custom_objects={'triplet_loss': triplet_loss})
vgg16_model.summary()
# Regions as input
in_roi = Input(shape=(num_rois, 4), name='input_roi')
# Must check dimension when using finetuned model
resnet_model = vgg16_model.layers[-4]
# Second to last conv2 layer
x = RoiPooling([1], num_rois)([resnet_model.layers[-12].output, in_roi])
# Normalization
x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='norm1')(x)
# PCA
x = TimeDistributed(
Dense(512,
name='pca',
kernel_initializer='identity',
bias_initializer='zeros'))(x)
# Normalization
x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='pca_norm')(x)
# Addition
rmac = Lambda(addition, output_shape=(512, ), name='rmac')(x)
# # Normalization
rmac_norm = Lambda(lambda x: K.l2_normalize(x, axis=1),
name='rmac_norm')(rmac)
# Define model
model = Model([resnet_model.get_input_at(0), in_roi], rmac_norm)
# Load PCA weights
mat = scipy.io.loadmat(utils.DATA_DIR + utils.PCA_FILE)
b = np.squeeze(mat['bias'], axis=1)
w = np.transpose(mat['weights'])
model.layers[-4].set_weights([w, b])
# save model into .h5
return model
def rmac(input_shape, num_rois):
# Load VGG16
# vgg16_model = VGG16(utils.DATA_DIR + utils.WEIGHTS_FILE, input_shape)
vgg16_model = VGG16(input_shape=input_shape, include_top=False)
# Regions as input
in_roi = Input(shape=(num_rois, 4), name='input_roi')
# ROI pooling
x = RoiPooling([1], num_rois)([vgg16_model.layers[-1].output, in_roi])
# print('ROI pooling layer name:', vgg16_model.layers[-1].output)
# Normalization
x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='norm1')(x)
# PCA
x = TimeDistributed(
Dense(512,
name='pca',
kernel_initializer='identity',
bias_initializer='zeros'))(x)
# Normalization
x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='pca_norm')(x)
# Addition
rmac = Lambda(addition, output_shape=(512, ), name='rmac')(x)
# # Normalization
rmac_norm = Lambda(lambda x: K.l2_normalize(x, axis=1),
name='rmac_norm')(rmac)
# Define model
model = Model(inputs=[vgg16_model.input, in_roi], outputs=rmac_norm)
# Load PCA weights
mat = scipy.io.loadmat(utils.DATA_DIR + utils.PCA_FILE)
b = np.squeeze(mat['bias'], axis=1)
w = np.transpose(mat['weights'])
model.layers[-4].set_weights([w, b])
# print('layer name:', model.layers[-4].name)
# for idx, layer in enumerate(model.layers):
# print(idx, layer.name)
return model
def classifer(base_input, input_rois, num_rois, nb_classes=2, trainable=False):
pooling_regions = 14
input_shape = (num_rois, 14, 14, 1024)
out_roi_pooling = RoiPooling(pooling_regions,
num_rois)([base_input, input_rois])
out = classifer_layer(out_roi_pooling,
input_shape=input_shape,
trainable=True)
out = TimeDistributed(Flatten())(out)
out_class = TimeDistributed(Dense(nb_classes,
activation='softmax',
kernel_initializer='zero'),
name='dense_class_{}'.format(nb_classes))(out)
# note: no regression target for bg class
out_regr = TimeDistributed(Dense(4 * (nb_classes - 1),
activation='linear',
kernel_initializer='zero'),
name='dense_regress_{}'.format(nb_classes))(out)
return [out_class, out_regr]
def rmac(input_shape, num_rois, rank):
# Load VGG16
vgg16_model = VGG16(
utils.DATA_DIR + "vgg16_weights_th_dim_ordering_th_kernels_" +
str(rank) + ".h5", input_shape)
# Regions as input
in_roi = Input(shape=(num_rois, 4), name='input_roi')
# ROI pooling
x = RoiPooling([1], num_rois)([vgg16_model.layers[-5].output, in_roi])
# Normalization
x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='norm1')(x)
# PCA
x = TimeDistributed(
Dense(512,
name='pca',
kernel_initializer='identity',
bias_initializer='zeros'))(x)
# Normalization
x = Lambda(lambda x: K.l2_normalize(x, axis=2), name='pca_norm')(x)
# Addition
rmac = Lambda(addition, output_shape=(512, ), name='rmac')(x)
# # Normalization
rmac_norm = Lambda(lambda x: K.l2_normalize(x, axis=1),
name='rmac_norm')(rmac)
# Define model
model = Model([vgg16_model.input, in_roi], rmac_norm)
# Load PCA weights
mat = scipy.io.loadmat(utils.DATA_DIR + utils.PCA_FILE)
b = np.squeeze(mat['bias'], axis=1)
w = np.transpose(mat['weights'])
model.layers[-4].set_weights([w, b])
del vgg16_model
return model
def classcify(self, base_net, rois, num_classes=20):
num_classes += 1
pooling_layer = RoiPooling(pool_size=14, num_rois=10)(
base_net, rois) #[num_rois, pool_size, pool_size, channels]
pool_net = Conv2D(256, (3, 3),
padding='same',
strides=[1, 1],
kernel_initializer='glorot_normal',
activation='relu')(pooling_layer)
pool_net = MaxPooling2D((3, 3), strides=(2, 2),
padding='same')(pool_net)
pool_net = Conv2D(256, (3, 3),
padding='same',
strides=[1, 1],
kernel_initializer='glorot_normal',
activation='relu')(pool_net)
pool_net = AveragePooling2D((7, 7), strides=(1, 1),
padding='same')(pool_net)
classes = Conv2D(num_classes * 4, (1, 1),
padding='same',
strides=[1, 1],
kernel_initializer='unifotm',
activation='sigmoid')(pool_net)
classes = Conv2D(num_classes, (1, 1),
padding='same',
strides=[1, 1],
kernel_initializer='unifotm',
activation='sigmoid')(classes)
classes = tf.nn.softmax(classes)
offset = Conv2D(24, (1, 1),
padding='same',
strides=[1, 1],
kernel_initializer='zeros',
activation='tanh')(pool_net)
offset = Conv2D(4, (1, 1),
padding='same',
strides=[1, 1],
kernel_initializer='zeros',
activation='tanh')(offset)
return classes, offset
def body_model(include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
w_decay=None):
pooling_regions = [1]
num_rois = 1
num_channels = 3
#num_rois = 2 if input_tensor is None:
#num_channels = 3 img_input = Input(shape=input_shape)
# else:
# if not K.is_keras_tensor(input_tensor):
# img_input = Input(tensor=input_tensor, shape=input_shape)
# else:
# img_input = input_tensor
# roi
in_img = Input(shape=(96, num_channels))
in_roi = Input(shape=(num_rois, 4))
if K.image_data_format() == 'channels_first':
channel_axis = 1
else:
channel_axis = 3
input = Input(shape=(299, 299, 3))
# Conv1 ~ 6
conv_1 = conv2d_bn(input, 32, 3, 3, strides=(2, 2), padding="same")
conv_2 = conv2d_bn(conv_1, 32, 3, 3, strides=(2, 2), padding="same")
conv_3 = conv2d_bn(conv_2, 64, 3, 3)
pool_4 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2),
border_mode="same")(conv_3)
# inception module
incep_1a = inception_1a(pool_4)
incep_1b = inception_1b(incep_1a)
# TODO : ROI layer
roi = RoiPooling(pooling_regions, num_rois)([incep_1b, in_roi])
incep_2a = inception_2a(roi)
incep_2b = inception_2b(incep_2a)
incep_3a = inception_3a(incep_2b)
incep_3b = inception_3b(incep_3a)
glob_pool = GlobalAveragePooling2D(pool_size=(6, 6),
strides=(1, 1))(incep_3b)
fc7 = Dense(256, activation='relu',
kernel_initializer='glorot_normal')(glob_pool)
fc7 = BatchNormalization(scale=True)(fc7)
fc7 = Dropout(0.7)(fc7)
predictions = Dense(16161,
kernel_initializer='glorot_normal',
activation='softmax')(fc7)
model = Model(input, predictions)
return model
dim_ordering = K.image_dim_ordering()
assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
pooling_regions = [1, 2, 4]
num_rois = 2
num_channels = 3
if dim_ordering == 'tf':
in_img = Input(shape=(None, None, num_channels))
elif dim_ordering == 'th':
in_img = Input(shape=(num_channels, None, None))
in_roi = Input(shape=(num_rois, 4))
out_roi_pool = RoiPooling(pooling_regions, num_rois)([in_img, in_roi])
model = Model([in_img, in_roi], out_roi_pool)
model.summary()
model.compile(loss='mse', optimizer='sgd')
for img_size in [8, 16, 32]:
if dim_ordering == 'th':
X_img = np.random.rand(1, num_channels, img_size, img_size)
row_length = [float(X_img.shape[2]) / i for i in pooling_regions]
col_length = [float(X_img.shape[3]) / i for i in pooling_regions]
elif dim_ordering == 'tf':
X_img = np.random.rand(1, img_size, img_size, num_channels)
row_length = [float(X_img.shape[1]) / i for i in pooling_regions]
if mode == 'tf':
feature_map = np.zeros((h, w, 512))
elif mode == 'th':
feature_map = np.zeros((512, h, w))
for i in range(h):
for j in range(w):
if np.random.rand() < 0.1:
if mode == 'tf':
feature_map[i, j, :] = np.random.rand()
elif mode == 'th':
feature_map[:, i, j] = np.random.rand()
roi_batch = np.array([[0, 0, 10, 10], [2, 2, 5, 5]])
print(feature_map.shape)
roi_pooled = RoiPooling(mode=mode).get_pooled_rois(feature_map, roi_batch)
print(roi_pooled.shape)
# region = RoiPooling(mode=mode).get_region(feature_map, roi_batch[0])
if mode == 'tf':
_, ax = plt.subplots(2)
ax[0].imshow(feature_map[..., 0])
xmin, ymin, xmax, ymax = roi_batch[0]
ax[0].add_patch(
patches.Rectangle((xmin, ymin),
xmax - xmin,
ymax - ymin,
edgecolor='r',
facecolor='none',
linewidth=1))