def create_googlenet(weights_path=None):
# creates GoogLeNet a.k.a. Inception v1 (Szegedy, 2015)
input = Input(shape=(3, 224, 224))
conv1_7x7_s2 = Convolution2D(64,
7,
7,
subsample=(2, 2),
border_mode='same',
activation='relu',
name='conv1/7x7_s2',
W_regularizer=l2(0.0002))(input)
conv1_zero_pad = ZeroPadding2D(padding=(1, 1))(conv1_7x7_s2)
pool1_helper = PoolHelper()(conv1_zero_pad)
pool1_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool1/3x3_s2')(pool1_helper)
pool1_norm1 = LRN(name='pool1/norm1')(pool1_3x3_s2)
conv2_3x3_reduce = Convolution2D(64,
1,
1,
border_mode='same',
activation='relu',
name='conv2/3x3_reduce',
W_regularizer=l2(0.0002))(pool1_norm1)
conv2_3x3 = Convolution2D(192,
3,
3,
border_mode='same',
activation='relu',
name='conv2/3x3',
W_regularizer=l2(0.0002))(conv2_3x3_reduce)
conv2_norm2 = LRN(name='conv2/norm2')(conv2_3x3)
conv2_zero_pad = ZeroPadding2D(padding=(1, 1))(conv2_norm2)
pool2_helper = PoolHelper()(conv2_zero_pad)
pool2_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool2/3x3_s2')(pool2_helper)
inception_3a_1x1 = Convolution2D(64,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/1x1',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3_reduce = Convolution2D(
96,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/3x3_reduce',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3 = Convolution2D(
128,
3,
3,
border_mode='same',
activation='relu',
name='inception_3a/3x3',
W_regularizer=l2(0.0002))(inception_3a_3x3_reduce)
inception_3a_5x5_reduce = Convolution2D(
16,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/5x5_reduce',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_5x5 = Convolution2D(
32,
5,
5,
border_mode='same',
activation='relu',
name='inception_3a/5x5',
W_regularizer=l2(0.0002))(inception_3a_5x5_reduce)
inception_3a_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_3a/pool')(pool2_3x3_s2)
inception_3a_pool_proj = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/pool_proj',
W_regularizer=l2(0.0002))(inception_3a_pool)
inception_3a_output = merge([
inception_3a_1x1, inception_3a_3x3, inception_3a_5x5,
inception_3a_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_3a/output')
inception_3b_1x1 = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/1x1',
W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3_reduce = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/3x3_reduce',
W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3 = Convolution2D(
192,
3,
3,
border_mode='same',
activation='relu',
name='inception_3b/3x3',
W_regularizer=l2(0.0002))(inception_3b_3x3_reduce)
inception_3b_5x5_reduce = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/5x5_reduce',
W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_5x5 = Convolution2D(
96,
5,
5,
border_mode='same',
activation='relu',
name='inception_3b/5x5',
W_regularizer=l2(0.0002))(inception_3b_5x5_reduce)
inception_3b_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_3b/pool')(inception_3a_output)
inception_3b_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/pool_proj',
W_regularizer=l2(0.0002))(inception_3b_pool)
inception_3b_output = merge([
inception_3b_1x1, inception_3b_3x3, inception_3b_5x5,
inception_3b_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_3b/output')
inception_3b_output_zero_pad = ZeroPadding2D(
padding=(1, 1))(inception_3b_output)
pool3_helper = PoolHelper()(inception_3b_output_zero_pad)
pool3_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool3/3x3_s2')(pool3_helper)
inception_4a_1x1 = Convolution2D(192,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/1x1',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3_reduce = Convolution2D(
96,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/3x3_reduce',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3 = Convolution2D(
208,
3,
3,
border_mode='same',
activation='relu',
name='inception_4a/3x3',
W_regularizer=l2(0.0002))(inception_4a_3x3_reduce)
inception_4a_5x5_reduce = Convolution2D(
16,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/5x5_reduce',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_5x5 = Convolution2D(
48,
5,
5,
border_mode='same',
activation='relu',
name='inception_4a/5x5',
W_regularizer=l2(0.0002))(inception_4a_5x5_reduce)
inception_4a_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4a/pool')(pool3_3x3_s2)
inception_4a_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/pool_proj',
W_regularizer=l2(0.0002))(inception_4a_pool)
inception_4a_output = merge([
inception_4a_1x1, inception_4a_3x3, inception_4a_5x5,
inception_4a_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4a/output')
loss1_ave_pool = AveragePooling2D(
pool_size=(5, 5), strides=(3, 3),
name='loss1/ave_pool')(inception_4a_output)
loss1_conv = Convolution2D(128,
1,
1,
border_mode='same',
activation='relu',
name='loss1/conv',
W_regularizer=l2(0.0002))(loss1_ave_pool)
loss1_flat = Flatten()(loss1_conv)
loss1_fc = Dense(1024,
activation='relu',
name='loss1/fc',
W_regularizer=l2(0.0002))(loss1_flat)
loss1_drop_fc = Dropout(0.7)(loss1_fc)
loss1_classifier = Dense(1000,
name='loss1/classifier',
W_regularizer=l2(0.0002))(loss1_drop_fc)
loss1_classifier_act = Activation('softmax')(loss1_classifier)
inception_4b_1x1 = Convolution2D(
160,
1,
1,
border_mode='same',
activation='relu',
name='inception_4b/1x1',
W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3_reduce = Convolution2D(
112,
1,
1,
border_mode='same',
activation='relu',
name='inception_4b/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3 = Convolution2D(
224,
3,
3,
border_mode='same',
activation='relu',
name='inception_4b/3x3',
W_regularizer=l2(0.0002))(inception_4b_3x3_reduce)
inception_4b_5x5_reduce = Convolution2D(
24,
1,
1,
border_mode='same',
activation='relu',
name='inception_4b/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_5x5 = Convolution2D(
64,
5,
5,
border_mode='same',
activation='relu',
name='inception_4b/5x5',
W_regularizer=l2(0.0002))(inception_4b_5x5_reduce)
inception_4b_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4b/pool')(inception_4a_output)
inception_4b_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_4b/pool_proj',
W_regularizer=l2(0.0002))(inception_4b_pool)
inception_4b_output = merge([
inception_4b_1x1, inception_4b_3x3, inception_4b_5x5,
inception_4b_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4b_output')
inception_4c_1x1 = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_4c/1x1',
W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3_reduce = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_4c/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3 = Convolution2D(
256,
3,
3,
border_mode='same',
activation='relu',
name='inception_4c/3x3',
W_regularizer=l2(0.0002))(inception_4c_3x3_reduce)
inception_4c_5x5_reduce = Convolution2D(
24,
1,
1,
border_mode='same',
activation='relu',
name='inception_4c/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_5x5 = Convolution2D(
64,
5,
5,
border_mode='same',
activation='relu',
name='inception_4c/5x5',
W_regularizer=l2(0.0002))(inception_4c_5x5_reduce)
inception_4c_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4c/pool')(inception_4b_output)
inception_4c_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_4c/pool_proj',
W_regularizer=l2(0.0002))(inception_4c_pool)
inception_4c_output = merge([
inception_4c_1x1, inception_4c_3x3, inception_4c_5x5,
inception_4c_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4c/output')
inception_4d_1x1 = Convolution2D(
112,
1,
1,
border_mode='same',
activation='relu',
name='inception_4d/1x1',
W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3_reduce = Convolution2D(
144,
1,
1,
border_mode='same',
activation='relu',
name='inception_4d/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3 = Convolution2D(
288,
3,
3,
border_mode='same',
activation='relu',
name='inception_4d/3x3',
W_regularizer=l2(0.0002))(inception_4d_3x3_reduce)
inception_4d_5x5_reduce = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_4d/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_5x5 = Convolution2D(
64,
5,
5,
border_mode='same',
activation='relu',
name='inception_4d/5x5',
W_regularizer=l2(0.0002))(inception_4d_5x5_reduce)
inception_4d_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4d/pool')(inception_4c_output)
inception_4d_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_4d/pool_proj',
W_regularizer=l2(0.0002))(inception_4d_pool)
inception_4d_output = merge([
inception_4d_1x1, inception_4d_3x3, inception_4d_5x5,
inception_4d_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4d/output')
loss2_ave_pool = AveragePooling2D(
pool_size=(5, 5), strides=(3, 3),
name='loss2/ave_pool')(inception_4d_output)
loss2_conv = Convolution2D(128,
1,
1,
border_mode='same',
activation='relu',
name='loss2/conv',
W_regularizer=l2(0.0002))(loss2_ave_pool)
loss2_flat = Flatten()(loss2_conv)
loss2_fc = Dense(1024,
activation='relu',
name='loss2/fc',
W_regularizer=l2(0.0002))(loss2_flat)
loss2_drop_fc = Dropout(0.7)(loss2_fc)
loss2_classifier = Dense(1000,
name='loss2/classifier',
W_regularizer=l2(0.0002))(loss2_drop_fc)
loss2_classifier_act = Activation('softmax')(loss2_classifier)
inception_4e_1x1 = Convolution2D(
256,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/1x1',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_reduce = Convolution2D(
160,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3 = Convolution2D(
320,
3,
3,
border_mode='same',
activation='relu',
name='inception_4e/3x3',
W_regularizer=l2(0.0002))(inception_4e_3x3_reduce)
inception_4e_5x5_reduce = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_5x5 = Convolution2D(
128,
5,
5,
border_mode='same',
activation='relu',
name='inception_4e/5x5',
W_regularizer=l2(0.0002))(inception_4e_5x5_reduce)
inception_4e_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4e/pool')(inception_4d_output)
inception_4e_pool_proj = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/pool_proj',
W_regularizer=l2(0.0002))(inception_4e_pool)
inception_4e_output = merge([
inception_4e_1x1, inception_4e_3x3, inception_4e_5x5,
inception_4e_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4e/output')
inception_4e_output_zero_pad = ZeroPadding2D(
padding=(1, 1))(inception_4e_output)
pool4_helper = PoolHelper()(inception_4e_output_zero_pad)
pool4_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool4/3x3_s2')(pool4_helper)
inception_5a_1x1 = Convolution2D(256,
1,
1,
border_mode='same',
activation='relu',
name='inception_5a/1x1',
W_regularizer=l2(0.0002))(pool4_3x3_s2)
inception_5a_3x3_reduce = Convolution2D(
160,
1,
1,
border_mode='same',
activation='relu',
name='inception_5a/3x3_reduce',
W_regularizer=l2(0.0002))(pool4_3x3_s2)
inception_5a_3x3 = Convolution2D(
320,
3,
3,
border_mode='same',
activation='relu',
name='inception_5a/3x3',
W_regularizer=l2(0.0002))(inception_5a_3x3_reduce)
inception_5a_5x5_reduce = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_5a/5x5_reduce',
W_regularizer=l2(0.0002))(pool4_3x3_s2)
inception_5a_5x5 = Convolution2D(
128,
5,
5,
border_mode='same',
activation='relu',
name='inception_5a/5x5',
W_regularizer=l2(0.0002))(inception_5a_5x5_reduce)
inception_5a_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_5a/pool')(pool4_3x3_s2)
inception_5a_pool_proj = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_5a/pool_proj',
W_regularizer=l2(0.0002))(inception_5a_pool)
inception_5a_output = merge([
inception_5a_1x1, inception_5a_3x3, inception_5a_5x5,
inception_5a_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_5a/output')
inception_5b_1x1 = Convolution2D(
384,
1,
1,
border_mode='same',
activation='relu',
name='inception_5b/1x1',
W_regularizer=l2(0.0002))(inception_5a_output)
inception_5b_3x3_reduce = Convolution2D(
192,
1,
1,
border_mode='same',
activation='relu',
name='inception_5b/3x3_reduce',
W_regularizer=l2(0.0002))(inception_5a_output)
inception_5b_3x3 = Convolution2D(
384,
3,
3,
border_mode='same',
activation='relu',
name='inception_5b/3x3',
W_regularizer=l2(0.0002))(inception_5b_3x3_reduce)
inception_5b_5x5_reduce = Convolution2D(
48,
1,
1,
border_mode='same',
activation='relu',
name='inception_5b/5x5_reduce',
W_regularizer=l2(0.0002))(inception_5a_output)
inception_5b_5x5 = Convolution2D(
128,
5,
5,
border_mode='same',
activation='relu',
name='inception_5b/5x5',
W_regularizer=l2(0.0002))(inception_5b_5x5_reduce)
inception_5b_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_5b/pool')(inception_5a_output)
inception_5b_pool_proj = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_5b/pool_proj',
W_regularizer=l2(0.0002))(inception_5b_pool)
inception_5b_output = merge([
inception_5b_1x1, inception_5b_3x3, inception_5b_5x5,
inception_5b_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_5b/output')
pool5_7x7_s1 = AveragePooling2D(pool_size=(7, 7),
strides=(1, 1),
name='pool5/7x7_s2')(inception_5b_output)
loss3_flat = Flatten()(pool5_7x7_s1)
pool5_drop_7x7_s1 = Dropout(0.4)(loss3_flat)
loss3_classifier = Dense(1000,
name='loss3/classifier',
W_regularizer=l2(0.0002))(pool5_drop_7x7_s1)
loss3_classifier_act = Activation('softmax', name='prob')(loss3_classifier)
googlenet = Model(input=input,
output=[
loss1_classifier_act, loss2_classifier_act,
loss3_classifier_act
])
if weights_path:
googlenet.load_weights(weights_path)
return googlenet
def create_model(weights_path=None, use_distribution=False, use_multigap=False,
use_semantics=False, rapid_style=False, use_comments=False, embedding_layer=None,
extra_conv_layer=False,load_weights_by_name=True,textInputMaxLength=100, embedding_dim=300):
input_image = Input(shape=(3, 224, 224))
if rapid_style:
if not use_semantics:
print("[WARN] Semantics is not enabled, rapid style parameter will be ignored")
if use_comments:
if embedding_layer is None:
print("[ERROR] Embedding layer is required for creating comments model")
return None
comment_input = Input(shape=(textInputMaxLength,), dtype='int32')
embedded_sequences = embedding_layer(comment_input)
x_text_aesthetics = GRU(embedding_dim,dropout_W = 0.3,dropout_U = 0.3,return_sequences=True, name='gru_aesthetics_1')(embedded_sequences)
x_text_aesthetics = GRU(embedding_dim,dropout_W = 0.3,dropout_U = 0.3,name='gru_aesthetics_2')(x_text_aesthetics)
if use_semantics and not rapid_style:
x_text_semantics = GRU(embedding_dim,dropout_W = 0.3,dropout_U = 0.3,return_sequences=True, name='gru_semantics_1')(embedded_sequences)
x_text_semantics = GRU(embedding_dim,dropout_W = 0.3,dropout_U = 0.3, name='gru_semantics_2')(x_text_semantics)
conv1_7x7_s2 = Convolution2D(64,7,7,subsample=(2,2),border_mode='same',activation='relu',name='conv1/7x7_s2',W_regularizer=l2(0.0002))(input_image)
conv1_zero_pad = ZeroPadding2D(padding=(1, 1))(conv1_7x7_s2)
pool1_helper = PoolHelper()(conv1_zero_pad)
pool1_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool1/3x3_s2')(pool1_helper)
pool1_norm1 = LRN(name='pool1/norm1')(pool1_3x3_s2)
conv2_3x3_reduce = Convolution2D(64,1,1,border_mode='same',activation='relu',name='conv2/3x3_reduce',W_regularizer=l2(0.0002))(pool1_norm1)
conv2_3x3 = Convolution2D(192,3,3,border_mode='same',activation='relu',name='conv2/3x3',W_regularizer=l2(0.0002))(conv2_3x3_reduce)
conv2_norm2 = LRN(name='conv2/norm2')(conv2_3x3)
conv2_zero_pad = ZeroPadding2D(padding=(1, 1))(conv2_norm2)
pool2_helper = PoolHelper()(conv2_zero_pad)
pool2_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool2/3x3_s2')(pool2_helper)
inception_3a_1x1 = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_3a/1x1',W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3_reduce = Convolution2D(96,1,1,border_mode='same',activation='relu',name='inception_3a/3x3_reduce',W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3 = Convolution2D(128,3,3,border_mode='same',activation='relu',name='inception_3a/3x3',W_regularizer=l2(0.0002))(inception_3a_3x3_reduce)
inception_3a_5x5_reduce = Convolution2D(16,1,1,border_mode='same',activation='relu',name='inception_3a/5x5_reduce',W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_5x5 = Convolution2D(32,5,5,border_mode='same',activation='relu',name='inception_3a/5x5',W_regularizer=l2(0.0002))(inception_3a_5x5_reduce)
inception_3a_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_3a/pool')(pool2_3x3_s2)
inception_3a_pool_proj = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_3a/pool_proj',W_regularizer=l2(0.0002))(inception_3a_pool)
inception_3a_output = merge([inception_3a_1x1,inception_3a_3x3,inception_3a_5x5,inception_3a_pool_proj],mode='concat',concat_axis=1,name='inception_3a/output')
inception_3b_1x1 = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_3b/1x1',W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3_reduce = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_3b/3x3_reduce',W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3 = Convolution2D(192,3,3,border_mode='same',activation='relu',name='inception_3b/3x3',W_regularizer=l2(0.0002))(inception_3b_3x3_reduce)
inception_3b_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_3b/5x5_reduce',W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_5x5 = Convolution2D(96,5,5,border_mode='same',activation='relu',name='inception_3b/5x5',W_regularizer=l2(0.0002))(inception_3b_5x5_reduce)
inception_3b_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_3b/pool')(inception_3a_output)
inception_3b_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_3b/pool_proj',W_regularizer=l2(0.0002))(inception_3b_pool)
inception_3b_output = merge([inception_3b_1x1,inception_3b_3x3,inception_3b_5x5,inception_3b_pool_proj],mode='concat',concat_axis=1,name='inception_3b/output')
inception_3b_output_zero_pad = ZeroPadding2D(padding=(1, 1))(inception_3b_output)
pool3_helper = PoolHelper()(inception_3b_output_zero_pad)
pool3_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool3/3x3_s2')(pool3_helper)
inception_4a_1x1 = Convolution2D(192,1,1,border_mode='same',activation='relu',name='inception_4a/1x1',W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3_reduce = Convolution2D(96,1,1,border_mode='same',activation='relu',name='inception_4a/3x3_reduce',W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3 = Convolution2D(208,3,3,border_mode='same',activation='relu',name='inception_4a/3x3',W_regularizer=l2(0.0002))(inception_4a_3x3_reduce)
inception_4a_5x5_reduce = Convolution2D(16,1,1,border_mode='same',activation='relu',name='inception_4a/5x5_reduce',W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_5x5 = Convolution2D(48,5,5,border_mode='same',activation='relu',name='inception_4a/5x5',W_regularizer=l2(0.0002))(inception_4a_5x5_reduce)
inception_4a_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4a/pool')(pool3_3x3_s2)
inception_4a_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_4a/pool_proj',W_regularizer=l2(0.0002))(inception_4a_pool)
inception_4a_output = merge([inception_4a_1x1,inception_4a_3x3,inception_4a_5x5,inception_4a_pool_proj],mode='concat',concat_axis=1,name='inception_4a/output')
if extra_conv_layer:
conv_4a_output = Convolution2D(624, 3, 3, activation='relu',name='conv_4a',border_mode = 'same',W_regularizer=l2(0.0002))(inception_4a_output)
inception_4a_gap = GlobalAveragePooling2D()(conv_4a_output)
inception_4b_1x1 = Convolution2D(160,1,1,border_mode='same',activation='relu',name='inception_4b/1x1',W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3_reduce = Convolution2D(112,1,1,border_mode='same',activation='relu',name='inception_4b/3x3_reduce',W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3 = Convolution2D(224,3,3,border_mode='same',activation='relu',name='inception_4b/3x3',W_regularizer=l2(0.0002))(inception_4b_3x3_reduce)
inception_4b_5x5_reduce = Convolution2D(24,1,1,border_mode='same',activation='relu',name='inception_4b/5x5_reduce',W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='inception_4b/5x5',W_regularizer=l2(0.0002))(inception_4b_5x5_reduce)
inception_4b_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4b/pool')(inception_4a_output)
inception_4b_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_4b/pool_proj',W_regularizer=l2(0.0002))(inception_4b_pool)
inception_4b_output = merge([inception_4b_1x1,inception_4b_3x3,inception_4b_5x5,inception_4b_pool_proj],mode='concat',concat_axis=1,name='inception_4b_output')
if extra_conv_layer:
conv_4b_output = Convolution2D(648, 3, 3, activation='relu',name='conv_4b',border_mode = 'same',W_regularizer=l2(0.0002))(inception_4b_output)
inception_4b_gap = GlobalAveragePooling2D()(conv_4b_output)
inception_4c_1x1 = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_4c/1x1',W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3_reduce = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_4c/3x3_reduce',W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3 = Convolution2D(256,3,3,border_mode='same',activation='relu',name='inception_4c/3x3',W_regularizer=l2(0.0002))(inception_4c_3x3_reduce)
inception_4c_5x5_reduce = Convolution2D(24,1,1,border_mode='same',activation='relu',name='inception_4c/5x5_reduce',W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='inception_4c/5x5',W_regularizer=l2(0.0002))(inception_4c_5x5_reduce)
inception_4c_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4c/pool')(inception_4b_output)
inception_4c_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_4c/pool_proj',W_regularizer=l2(0.0002))(inception_4c_pool)
inception_4c_output = merge([inception_4c_1x1,inception_4c_3x3,inception_4c_5x5,inception_4c_pool_proj],mode='concat',concat_axis=1,name='inception_4c/output')
if extra_conv_layer:
conv_4c_output = Convolution2D(663, 3, 3, activation='relu',name='conv_4c',border_mode = 'same',W_regularizer=l2(0.0002))(inception_4c_output)
inception_4c_gap = GlobalAveragePooling2D()(conv_4c_output)
inception_4d_1x1 = Convolution2D(112,1,1,border_mode='same',activation='relu',name='inception_4d/1x1',W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3_reduce = Convolution2D(144,1,1,border_mode='same',activation='relu',name='inception_4d/3x3_reduce',W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3 = Convolution2D(288,3,3,border_mode='same',activation='relu',name='inception_4d/3x3',W_regularizer=l2(0.0002))(inception_4d_3x3_reduce)
inception_4d_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_4d/5x5_reduce',W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='inception_4d/5x5',W_regularizer=l2(0.0002))(inception_4d_5x5_reduce)
inception_4d_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4d/pool')(inception_4c_output)
inception_4d_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_4d/pool_proj',W_regularizer=l2(0.0002))(inception_4d_pool)
inception_4d_output = merge([inception_4d_1x1,inception_4d_3x3,inception_4d_5x5,inception_4d_pool_proj],mode='concat',concat_axis=1,name='inception_4d/output')
if extra_conv_layer:
conv_4d_output = Convolution2D(704, 3, 3, activation='relu',name='conv_4d',border_mode = 'same',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4d_gap = GlobalAveragePooling2D()(conv_4d_output)
if use_semantics:
inception_4e_1x1_aesthetics = Convolution2D(256,1,1,border_mode='same',activation='relu',name='inception_4e/1x1_aesthetics',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_reduce_aesthetics = Convolution2D(160,1,1,border_mode='same',activation='relu',name='inception_4e/3x3_reduce_aesthetics',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_aesthetics= Convolution2D(320,3,3,border_mode='same',activation='relu',name='inception_4e/3x3_aesthetics',W_regularizer=l2(0.0002))(inception_4e_3x3_reduce_aesthetics)
inception_4e_5x5_reduce_aesthetics = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_4e/5x5_reduce_aesthetics',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_5x5_aesthetics = Convolution2D(128,5,5,border_mode='same',activation='relu',name='inception_4e/5x5_aesthetics',W_regularizer=l2(0.0002))(inception_4e_5x5_reduce_aesthetics)
inception_4e_pool_aesthetics = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4e/pool_aesthetics')(inception_4d_output)
inception_4e_pool_proj_aesthetics = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_4e/pool_proj_aesthetics',W_regularizer=l2(0.0002))(inception_4e_pool_aesthetics)
inception_4e_output_aesthetics = merge([inception_4e_1x1_aesthetics,inception_4e_3x3_aesthetics,inception_4e_5x5_aesthetics,inception_4e_pool_proj_aesthetics],mode='concat',concat_axis=1,name='inception_4e/output_aesthetics')
conv_output_aesthetics = Convolution2D(1024, 3, 3, activation='relu',name='conv_6_1',border_mode = 'same',W_regularizer=l2(0.0002))(inception_4e_output_aesthetics)
if rapid_style:
conv1_7x7_s2 = Convolution2D(64,7,7,subsample=(2,2),border_mode='same',activation='relu',name='semantic_conv1/7x7_s2',W_regularizer=l2(0.0002))(input_image)
conv1_zero_pad = ZeroPadding2D(padding=(1, 1))(conv1_7x7_s2)
pool1_helper = PoolHelper()(conv1_zero_pad)
pool1_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='semantic_pool1/3x3_s2')(pool1_helper)
pool1_norm1 = LRN(name='semantic_pool1/norm1')(pool1_3x3_s2)
conv2_3x3_reduce = Convolution2D(64,1,1,border_mode='same',activation='relu',name='semantic_conv2/3x3_reduce',W_regularizer=l2(0.0002))(pool1_norm1)
conv2_3x3 = Convolution2D(192,3,3,border_mode='same',activation='relu',name='semantic_conv2/3x3',W_regularizer=l2(0.0002))(conv2_3x3_reduce)
conv2_norm2 = LRN(name='semantic_conv2/norm2')(conv2_3x3)
conv2_zero_pad = ZeroPadding2D(padding=(1, 1))(conv2_norm2)
pool2_helper = PoolHelper()(conv2_zero_pad)
pool2_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='semantic_pool2/3x3_s2')(pool2_helper)
inception_3a_1x1 = Convolution2D(64,1,1,border_mode='same',activation='relu',name='semantic_inception_3a/1x1',W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3_reduce = Convolution2D(96,1,1,border_mode='same',activation='relu',name='semantic_inception_3a/3x3_reduce',W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3 = Convolution2D(128,3,3,border_mode='same',activation='relu',name='semantic_inception_3a/3x3',W_regularizer=l2(0.0002))(inception_3a_3x3_reduce)
inception_3a_5x5_reduce = Convolution2D(16,1,1,border_mode='same',activation='relu',name='semantic_inception_3a/5x5_reduce',W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_5x5 = Convolution2D(32,5,5,border_mode='same',activation='relu',name='semantic_inception_3a/5x5',W_regularizer=l2(0.0002))(inception_3a_5x5_reduce)
inception_3a_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='semantic_inception_3a/pool')(pool2_3x3_s2)
inception_3a_pool_proj = Convolution2D(32,1,1,border_mode='same',activation='relu',name='semantic_inception_3a/pool_proj',W_regularizer=l2(0.0002))(inception_3a_pool)
inception_3a_output = merge([inception_3a_1x1,inception_3a_3x3,inception_3a_5x5,inception_3a_pool_proj],mode='concat',concat_axis=1,name='semantic_inception_3a/output')
inception_3b_1x1 = Convolution2D(128,1,1,border_mode='same',activation='relu',name='semantic_inception_3b/1x1',W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3_reduce = Convolution2D(128,1,1,border_mode='same',activation='relu',name='semantic_inception_3b/3x3_reduce',W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3 = Convolution2D(192,3,3,border_mode='same',activation='relu',name='semantic_inception_3b/3x3',W_regularizer=l2(0.0002))(inception_3b_3x3_reduce)
inception_3b_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='semantic_inception_3b/5x5_reduce',W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_5x5 = Convolution2D(96,5,5,border_mode='same',activation='relu',name='semantic_inception_3b/5x5',W_regularizer=l2(0.0002))(inception_3b_5x5_reduce)
inception_3b_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='semantic_inception_3b/pool')(inception_3a_output)
inception_3b_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='semantic_inception_3b/pool_proj',W_regularizer=l2(0.0002))(inception_3b_pool)
inception_3b_output = merge([inception_3b_1x1,inception_3b_3x3,inception_3b_5x5,inception_3b_pool_proj],mode='concat',concat_axis=1,name='semantic_inception_3b/output')
inception_3b_output_zero_pad = ZeroPadding2D(padding=(1, 1))(inception_3b_output)
pool3_helper = PoolHelper()(inception_3b_output_zero_pad)
pool3_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='semantic_pool3/3x3_s2')(pool3_helper)
inception_4a_1x1 = Convolution2D(192,1,1,border_mode='same',activation='relu',name='semantic_inception_4a/1x1',W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3_reduce = Convolution2D(96,1,1,border_mode='same',activation='relu',name='semantic_inception_4a/3x3_reduce',W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3 = Convolution2D(208,3,3,border_mode='same',activation='relu',name='semantic_inception_4a/3x3',W_regularizer=l2(0.0002))(inception_4a_3x3_reduce)
inception_4a_5x5_reduce = Convolution2D(16,1,1,border_mode='same',activation='relu',name='semantic_inception_4a/5x5_reduce',W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_5x5 = Convolution2D(48,5,5,border_mode='same',activation='relu',name='semantic_inception_4a/5x5',W_regularizer=l2(0.0002))(inception_4a_5x5_reduce)
inception_4a_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='semantic_inception_4a/pool')(pool3_3x3_s2)
inception_4a_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='semantic_inception_4a/pool_proj',W_regularizer=l2(0.0002))(inception_4a_pool)
inception_4a_output = merge([inception_4a_1x1,inception_4a_3x3,inception_4a_5x5,inception_4a_pool_proj],mode='concat',concat_axis=1,name='semantic_inception_4a/output')
inception_4b_1x1 = Convolution2D(160,1,1,border_mode='same',activation='relu',name='semantic_inception_4b/1x1',W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3_reduce = Convolution2D(112,1,1,border_mode='same',activation='relu',name='semantic_inception_4b/3x3_reduce',W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3 = Convolution2D(224,3,3,border_mode='same',activation='relu',name='semantic_inception_4b/3x3',W_regularizer=l2(0.0002))(inception_4b_3x3_reduce)
inception_4b_5x5_reduce = Convolution2D(24,1,1,border_mode='same',activation='relu',name='semantic_inception_4b/5x5_reduce',W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='semantic_inception_4b/5x5',W_regularizer=l2(0.0002))(inception_4b_5x5_reduce)
inception_4b_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='semantic_inception_4b/pool')(inception_4a_output)
inception_4b_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='semantic_inception_4b/pool_proj',W_regularizer=l2(0.0002))(inception_4b_pool)
inception_4b_output = merge([inception_4b_1x1,inception_4b_3x3,inception_4b_5x5,inception_4b_pool_proj],mode='concat',concat_axis=1,name='semantic_inception_4b_output')
inception_4c_1x1 = Convolution2D(128,1,1,border_mode='same',activation='relu',name='semantic_inception_4c/1x1',W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3_reduce = Convolution2D(128,1,1,border_mode='same',activation='relu',name='semantic_inception_4c/3x3_reduce',W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3 = Convolution2D(256,3,3,border_mode='same',activation='relu',name='semantic_inception_4c/3x3',W_regularizer=l2(0.0002))(inception_4c_3x3_reduce)
inception_4c_5x5_reduce = Convolution2D(24,1,1,border_mode='same',activation='relu',name='semantic_inception_4c/5x5_reduce',W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='semantic_inception_4c/5x5',W_regularizer=l2(0.0002))(inception_4c_5x5_reduce)
inception_4c_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='semantic_inception_4c/pool')(inception_4b_output)
inception_4c_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='semantic_inception_4c/pool_proj',W_regularizer=l2(0.0002))(inception_4c_pool)
inception_4c_output = merge([inception_4c_1x1,inception_4c_3x3,inception_4c_5x5,inception_4c_pool_proj],mode='concat',concat_axis=1,name='semantic_inception_4c/output')
inception_4d_1x1 = Convolution2D(112,1,1,border_mode='same',activation='relu',name='semantic_inception_4d/1x1',W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3_reduce = Convolution2D(144,1,1,border_mode='same',activation='relu',name='semantic_inception_4d/3x3_reduce',W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3 = Convolution2D(288,3,3,border_mode='same',activation='relu',name='semantic_inception_4d/3x3',W_regularizer=l2(0.0002))(inception_4d_3x3_reduce)
inception_4d_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='semantic_inception_4d/5x5_reduce',W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='semantic_inception_4d/5x5',W_regularizer=l2(0.0002))(inception_4d_5x5_reduce)
inception_4d_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='semantic_inception_4d/pool')(inception_4c_output)
inception_4d_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='semantic_inception_4d/pool_proj',W_regularizer=l2(0.0002))(inception_4d_pool)
inception_4d_output = merge([inception_4d_1x1,inception_4d_3x3,inception_4d_5x5,inception_4d_pool_proj],mode='concat',concat_axis=1,name='semantic_inception_4d/output')
inception_4e_1x1_semantics = Convolution2D(256,1,1,border_mode='same',activation='relu',name='inception_4e/1x1_semantics',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_reduce_semantics = Convolution2D(160,1,1,border_mode='same',activation='relu',name='inception_4e/3x3_reduce_semantics',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_semantics = Convolution2D(320,3,3,border_mode='same',activation='relu',name='inception_4e/3x3_semantics',W_regularizer=l2(0.0002))(inception_4e_3x3_reduce_semantics)
inception_4e_5x5_reduce_semantics = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_4e/5x5_reduce_semantics',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_5x5_semantics = Convolution2D(128,5,5,border_mode='same',activation='relu',name='inception_4e/5x5_semantics',W_regularizer=l2(0.0002))(inception_4e_5x5_reduce_semantics)
inception_4e_pool_semantics = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4e/pool_semantics')(inception_4d_output)
inception_4e_pool_proj_semantics = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_4e/pool_proj_semantics',W_regularizer=l2(0.0002))(inception_4e_pool_semantics)
inception_4e_output_semantics = merge([inception_4e_1x1_semantics,inception_4e_3x3_semantics,inception_4e_5x5_semantics,inception_4e_pool_proj_semantics],mode='concat',concat_axis=1,name='inception_4e/output_semantics')
conv_output_semantics = Convolution2D(1024, 3, 3, activation='relu',name='conv_6_1_semantics',border_mode = 'same',W_regularizer=l2(0.0002))(inception_4e_output_semantics)
if not rapid_style:
x_semantics = GlobalAveragePooling2D()(conv_output_semantics)
if use_comments:
x_semantics = merge([x_semantics, x_text_semantics],mode='concat',concat_axis=1)
output_semantics = Dense(65, activation = 'softmax', name="output_semantics")(x_semantics)
else:
inception_4e_1x1 = Convolution2D(256,1,1,border_mode='same',activation='relu',name='inception_4e/1x1',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_reduce = Convolution2D(160,1,1,border_mode='same',activation='relu',name='inception_4e/3x3_reduce',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3 = Convolution2D(320,3,3,border_mode='same',activation='relu',name='inception_4e/3x3',W_regularizer=l2(0.0002))(inception_4e_3x3_reduce)
inception_4e_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_4e/5x5_reduce',W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_5x5 = Convolution2D(128,5,5,border_mode='same',activation='relu',name='inception_4e/5x5',W_regularizer=l2(0.0002))(inception_4e_5x5_reduce)
inception_4e_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4e/pool')(inception_4d_output)
inception_4e_pool_proj = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_4e/pool_proj',W_regularizer=l2(0.0002))(inception_4e_pool)
inception_4e_output = merge([inception_4e_1x1,inception_4e_3x3,inception_4e_5x5,inception_4e_pool_proj],mode='concat',concat_axis=1,name='inception_4e/output')
conv_output_aesthetics = Convolution2D(1024, 3, 3, activation='relu',name='conv_6_1',border_mode = 'same',W_regularizer=l2(0.0002))(inception_4e_output)
if rapid_style:
merged_conv = merge([conv_output_aesthetics, conv_output_semantics], mode='concat', concat_axis=1)
x_aesthetics = GlobalAveragePooling2D()(merged_conv)
else:
x_aesthetics = GlobalAveragePooling2D()(conv_output_aesthetics)
if use_multigap:
if use_comments:
x_aesthetics = merge([x_aesthetics, x_text_aesthetics, inception_4a_gap, inception_4b_gap, inception_4c_gap, inception_4d_gap],mode='concat',concat_axis=1)
else:
x_aesthetics = merge([x_aesthetics, inception_4a_gap, inception_4b_gap, inception_4c_gap, inception_4d_gap],mode='concat',concat_axis=1)
else:
if use_comments:
x_aesthetics = merge([x_aesthetics, x_text_aesthetics],mode='concat',concat_axis=1)
if use_distribution:
if use_multigap:
output_aesthetics = Dense(10, activation = 'softmax', name="main_output__")(x_aesthetics)
else:
output_aesthetics = Dense(10, activation = 'softmax', name="main_output_")(x_aesthetics)
else:
if use_multigap:
output_aesthetics = Dense(2, activation = 'softmax', name="main_output_")(x_aesthetics)
else:
output_aesthetics = Dense(2, activation = 'softmax', name="main_output")(x_aesthetics)
if use_semantics and not rapid_style:
if use_comments:
googlenet = Model(input=[input_image, comment_input], output=[output_aesthetics,output_semantics])
else:
googlenet = Model(input=input_image, output=[output_aesthetics,output_semantics])
else:
if use_comments:
googlenet = Model(input=[input_image, comment_input], output=output_aesthetics)
else:
googlenet = Model(input=input_image, output=output_aesthetics)
if weights_path:
if use_semantics:
googlenet.load_weights('weights/named_googlenet_semantics_weights.h5', by_name=True)
googlenet.load_weights(weights_path,by_name=load_weights_by_name)
if rapid_style:
for i, layer in enumerate(googlenet.layers):
if 'semantic' in layer.name:
# print("{} - {}".format(i, layer.name))
layer.trainable = False
return googlenet
def create_achu_net(weights_path=None, img_chs = 3, img_rows=200, img_cols=200 , nb_output_classes = 7, drop_out_rate = 0.4, n_hidden = 256, n_samples = 100, n_timesteps = 16):
# creates alinet based LSTM
input_shape = (n_timesteps, img_chs, img_rows, img_cols)
input = Input(shape=input_shape)
K.set_image_dim_ordering('th')
conv1_7x7_s2 = TimeDistributed(Convolution2D(64,7,7,subsample=(2,2),border_mode='same',activation='relu',name='conv1/7x7_s2',W_regularizer=l2(0.0002)))(input)
conv1_zero_pad = TimeDistributed(ZeroPadding2D(padding=(1, 1)))(conv1_7x7_s2)
pool1_helper = TimeDistributed(PoolHelper())(conv1_zero_pad)
pool1_3x3_s2 = TimeDistributed(MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool1/3x3_s2'))(pool1_helper)
pool1_norm1 = TimeDistributed(LRN(name='pool1/norm1'))(pool1_3x3_s2)
conv2_3x3_reduce = TimeDistributed(Convolution2D(64,1,1,border_mode='same',activation='relu',name='conv2/3x3_reduce',W_regularizer=l2(0.0002)))(pool1_norm1)
conv2_3x3 = TimeDistributed(Convolution2D(192,3,3,border_mode='same',activation='relu',name='conv2/3x3',W_regularizer=l2(0.0002)))(conv2_3x3_reduce)
conv2_norm2 = TimeDistributed(LRN(name='conv2/norm2'))(conv2_3x3)
conv2_zero_pad = TimeDistributed(ZeroPadding2D(padding=(1, 1)))(conv2_norm2)
pool2_helper = TimeDistributed(PoolHelper())(conv2_zero_pad)
pool2_3x3_s2 = TimeDistributed(MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool2/3x3_s2'))(pool2_helper)
inception_3a_1x1 = TimeDistributed(Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_3a/1x1',W_regularizer=l2(0.0002)))(pool2_3x3_s2)
inception_3a_3x3_reduce = TimeDistributed(Convolution2D(96,1,1,border_mode='same',activation='relu',name='inception_3a/3x3_reduce',W_regularizer=l2(0.0002)))(pool2_3x3_s2)
inception_3a_3x3 = TimeDistributed(Convolution2D(128,3,3,border_mode='same',activation='relu',name='inception_3a/3x3',W_regularizer=l2(0.0002)))(inception_3a_3x3_reduce)
inception_3a_5x5_reduce = TimeDistributed(Convolution2D(16,1,1,border_mode='same',activation='relu',name='inception_3a/5x5_reduce',W_regularizer=l2(0.0002)))(pool2_3x3_s2)
inception_3a_5x5 = TimeDistributed(Convolution2D(32,5,5,border_mode='same',activation='relu',name='inception_3a/5x5',W_regularizer=l2(0.0002)))(inception_3a_5x5_reduce)
inception_3a_pool = TimeDistributed(MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_3a/pool'))(pool2_3x3_s2)
inception_3a_pool_proj = TimeDistributed(Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_3a/pool_proj',W_regularizer=l2(0.0002)))(inception_3a_pool)
inception_3a_output = merge([inception_3a_1x1,inception_3a_3x3,inception_3a_5x5,inception_3a_pool_proj],mode='concat',concat_axis=2,name='inception_3a/output')
inception_3b_1x1 = TimeDistributed(Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_3b/1x1',W_regularizer=l2(0.0002)))(inception_3a_output)
inception_3b_3x3_reduce = TimeDistributed(Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_3b/3x3_reduce',W_regularizer=l2(0.0002)))(inception_3a_output)
inception_3b_3x3 = TimeDistributed(Convolution2D(192,3,3,border_mode='same',activation='relu',name='inception_3b/3x3',W_regularizer=l2(0.0002)))(inception_3b_3x3_reduce)
inception_3b_5x5_reduce = TimeDistributed(Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_3b/5x5_reduce',W_regularizer=l2(0.0002)))(inception_3a_output)
inception_3b_5x5 = TimeDistributed(Convolution2D(96,5,5,border_mode='same',activation='relu',name='inception_3b/5x5',W_regularizer=l2(0.0002)))(inception_3b_5x5_reduce)
inception_3b_pool = TimeDistributed(MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_3b/pool'))(inception_3a_output)
inception_3b_pool_proj = TimeDistributed(Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_3b/pool_proj',W_regularizer=l2(0.0002)))(inception_3b_pool)
inception_3b_output = merge([inception_3b_1x1,inception_3b_3x3,inception_3b_5x5,inception_3b_pool_proj],mode='concat',concat_axis=2,name='inception_3b/output')
inception_3b_output_zero_pad = TimeDistributed(ZeroPadding2D(padding=(1, 1)))(inception_3b_output)
pool3_helper = TimeDistributed(PoolHelper())(inception_3b_output_zero_pad)
pool3_3x3_s2 = TimeDistributed(MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool3/3x3_s2'))(pool3_helper)
inception_4a_1x1 = TimeDistributed(Convolution2D(192,1,1,border_mode='same',activation='relu',name='inception_4a/1x1',W_regularizer=l2(0.0002)))(pool3_3x3_s2)
inception_4a_3x3_reduce = TimeDistributed(Convolution2D(96,1,1,border_mode='same',activation='relu',name='inception_4a/3x3_reduce',W_regularizer=l2(0.0002)))(pool3_3x3_s2)
inception_4a_3x3 = TimeDistributed(Convolution2D(208,3,3,border_mode='same',activation='relu',name='inception_4a/3x3',W_regularizer=l2(0.0002)))(inception_4a_3x3_reduce)
inception_4a_5x5_reduce = TimeDistributed(Convolution2D(16,1,1,border_mode='same',activation='relu',name='inception_4a/5x5_reduce',W_regularizer=l2(0.0002)))(pool3_3x3_s2)
inception_4a_5x5 = TimeDistributed(Convolution2D(48,5,5,border_mode='same',activation='relu',name='inception_4a/5x5',W_regularizer=l2(0.0002)))(inception_4a_5x5_reduce)
inception_4a_pool = TimeDistributed(MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4a/pool'))(pool3_3x3_s2)
inception_4a_pool_proj = TimeDistributed(Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_4a/pool_proj',W_regularizer=l2(0.0002)))(inception_4a_pool)
inception_4a_output = merge([inception_4a_1x1,inception_4a_3x3,inception_4a_5x5,inception_4a_pool_proj],mode='concat',concat_axis=2,name='inception_4a/output')
loss1_max_pool = TimeDistributed(MaxPooling2D(pool_size=(3,3),strides=(2,2),name='loss1/max_pool'))(inception_4a_output)
loss1_conv = TimeDistributed(Convolution2D(128,1,1,border_mode='same',activation='relu',name='loss1/conv',W_regularizer=l2(0.0002)))(loss1_max_pool)
loss1_flat = TimeDistributed(Flatten())(loss1_conv)
#LSTMs
lstm_bd_1 = Bidirectional(LSTM(n_hidden, return_sequences=True, name='lstm1'))(loss1_flat)
lstm_drop_1 = Dropout(drop_out_rate)(lstm_bd_1)
lstm_bd_2 = Bidirectional(LSTM(n_hidden, return_sequences=True, name='lstm2'))(lstm_drop_1)
lstm_drop_2 = Dropout(drop_out_rate)(lstm_bd_2)
lstm_bd_3 = Bidirectional(LSTM(n_hidden, return_sequences=True, name='lstm3'))(lstm_drop_2)
lstm_drop_3 = Dropout(drop_out_rate)(lstm_bd_3)
lstm_bd_4 = Bidirectional(LSTM(n_hidden, return_sequences=True, name='lstm4'))(lstm_drop_3)
lstm_drop_4 = Dropout(drop_out_rate)(lstm_bd_4)
#lstm_max_pool = TimeDistributed(MaxPooling2D(pool_size=(3,3),strides=(2,2),name='lstm/max_pool'))(lstm_drop_4)
loss1_fc_2 = TimeDistributed(Dense(1024,activation='relu',name='loss1/fc_2',W_regularizer=l2(0.0002)))(lstm_drop_4)
loss1_drop_fc_2 = Dropout(drop_out_rate)(loss1_fc_2)
loss1_classifier = Dense(nb_output_classes,name='loss1/classifier',W_regularizer=l2(0.0002))(loss1_drop_fc_2)
loss1_classifier_act = Activation('softmax')(loss1_classifier)
achu_net = Model(input=input, output=[loss1_classifier_act])
if weights_path:
achu_net.load_weights(weights_path)
return achu_net
def create_posenet(weights_path=None):
# creates PoseNet base on the GoogLeNet a.k.a. Inception v1 (Szegedy, 2015)
# Set the input dimension (the same as GoogLeNet)
input = Input(shape=(3, 224, 224))
# Convolutional layer
conv1_7x7_s2 = Convolution2D(64,7,7,subsample=(2,2),border_mode='same',activation='relu',name='conv1/7x7_s2',
W_regularizer=l2(0.0002))(input)
# Zero padding
conv1_zero_pad = ZeroPadding2D(padding=(1, 1))(conv1_7x7_s2)
pool1_helper = PoolHelper()(conv1_zero_pad)
# Max Pooling layer
pool1_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool1/3x3_s2')(pool1_helper)
# Normalizatio after max pooling
pool1_norm1 = LRN(name='pool1/norm1')(pool1_3x3_s2)
# Reduction layer
conv2_3x3_reduce = Convolution2D(64,1,1,border_mode='same',activation='relu',name='conv2/3x3_reduce',
W_regularizer=l2(0.0002))(pool1_norm1)
# Convolutional layer
conv2_3x3 = Convolution2D(192,3,3,border_mode='same',activation='relu',name='conv2/3x3',
W_regularizer=l2(0.0002))(conv2_3x3_reduce)
# Normalizaion
conv2_norm2 = LRN(name='conv2/norm2')(conv2_3x3)
# Zero padding
conv2_zero_pad = ZeroPadding2D(padding=(1, 1))(conv2_norm2)
pool2_helper = PoolHelper()(conv2_zero_pad)
# Max Pooling layer
pool2_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool2/3x3_s2')(pool2_helper)
## First inception module
inception_3a_1x1 = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_3a/1x1',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3_reduce = Convolution2D(96,1,1,border_mode='same',activation='relu',name='inception_3a/3x3_reduce',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3 = Convolution2D(128,3,3,border_mode='same',activation='relu',name='inception_3a/3x3',
W_regularizer=l2(0.0002))(inception_3a_3x3_reduce)
inception_3a_5x5_reduce = Convolution2D(16,1,1,border_mode='same',activation='relu',name='inception_3a/5x5_reduce',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_5x5 = Convolution2D(32,5,5,border_mode='same',activation='relu',name='inception_3a/5x5',
W_regularizer=l2(0.0002))(inception_3a_5x5_reduce)
inception_3a_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',
name='inception_3a/pool')(pool2_3x3_s2)
inception_3a_pool_proj = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_3a/pool_proj',
W_regularizer=l2(0.0002))(inception_3a_pool)
inception_3a_output = merge([inception_3a_1x1,inception_3a_3x3,inception_3a_5x5,inception_3a_pool_proj],
mode='concat',concat_axis=1,name='inception_3a/output')
## Second inception module
# 1x1 convolution layer
inception_3b_1x1 = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_3b/1x1',
W_regularizer=l2(0.0002))(inception_3a_output)
# 3x3 reduction layerc and its onvulutional layer
inception_3b_3x3_reduce = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_3b/3x3_reduce',
W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3 = Convolution2D(192,3,3,border_mode='same',activation='relu',name='inception_3b/3x3',
W_regularizer=l2(0.0002))(inception_3b_3x3_reduce)
# 5x5 reduction layerc and its onvulutional layer
inception_3b_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_3b/5x5_reduce',
W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_5x5 = Convolution2D(96,5,5,border_mode='same',activation='relu',name='inception_3b/5x5',
W_regularizer=l2(0.0002))(inception_3b_5x5_reduce)
# Max Pooling
inception_3b_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_3b/pool')(inception_3a_output)
inception_3b_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_3b/pool_proj',
W_regularizer=l2(0.0002))(inception_3b_pool)
inception_3b_output = merge([inception_3b_1x1,inception_3b_3x3,inception_3b_5x5,inception_3b_pool_proj],
mode='concat',concat_axis=1,name='inception_3b/output')
# Zero padding
inception_3b_output_zero_pad = ZeroPadding2D(padding=(1, 1))(inception_3b_output)
pool3_helper = PoolHelper()(inception_3b_output_zero_pad)
# Max Pooling
pool3_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool3/3x3_s2')(pool3_helper)
## Third inception module
inception_4a_1x1 = Convolution2D(192,1,1,border_mode='same',activation='relu',name='inception_4a/1x1',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3_reduce = Convolution2D(96,1,1,border_mode='same',activation='relu',name='inception_4a/3x3_reduce',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3 = Convolution2D(208,3,3,border_mode='same',activation='relu',name='inception_4a/3x3',
W_regularizer=l2(0.0002))(inception_4a_3x3_reduce)
inception_4a_5x5_reduce = Convolution2D(16,1,1,border_mode='same',activation='relu',name='inception_4a/5x5_reduce',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_5x5 = Convolution2D(48,5,5,border_mode='same',activation='relu',name='inception_4a/5x5',
W_regularizer=l2(0.0002))(inception_4a_5x5_reduce)
inception_4a_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4a/pool')(pool3_3x3_s2)
inception_4a_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_4a/pool_proj',
W_regularizer=l2(0.0002))(inception_4a_pool)
inception_4a_output = merge([inception_4a_1x1,inception_4a_3x3,inception_4a_5x5,inception_4a_pool_proj],
mode='concat',concat_axis=1,name='inception_4a/output')
## First classification branch
# Average Pooling
loss1_ave_pool = AveragePooling2D(pool_size=(5,5),strides=(3,3),name='loss1/ave_pool')(inception_4a_output)
# Convolutional Layer for loss
loss1_conv = Convolution2D(128,1,1,border_mode='same',activation='relu',name='loss1/conv',
W_regularizer=l2(0.0002))(loss1_ave_pool)
# Flatten for the next full connection layer
loss1_flat = Flatten()(loss1_conv)
# Full connection layer
loss1_fc = Dense(1024,activation='relu',name='loss1/fc',W_regularizer=l2(0.0002))(loss1_flat)
# Dropout
loss1_drop_fc = Dropout(0.7)(loss1_fc)
# 3-D Position Regression
cls1_fc_pose_xyz = Dense(3, name='posexyz1',W_regularizer=l2(0.0002))(loss1_drop_fc) # New and different from the original GoogLeNet. The classification is replaced by regression.
# 4-D Orientation Regression
cls1_fc_pose_wpqr = Dense(4,name='posewpqr1',W_regularizer=l2(0.0002))(loss1_drop_fc) # New
## Fourth inception module
inception_4b_1x1 = Convolution2D(160,1,1,border_mode='same',activation='relu',name='inception_4b/1x1',
W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3_reduce = Convolution2D(112,1,1,border_mode='same',activation='relu',name='inception_4b/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3 = Convolution2D(224,3,3,border_mode='same',activation='relu',name='inception_4b/3x3',
W_regularizer=l2(0.0002))(inception_4b_3x3_reduce)
inception_4b_5x5_reduce = Convolution2D(24,1,1,border_mode='same',activation='relu',name='inception_4b/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='inception_4b/5x5',
W_regularizer=l2(0.0002))(inception_4b_5x5_reduce)
inception_4b_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4b/pool')(inception_4a_output)
inception_4b_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_4b/pool_proj',
W_regularizer=l2(0.0002))(inception_4b_pool)
inception_4b_output = merge([inception_4b_1x1,inception_4b_3x3,inception_4b_5x5,inception_4b_pool_proj],
mode='concat',concat_axis=1,name='inception_4b_output')
## Fifth inception module
inception_4c_1x1 = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_4c/1x1',
W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3_reduce = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_4c/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3 = Convolution2D(256,3,3,border_mode='same',activation='relu',name='inception_4c/3x3',
W_regularizer=l2(0.0002))(inception_4c_3x3_reduce)
inception_4c_5x5_reduce = Convolution2D(24,1,1,border_mode='same',activation='relu',name='inception_4c/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='inception_4c/5x5',
W_regularizer=l2(0.0002))(inception_4c_5x5_reduce)
inception_4c_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4c/pool')(inception_4b_output)
inception_4c_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_4c/pool_proj',
W_regularizer=l2(0.0002))(inception_4c_pool)
inception_4c_output = merge([inception_4c_1x1,inception_4c_3x3,inception_4c_5x5,inception_4c_pool_proj],
mode='concat',concat_axis=1,name='inception_4c/output')
## Sixth inception module
inception_4d_1x1 = Convolution2D(112,1,1,border_mode='same',activation='relu',name='inception_4d/1x1',
W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3_reduce = Convolution2D(144,1,1,border_mode='same',activation='relu',name='inception_4d/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3 = Convolution2D(288,3,3,border_mode='same',activation='relu',name='inception_4d/3x3',
W_regularizer=l2(0.0002))(inception_4d_3x3_reduce)
inception_4d_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_4d/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='inception_4d/5x5',
W_regularizer=l2(0.0002))(inception_4d_5x5_reduce)
inception_4d_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4d/pool')(inception_4c_output)
inception_4d_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_4d/pool_proj',
W_regularizer=l2(0.0002))(inception_4d_pool)
inception_4d_output = merge([inception_4d_1x1,inception_4d_3x3,inception_4d_5x5,inception_4d_pool_proj],
mode='concat',concat_axis=1,name='inception_4d/output')
## Second classification branch
# Average Pooling
loss2_ave_pool = AveragePooling2D(pool_size=(5,5),strides=(3,3),name='loss2/ave_pool')(inception_4d_output)
# Convolutional Layer for loss
loss2_conv = Convolution2D(128,1,1,border_mode='same',activation='relu',name='loss2/conv',
W_regularizer=l2(0.0002))(loss2_ave_pool)
# Flatten for the next full connection layer
loss2_flat = Flatten()(loss2_conv)
# Fully connected layer
loss2_fc = Dense(1024,activation='relu',name='loss2/fc',W_regularizer=l2(0.0002))(loss2_flat)
# Dropout layer
loss2_drop_fc = Dropout(0.7)(loss2_fc)
# 3-D position regression
cls2_fc_pose_xyz = Dense(3, name='posexyz2',W_regularizer=l2(0.0002))(loss2_drop_fc) # New
# 4-D orientation regression
cls2_fc_pose_wpqr = Dense(4,name='posewpqr2',W_regularizer=l2(0.0002))(loss2_drop_fc) # New
## Seventh inceptio module
inception_4e_1x1 = Convolution2D(256,1,1,border_mode='same',activation='relu',name='inception_4e/1x1',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_reduce = Convolution2D(160,1,1,border_mode='same',activation='relu',name='inception_4e/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3 = Convolution2D(320,3,3,border_mode='same',activation='relu',name='inception_4e/3x3',
W_regularizer=l2(0.0002))(inception_4e_3x3_reduce)
inception_4e_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_4e/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_5x5 = Convolution2D(128,5,5,border_mode='same',activation='relu',name='inception_4e/5x5',
W_regularizer=l2(0.0002))(inception_4e_5x5_reduce)
inception_4e_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4e/pool')(inception_4d_output)
inception_4e_pool_proj = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_4e/pool_proj',
W_regularizer=l2(0.0002))(inception_4e_pool)
inception_4e_output = merge([inception_4e_1x1,inception_4e_3x3,inception_4e_5x5,inception_4e_pool_proj],
mode='concat',concat_axis=1,name='inception_4e/output')
# Zero padding
inception_4e_output_zero_pad = ZeroPadding2D(padding=(1, 1))(inception_4e_output)
pool4_helper = PoolHelper()(inception_4e_output_zero_pad)
# Max pooling
pool4_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool4/3x3_s2')(pool4_helper)
## Eighth inception module
inception_5a_1x1 = Convolution2D(256,1,1,border_mode='same',activation='relu',name='inception_5a/1x1',
W_regularizer=l2(0.0002))(pool4_3x3_s2)
inception_5a_3x3_reduce = Convolution2D(160,1,1,border_mode='same',activation='relu',name='inception_5a/3x3_reduce',
W_regularizer=l2(0.0002))(pool4_3x3_s2)
inception_5a_3x3 = Convolution2D(320,3,3,border_mode='same',activation='relu',name='inception_5a/3x3',
W_regularizer=l2(0.0002))(inception_5a_3x3_reduce)
inception_5a_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_5a/5x5_reduce',
W_regularizer=l2(0.0002))(pool4_3x3_s2)
inception_5a_5x5 = Convolution2D(128,5,5,border_mode='same',activation='relu',name='inception_5a/5x5',
W_regularizer=l2(0.0002))(inception_5a_5x5_reduce)
inception_5a_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_5a/pool')(pool4_3x3_s2)
inception_5a_pool_proj = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_5a/pool_proj',
W_regularizer=l2(0.0002))(inception_5a_pool)
inception_5a_output = merge([inception_5a_1x1,inception_5a_3x3,inception_5a_5x5,inception_5a_pool_proj],
mode='concat',concat_axis=1,name='inception_5a/output')
## Ninth inception module
inception_5b_1x1 = Convolution2D(384,1,1,border_mode='same',activation='relu',name='inception_5b/1x1',
W_regularizer=l2(0.0002))(inception_5a_output)
inception_5b_3x3_reduce = Convolution2D(192,1,1,border_mode='same',activation='relu',name='inception_5b/3x3_reduce',
W_regularizer=l2(0.0002))(inception_5a_output)
inception_5b_3x3 = Convolution2D(384,3,3,border_mode='same',activation='relu',name='inception_5b/3x3',
W_regularizer=l2(0.0002))(inception_5b_3x3_reduce)
inception_5b_5x5_reduce = Convolution2D(48,1,1,border_mode='same',activation='relu',name='inception_5b/5x5_reduce',
W_regularizer=l2(0.0002))(inception_5a_output)
inception_5b_5x5 = Convolution2D(128,5,5,border_mode='same',activation='relu',name='inception_5b/5x5',
W_regularizer=l2(0.0002))(inception_5b_5x5_reduce)
inception_5b_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_5b/pool')(inception_5a_output)
inception_5b_pool_proj = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_5b/pool_proj',
W_regularizer=l2(0.0002))(inception_5b_pool)
inception_5b_output = merge([inception_5b_1x1,inception_5b_3x3,inception_5b_5x5,inception_5b_pool_proj],
mode='concat',concat_axis=1,name='inception_5b/output')
## the following layers only for bayesian_posenet
# Dropout layer(Bayesian)
pool5_drop_bay = Dropout(0.5)(inception_5b_output) #bay
# Average pooling
pool5_7x7_s1 = AveragePooling2D(pool_size=(7,7),strides=(1,1),name='pool5/7x7_s2')(pool5_drop_bay)
# Flatten
loss3_flat = Flatten()(pool5_7x7_s1)
# Fully connected layer
loss3_poseregressor = Dense(2048,,activation='relu', name='loss3/poseregressor',W_regularizer=l2(0.0002))(loss3_flat)
# Dropout
pool5_drop_7x7_s1 = Dropout(0.5)(loss3_poseregressor)
# 3-D position regression
cls3_fc_pose_xyz = Dense(3, name='posexyz3',W_regularizer=l2(0.0002))(pool5_drop_7x7_s1) # New
# 4-D orientation regression
cls3_fc_pose_wpqr = Dense(4,name='posewpqr3',W_regularizer=l2(0.0002))(pool5_drop_7x7_s1) # New
posenet = Model(input=input, output=[cls1_fc_pose_xyz,cls2_fc_pose_xyz,cls3_fc_pose_xyz,
cls1_fc_pose_wpqr,cls2_fc_pose_wpqr,cls3_fc_pose_wpqr])
if weights_path:
posenet.load_weights(weights_path)
#sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
#posenet.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])
return posenet
def googlenet_model(img_rows, img_cols, channel=1, num_classes=None):
"""
GoogLeNet a.k.a. Inception v1 for Keras
Model Schema is based on
https://gist.github.com/joelouismarino/a2ede9ab3928f999575423b9887abd14
ImageNet Pretrained Weights
https://drive.google.com/open?id=0B319laiAPjU3RE1maU9MMlh2dnc
Blog Post:
http://joelouismarino.github.io/blog_posts/blog_googlenet_keras.html
Parameters:
img_rows, img_cols - resolution of inputs
channel - 1 for grayscale, 3 for color
num_classes - number of class labels for our classification task
"""
input = Input(shape=(channel, img_rows, img_cols))
conv1_7x7_s2 = Convolution2D(64,
7,
7,
subsample=(2, 2),
border_mode='same',
activation='relu',
name='conv1/7x7_s2',
W_regularizer=l2(0.0002))(input)
conv1_zero_pad = ZeroPadding2D(padding=(1, 1))(conv1_7x7_s2)
pool1_helper = PoolHelper()(conv1_zero_pad)
pool1_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool1/3x3_s2')(pool1_helper)
pool1_norm1 = LRN(name='pool1/norm1')(pool1_3x3_s2)
# pool1_norm1 = BatchNormalization(name='pool1/norm1')(pool1_3x3_s2)
conv2_3x3_reduce = Convolution2D(64,
1,
1,
border_mode='same',
activation='relu',
name='conv2/3x3_reduce',
W_regularizer=l2(0.0002))(pool1_norm1)
conv2_3x3 = Convolution2D(192,
3,
3,
border_mode='same',
activation='relu',
name='conv2/3x3',
W_regularizer=l2(0.0002))(conv2_3x3_reduce)
conv2_norm2 = LRN(name='conv2/norm2')(conv2_3x3)
# conv2_norm2 = BatchNormalization(name='conv2/norm2')(conv2_3x3)
conv2_zero_pad = ZeroPadding2D(padding=(1, 1))(conv2_norm2)
pool2_helper = PoolHelper()(conv2_zero_pad)
pool2_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool2/3x3_s2')(pool2_helper)
inception_3a_1x1 = Convolution2D(64,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/1x1',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3_reduce = Convolution2D(
96,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/3x3_reduce',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3 = Convolution2D(
128,
3,
3,
border_mode='same',
activation='relu',
name='inception_3a/3x3',
W_regularizer=l2(0.0002))(inception_3a_3x3_reduce)
inception_3a_5x5_reduce = Convolution2D(
16,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/5x5_reduce',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_5x5 = Convolution2D(
32,
5,
5,
border_mode='same',
activation='relu',
name='inception_3a/5x5',
W_regularizer=l2(0.0002))(inception_3a_5x5_reduce)
inception_3a_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_3a/pool')(pool2_3x3_s2)
inception_3a_pool_proj = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/pool_proj',
W_regularizer=l2(0.0002))(inception_3a_pool)
inception_3a_output = merge([
inception_3a_1x1, inception_3a_3x3, inception_3a_5x5,
inception_3a_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_3a/output')
inception_3b_1x1 = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/1x1',
W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3_reduce = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/3x3_reduce',
W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3 = Convolution2D(
192,
3,
3,
border_mode='same',
activation='relu',
name='inception_3b/3x3',
W_regularizer=l2(0.0002))(inception_3b_3x3_reduce)
inception_3b_5x5_reduce = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/5x5_reduce',
W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_5x5 = Convolution2D(
96,
5,
5,
border_mode='same',
activation='relu',
name='inception_3b/5x5',
W_regularizer=l2(0.0002))(inception_3b_5x5_reduce)
inception_3b_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_3b/pool')(inception_3a_output)
inception_3b_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/pool_proj',
W_regularizer=l2(0.0002))(inception_3b_pool)
inception_3b_output = merge([
inception_3b_1x1, inception_3b_3x3, inception_3b_5x5,
inception_3b_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_3b/output')
inception_3b_output_zero_pad = ZeroPadding2D(
padding=(1, 1))(inception_3b_output)
pool3_helper = PoolHelper()(inception_3b_output_zero_pad)
pool3_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool3/3x3_s2')(pool3_helper)
inception_4a_1x1 = Convolution2D(192,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/1x1',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3_reduce = Convolution2D(
96,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/3x3_reduce',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3 = Convolution2D(
208,
3,
3,
border_mode='same',
activation='relu',
name='inception_4a/3x3',
W_regularizer=l2(0.0002))(inception_4a_3x3_reduce)
inception_4a_5x5_reduce = Convolution2D(
16,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/5x5_reduce',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_5x5 = Convolution2D(
48,
5,
5,
border_mode='same',
activation='relu',
name='inception_4a/5x5',
W_regularizer=l2(0.0002))(inception_4a_5x5_reduce)
inception_4a_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4a/pool')(pool3_3x3_s2)
inception_4a_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/pool_proj',
W_regularizer=l2(0.0002))(inception_4a_pool)
inception_4a_output = merge([
inception_4a_1x1, inception_4a_3x3, inception_4a_5x5,
inception_4a_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4a/output')
loss1_ave_pool = AveragePooling2D(
pool_size=(5, 5), strides=(3, 3),
name='loss1/ave_pool')(inception_4a_output)
loss1_conv = Convolution2D(128,
1,
1,
border_mode='same',
activation='relu',
name='loss1/conv',
W_regularizer=l2(0.0002))(loss1_ave_pool)
loss1_flat = Flatten()(loss1_conv)
loss1_fc = Dense(1024,
activation='relu',
name='loss1/fc',
W_regularizer=l2(0.0002))(loss1_flat)
loss1_drop_fc = Dropout(0.7)(loss1_fc)
loss1_classifier = Dense(1000,
name='loss1/classifier',
W_regularizer=l2(0.0002))(loss1_drop_fc)
loss1_classifier_act = Activation('softmax')(loss1_classifier)
inception_4b_1x1 = Convolution2D(
160,
1,
1,
border_mode='same',
activation='relu',
name='inception_4b/1x1',
W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3_reduce = Convolution2D(
112,
1,
1,
border_mode='same',
activation='relu',
name='inception_4b/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3 = Convolution2D(
224,
3,
3,
border_mode='same',
activation='relu',
name='inception_4b/3x3',
W_regularizer=l2(0.0002))(inception_4b_3x3_reduce)
inception_4b_5x5_reduce = Convolution2D(
24,
1,
1,
border_mode='same',
activation='relu',
name='inception_4b/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_5x5 = Convolution2D(
64,
5,
5,
border_mode='same',
activation='relu',
name='inception_4b/5x5',
W_regularizer=l2(0.0002))(inception_4b_5x5_reduce)
inception_4b_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4b/pool')(inception_4a_output)
inception_4b_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_4b/pool_proj',
W_regularizer=l2(0.0002))(inception_4b_pool)
inception_4b_output = merge([
inception_4b_1x1, inception_4b_3x3, inception_4b_5x5,
inception_4b_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4b_output')
inception_4c_1x1 = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_4c/1x1',
W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3_reduce = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_4c/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3 = Convolution2D(
256,
3,
3,
border_mode='same',
activation='relu',
name='inception_4c/3x3',
W_regularizer=l2(0.0002))(inception_4c_3x3_reduce)
inception_4c_5x5_reduce = Convolution2D(
24,
1,
1,
border_mode='same',
activation='relu',
name='inception_4c/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_5x5 = Convolution2D(
64,
5,
5,
border_mode='same',
activation='relu',
name='inception_4c/5x5',
W_regularizer=l2(0.0002))(inception_4c_5x5_reduce)
inception_4c_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4c/pool')(inception_4b_output)
inception_4c_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_4c/pool_proj',
W_regularizer=l2(0.0002))(inception_4c_pool)
inception_4c_output = merge([
inception_4c_1x1, inception_4c_3x3, inception_4c_5x5,
inception_4c_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4c/output')
inception_4d_1x1 = Convolution2D(
112,
1,
1,
border_mode='same',
activation='relu',
name='inception_4d/1x1',
W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3_reduce = Convolution2D(
144,
1,
1,
border_mode='same',
activation='relu',
name='inception_4d/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3 = Convolution2D(
288,
3,
3,
border_mode='same',
activation='relu',
name='inception_4d/3x3',
W_regularizer=l2(0.0002))(inception_4d_3x3_reduce)
inception_4d_5x5_reduce = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_4d/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_5x5 = Convolution2D(
64,
5,
5,
border_mode='same',
activation='relu',
name='inception_4d/5x5',
W_regularizer=l2(0.0002))(inception_4d_5x5_reduce)
inception_4d_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4d/pool')(inception_4c_output)
inception_4d_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_4d/pool_proj',
W_regularizer=l2(0.0002))(inception_4d_pool)
inception_4d_output = merge([
inception_4d_1x1, inception_4d_3x3, inception_4d_5x5,
inception_4d_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4d/output')
loss2_ave_pool = AveragePooling2D(
pool_size=(5, 5), strides=(3, 3),
name='loss2/ave_pool')(inception_4d_output)
loss2_conv = Convolution2D(128,
1,
1,
border_mode='same',
activation='relu',
name='loss2/conv',
W_regularizer=l2(0.0002))(loss2_ave_pool)
loss2_flat = Flatten()(loss2_conv)
loss2_fc = Dense(1024,
activation='relu',
name='loss2/fc',
W_regularizer=l2(0.0002))(loss2_flat)
loss2_drop_fc = Dropout(0.7)(loss2_fc)
loss2_classifier = Dense(1000,
name='loss2/classifier',
W_regularizer=l2(0.0002))(loss2_drop_fc)
loss2_classifier_act = Activation('softmax')(loss2_classifier)
inception_4e_1x1 = Convolution2D(
256,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/1x1',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_reduce = Convolution2D(
160,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3 = Convolution2D(
320,
3,
3,
border_mode='same',
activation='relu',
name='inception_4e/3x3',
W_regularizer=l2(0.0002))(inception_4e_3x3_reduce)
inception_4e_5x5_reduce = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_5x5 = Convolution2D(
128,
5,
5,
border_mode='same',
activation='relu',
name='inception_4e/5x5',
W_regularizer=l2(0.0002))(inception_4e_5x5_reduce)
inception_4e_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4e/pool')(inception_4d_output)
inception_4e_pool_proj = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/pool_proj',
W_regularizer=l2(0.0002))(inception_4e_pool)
inception_4e_output = merge([
inception_4e_1x1, inception_4e_3x3, inception_4e_5x5,
inception_4e_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4e/output')
inception_4e_output_zero_pad = ZeroPadding2D(
padding=(1, 1))(inception_4e_output)
pool4_helper = PoolHelper()(inception_4e_output_zero_pad)
pool4_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool4/3x3_s2')(pool4_helper)
inception_5a_1x1 = Convolution2D(256,
1,
1,
border_mode='same',
activation='relu',
name='inception_5a/1x1',
W_regularizer=l2(0.0002))(pool4_3x3_s2)
inception_5a_3x3_reduce = Convolution2D(
160,
1,
1,
border_mode='same',
activation='relu',
name='inception_5a/3x3_reduce',
W_regularizer=l2(0.0002))(pool4_3x3_s2)
inception_5a_3x3 = Convolution2D(
320,
3,
3,
border_mode='same',
activation='relu',
name='inception_5a/3x3',
W_regularizer=l2(0.0002))(inception_5a_3x3_reduce)
inception_5a_5x5_reduce = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_5a/5x5_reduce',
W_regularizer=l2(0.0002))(pool4_3x3_s2)
inception_5a_5x5 = Convolution2D(
128,
5,
5,
border_mode='same',
activation='relu',
name='inception_5a/5x5',
W_regularizer=l2(0.0002))(inception_5a_5x5_reduce)
inception_5a_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_5a/pool')(pool4_3x3_s2)
inception_5a_pool_proj = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_5a/pool_proj',
W_regularizer=l2(0.0002))(inception_5a_pool)
inception_5a_output = merge([
inception_5a_1x1, inception_5a_3x3, inception_5a_5x5,
inception_5a_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_5a/output')
inception_5b_1x1 = Convolution2D(
384,
1,
1,
border_mode='same',
activation='relu',
name='inception_5b/1x1',
W_regularizer=l2(0.0002))(inception_5a_output)
inception_5b_3x3_reduce = Convolution2D(
192,
1,
1,
border_mode='same',
activation='relu',
name='inception_5b/3x3_reduce',
W_regularizer=l2(0.0002))(inception_5a_output)
inception_5b_3x3 = Convolution2D(
384,
3,
3,
border_mode='same',
activation='relu',
name='inception_5b/3x3',
W_regularizer=l2(0.0002))(inception_5b_3x3_reduce)
inception_5b_5x5_reduce = Convolution2D(
48,
1,
1,
border_mode='same',
activation='relu',
name='inception_5b/5x5_reduce',
W_regularizer=l2(0.0002))(inception_5a_output)
inception_5b_5x5 = Convolution2D(
128,
5,
5,
border_mode='same',
activation='relu',
name='inception_5b/5x5',
W_regularizer=l2(0.0002))(inception_5b_5x5_reduce)
inception_5b_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_5b/pool')(inception_5a_output)
inception_5b_pool_proj = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_5b/pool_proj',
W_regularizer=l2(0.0002))(inception_5b_pool)
inception_5b_output = merge([
inception_5b_1x1, inception_5b_3x3, inception_5b_5x5,
inception_5b_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_5b/output')
pool5_7x7_s1 = AveragePooling2D(pool_size=(7, 7),
strides=(1, 1),
name='pool5/7x7_s2')(inception_5b_output)
loss3_flat = Flatten()(pool5_7x7_s1)
pool5_drop_7x7_s1 = Dropout(0.4)(loss3_flat)
loss3_classifier = Dense(1000,
name='loss3/classifier',
W_regularizer=l2(0.0002))(pool5_drop_7x7_s1)
loss3_classifier_act = Activation('softmax', name='prob')(loss3_classifier)
# Create model
model = Model(input=input,
output=[
loss1_classifier_act, loss2_classifier_act,
loss3_classifier_act
])
# Load ImageNet pre-trained data
model.load_weights('/home/cnd/.keras/models/googlenet_weights.h5')
# Truncate and replace softmax layer for transfer learning
# Cannot use model.layers.pop() since model is not of Sequential() type
# The method below works since pre-trained weights are stored in layers but not in the model
loss3_classifier_statefarm = Dense(
num_classes, name='loss3/classifier',
W_regularizer=l2(0.0002))(pool5_drop_7x7_s1)
loss3_classifier_act_statefarm = Activation(
'softmax', name='prob')(loss3_classifier_statefarm)
loss2_classifier_statefarm = Dense(num_classes,
name='loss2/classifier',
W_regularizer=l2(0.0002))(loss2_drop_fc)
loss2_classifier_act_statefarm = Activation('softmax')(
loss2_classifier_statefarm)
loss1_classifier_statefarm = Dense(num_classes,
name='loss1/classifier',
W_regularizer=l2(0.0002))(loss1_drop_fc)
loss1_classifier_act_statefarm = Activation('softmax')(
loss1_classifier_statefarm)
# Create another model with our customized softmax
model = Model(input=input,
output=[
loss1_classifier_act_statefarm,
loss2_classifier_act_statefarm,
loss3_classifier_act_statefarm
])
# Learning rate is changed to 0.001
sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def create_alinet(weights_path=None,
input_shape=(3, 48, 48),
nb_output_classes=7,
drop_out_rate=0.4):
# creates alinet Ali Mollahosseini 2015
input = Input(shape=input_shape)
conv1_7x7_s2 = Convolution2D(64,
7,
7,
subsample=(2, 2),
border_mode='same',
activation='relu',
name='conv1/7x7_s2',
W_regularizer=l2(0.0002))(input)
conv1_zero_pad = ZeroPadding2D(padding=(1, 1))(conv1_7x7_s2)
pool1_helper = PoolHelper()(conv1_zero_pad)
pool1_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool1/3x3_s2')(pool1_helper)
pool1_norm1 = LRN(name='pool1/norm1')(pool1_3x3_s2)
conv2_3x3_reduce = Convolution2D(64,
1,
1,
border_mode='same',
activation='relu',
name='conv2/3x3_reduce',
W_regularizer=l2(0.0002))(pool1_norm1)
conv2_3x3 = Convolution2D(192,
3,
3,
border_mode='same',
activation='relu',
name='conv2/3x3',
W_regularizer=l2(0.0002))(conv2_3x3_reduce)
conv2_norm2 = LRN(name='conv2/norm2')(conv2_3x3)
conv2_zero_pad = ZeroPadding2D(padding=(1, 1))(conv2_norm2)
pool2_helper = PoolHelper()(conv2_zero_pad)
pool2_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool2/3x3_s2')(pool2_helper)
inception_3a_1x1 = Convolution2D(64,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/1x1',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3_reduce = Convolution2D(
96,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/3x3_reduce',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3 = Convolution2D(
128,
3,
3,
border_mode='same',
activation='relu',
name='inception_3a/3x3',
W_regularizer=l2(0.0002))(inception_3a_3x3_reduce)
inception_3a_5x5_reduce = Convolution2D(
16,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/5x5_reduce',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_5x5 = Convolution2D(
32,
5,
5,
border_mode='same',
activation='relu',
name='inception_3a/5x5',
W_regularizer=l2(0.0002))(inception_3a_5x5_reduce)
inception_3a_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_3a/pool')(pool2_3x3_s2)
inception_3a_pool_proj = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/pool_proj',
W_regularizer=l2(0.0002))(inception_3a_pool)
inception_3a_output = merge([
inception_3a_1x1, inception_3a_3x3, inception_3a_5x5,
inception_3a_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_3a/output')
inception_3b_1x1 = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/1x1',
W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3_reduce = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/3x3_reduce',
W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3 = Convolution2D(
192,
3,
3,
border_mode='same',
activation='relu',
name='inception_3b/3x3',
W_regularizer=l2(0.0002))(inception_3b_3x3_reduce)
inception_3b_5x5_reduce = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/5x5_reduce',
W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_5x5 = Convolution2D(
96,
5,
5,
border_mode='same',
activation='relu',
name='inception_3b/5x5',
W_regularizer=l2(0.0002))(inception_3b_5x5_reduce)
inception_3b_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_3b/pool')(inception_3a_output)
inception_3b_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/pool_proj',
W_regularizer=l2(0.0002))(inception_3b_pool)
inception_3b_output = merge([
inception_3b_1x1, inception_3b_3x3, inception_3b_5x5,
inception_3b_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_3b/output')
inception_3b_output_zero_pad = ZeroPadding2D(
padding=(1, 1))(inception_3b_output)
pool3_helper = PoolHelper()(inception_3b_output_zero_pad)
pool3_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool3/3x3_s2')(pool3_helper)
inception_4a_1x1 = Convolution2D(192,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/1x1',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3_reduce = Convolution2D(
96,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/3x3_reduce',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3 = Convolution2D(
208,
3,
3,
border_mode='same',
activation='relu',
name='inception_4a/3x3',
W_regularizer=l2(0.0002))(inception_4a_3x3_reduce)
inception_4a_5x5_reduce = Convolution2D(
16,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/5x5_reduce',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_5x5 = Convolution2D(
48,
5,
5,
border_mode='same',
activation='relu',
name='inception_4a/5x5',
W_regularizer=l2(0.0002))(inception_4a_5x5_reduce)
inception_4a_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4a/pool')(pool3_3x3_s2)
inception_4a_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/pool_proj',
W_regularizer=l2(0.0002))(inception_4a_pool)
inception_4a_output = merge([
inception_4a_1x1, inception_4a_3x3, inception_4a_5x5,
inception_4a_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4a/output')
loss1_max_pool = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
name='loss1/max_pool')(inception_4a_output)
loss1_conv = Convolution2D(128,
1,
1,
border_mode='same',
activation='relu',
name='loss1/conv',
W_regularizer=l2(0.0002))(loss1_max_pool)
loss1_flat = Flatten()(loss1_conv)
loss1_fc_1 = Dense(4096,
activation='relu',
name='loss1/fc_1',
W_regularizer=l2(0.0002))(loss1_flat)
loss1_drop_fc_1 = Dropout(drop_out_rate)(loss1_fc_1)
loss1_fc_2 = Dense(1024,
activation='relu',
name='loss1/fc_2',
W_regularizer=l2(0.0002))(loss1_drop_fc_1)
loss1_drop_fc_2 = Dropout(drop_out_rate)(loss1_fc_2)
loss1_classifier = Dense(nb_output_classes,
name='loss1/classifier',
W_regularizer=l2(0.0002))(loss1_drop_fc_2)
loss1_classifier_act = Activation('softmax')(loss1_classifier)
alinet = Model(input=input, output=[loss1_classifier_act])
if weights_path:
alinet.load_weights(weights_path)
return alinet
def create_googlenet(weights_path=None, heatmap=False):
# creates GoogLeNet a.k.a. Inception v1 (Szegedy, 2015)
input = Input(shape=(3, 224, 224))
conv1_7x7_s2 = Convolution2D(64,
7,
7,
subsample=(2, 2),
border_mode='same',
activation='relu',
name='conv1/7x7_s2',
W_regularizer=l2(0.0002))(input)
conv1_zero_pad = ZeroPadding2D(padding=(1, 1))(conv1_7x7_s2)
pool1_helper = PoolHelper()(conv1_zero_pad)
pool1_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool1/3x3_s2')(pool1_helper)
pool1_norm1 = LRN(name='pool1/norm1')(pool1_3x3_s2)
conv2_3x3_reduce = Convolution2D(64,
1,
1,
border_mode='same',
activation='relu',
name='conv2/3x3_reduce',
W_regularizer=l2(0.0002))(pool1_norm1)
conv2_3x3 = Convolution2D(192,
3,
3,
border_mode='same',
activation='relu',
name='conv2/3x3',
W_regularizer=l2(0.0002))(conv2_3x3_reduce)
conv2_norm2 = LRN(name='conv2/norm2')(conv2_3x3)
conv2_zero_pad = ZeroPadding2D(padding=(1, 1))(conv2_norm2)
pool2_helper = PoolHelper()(conv2_zero_pad)
pool2_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool2/3x3_s2')(pool2_helper)
inception_3a_1x1 = Convolution2D(64,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/1x1',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3_reduce = Convolution2D(
96,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/3x3_reduce',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3 = Convolution2D(
128,
3,
3,
border_mode='same',
activation='relu',
name='inception_3a/3x3',
W_regularizer=l2(0.0002))(inception_3a_3x3_reduce)
inception_3a_5x5_reduce = Convolution2D(
16,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/5x5_reduce',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_5x5 = Convolution2D(
32,
5,
5,
border_mode='same',
activation='relu',
name='inception_3a/5x5',
W_regularizer=l2(0.0002))(inception_3a_5x5_reduce)
inception_3a_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_3a/pool')(pool2_3x3_s2)
inception_3a_pool_proj = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/pool_proj',
W_regularizer=l2(0.0002))(inception_3a_pool)
inception_3a_output = merge([
inception_3a_1x1, inception_3a_3x3, inception_3a_5x5,
inception_3a_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_3a/output')
inception_3b_1x1 = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/1x1',
W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3_reduce = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/3x3_reduce',
W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3 = Convolution2D(
192,
3,
3,
border_mode='same',
activation='relu',
name='inception_3b/3x3',
W_regularizer=l2(0.0002))(inception_3b_3x3_reduce)
inception_3b_5x5_reduce = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/5x5_reduce',
W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_5x5 = Convolution2D(
96,
5,
5,
border_mode='same',
activation='relu',
name='inception_3b/5x5',
W_regularizer=l2(0.0002))(inception_3b_5x5_reduce)
inception_3b_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_3b/pool')(inception_3a_output)
inception_3b_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/pool_proj',
W_regularizer=l2(0.0002))(inception_3b_pool)
inception_3b_output = merge([
inception_3b_1x1, inception_3b_3x3, inception_3b_5x5,
inception_3b_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_3b/output')
inception_3b_output_zero_pad = ZeroPadding2D(
padding=(1, 1))(inception_3b_output)
pool3_helper = PoolHelper()(inception_3b_output_zero_pad)
pool3_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool3/3x3_s2')(pool3_helper)
inception_4a_1x1 = Convolution2D(192,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/1x1',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3_reduce = Convolution2D(
96,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/3x3_reduce',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3 = Convolution2D(
208,
3,
3,
border_mode='same',
activation='relu',
name='inception_4a/3x3',
W_regularizer=l2(0.0002))(inception_4a_3x3_reduce)
inception_4a_5x5_reduce = Convolution2D(
16,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/5x5_reduce',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_5x5 = Convolution2D(
48,
5,
5,
border_mode='same',
activation='relu',
name='inception_4a/5x5',
W_regularizer=l2(0.0002))(inception_4a_5x5_reduce)
inception_4a_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4a/pool')(pool3_3x3_s2)
inception_4a_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/pool_proj',
W_regularizer=l2(0.0002))(inception_4a_pool)
inception_4a_output = merge([
inception_4a_1x1, inception_4a_3x3, inception_4a_5x5,
inception_4a_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4a/output')
inception_4b_1x1 = Convolution2D(
160,
1,
1,
border_mode='same',
activation='relu',
name='inception_4b/1x1',
W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3_reduce = Convolution2D(
112,
1,
1,
border_mode='same',
activation='relu',
name='inception_4b/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3 = Convolution2D(
224,
3,
3,
border_mode='same',
activation='relu',
name='inception_4b/3x3',
W_regularizer=l2(0.0002))(inception_4b_3x3_reduce)
inception_4b_5x5_reduce = Convolution2D(
24,
1,
1,
border_mode='same',
activation='relu',
name='inception_4b/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_5x5 = Convolution2D(
64,
5,
5,
border_mode='same',
activation='relu',
name='inception_4b/5x5',
W_regularizer=l2(0.0002))(inception_4b_5x5_reduce)
inception_4b_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4b/pool')(inception_4a_output)
inception_4b_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_4b/pool_proj',
W_regularizer=l2(0.0002))(inception_4b_pool)
inception_4b_output = merge([
inception_4b_1x1, inception_4b_3x3, inception_4b_5x5,
inception_4b_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4b_output')
inception_4c_1x1 = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_4c/1x1',
W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3_reduce = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_4c/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3 = Convolution2D(
256,
3,
3,
border_mode='same',
activation='relu',
name='inception_4c/3x3',
W_regularizer=l2(0.0002))(inception_4c_3x3_reduce)
inception_4c_5x5_reduce = Convolution2D(
24,
1,
1,
border_mode='same',
activation='relu',
name='inception_4c/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_5x5 = Convolution2D(
64,
5,
5,
border_mode='same',
activation='relu',
name='inception_4c/5x5',
W_regularizer=l2(0.0002))(inception_4c_5x5_reduce)
inception_4c_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4c/pool')(inception_4b_output)
inception_4c_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_4c/pool_proj',
W_regularizer=l2(0.0002))(inception_4c_pool)
inception_4c_output = merge([
inception_4c_1x1, inception_4c_3x3, inception_4c_5x5,
inception_4c_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4c/output')
inception_4d_1x1 = Convolution2D(
112,
1,
1,
border_mode='same',
activation='relu',
name='inception_4d/1x1',
W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3_reduce = Convolution2D(
144,
1,
1,
border_mode='same',
activation='relu',
name='inception_4d/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3 = Convolution2D(
288,
3,
3,
border_mode='same',
activation='relu',
name='inception_4d/3x3',
W_regularizer=l2(0.0002))(inception_4d_3x3_reduce)
inception_4d_5x5_reduce = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_4d/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_5x5 = Convolution2D(
64,
5,
5,
border_mode='same',
activation='relu',
name='inception_4d/5x5',
W_regularizer=l2(0.0002))(inception_4d_5x5_reduce)
inception_4d_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4d/pool')(inception_4c_output)
inception_4d_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_4d/pool_proj',
W_regularizer=l2(0.0002))(inception_4d_pool)
inception_4d_output = merge([
inception_4d_1x1, inception_4d_3x3, inception_4d_5x5,
inception_4d_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4d/output')
inception_4e_1x1_aesthetics = Convolution2D(
256,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/1x1_aesthetics',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_reduce_aesthetics = Convolution2D(
160,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/3x3_reduce_aesthetics',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_aesthetics = Convolution2D(
320,
3,
3,
border_mode='same',
activation='relu',
name='inception_4e/3x3_aesthetics',
W_regularizer=l2(0.0002))(inception_4e_3x3_reduce_aesthetics)
inception_4e_5x5_reduce_aesthetics = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/5x5_reduce_aesthetics',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_5x5_aesthetics = Convolution2D(
128,
5,
5,
border_mode='same',
activation='relu',
name='inception_4e/5x5_aesthetics',
W_regularizer=l2(0.0002))(inception_4e_5x5_reduce_aesthetics)
inception_4e_pool_aesthetics = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4e/pool_aesthetics')(inception_4d_output)
inception_4e_pool_proj_aesthetics = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/pool_proj_aesthetics',
W_regularizer=l2(0.0002))(inception_4e_pool_aesthetics)
inception_4e_output_aesthetics = merge(
[
inception_4e_1x1_aesthetics, inception_4e_3x3_aesthetics,
inception_4e_5x5_aesthetics, inception_4e_pool_proj_aesthetics
],
mode='concat',
concat_axis=1,
name='inception_4e/output_aesthetics')
conv_output_aesthetics = Convolution2D(
1024,
3,
3,
activation='relu',
name='conv_6_1_aesthetics',
border_mode='same',
W_regularizer=l2(0.0002))(inception_4e_output_aesthetics)
x_aesthetics = GlobalAveragePooling2D()(conv_output_aesthetics)
output_aesthetics = Dense(10,
activation='softmax',
name="output_aesthetics")(x_aesthetics)
inception_4e_1x1_semantics = Convolution2D(
256,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/1x1_semantics',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_reduce_semantics = Convolution2D(
160,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/3x3_reduce_semantics',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_semantics = Convolution2D(
320,
3,
3,
border_mode='same',
activation='relu',
name='inception_4e/3x3_semantics',
W_regularizer=l2(0.0002))(inception_4e_3x3_reduce_semantics)
inception_4e_5x5_reduce_semantics = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/5x5_reduce_semantics',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_5x5_semantics = Convolution2D(
128,
5,
5,
border_mode='same',
activation='relu',
name='inception_4e/5x5_semantics',
W_regularizer=l2(0.0002))(inception_4e_5x5_reduce_semantics)
inception_4e_pool_semantics = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4e/pool_semantics')(inception_4d_output)
inception_4e_pool_proj_semantics = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/pool_proj_semantics',
W_regularizer=l2(0.0002))(inception_4e_pool_semantics)
inception_4e_output_semantics = merge([
inception_4e_1x1_semantics, inception_4e_3x3_semantics,
inception_4e_5x5_semantics, inception_4e_pool_proj_semantics
],
mode='concat',
concat_axis=1,
name='inception_4e/output_semantics')
conv_output_semantics = Convolution2D(
1024,
3,
3,
activation='relu',
name='conv_6_1_semantics',
border_mode='same',
W_regularizer=l2(0.0002))(inception_4e_output_semantics)
x_semantics = GlobalAveragePooling2D()(conv_output_semantics)
output_semantics = Dense(65, activation='softmax',
name="output_semantics")(x_semantics)
if heatmap:
googlenet = Model(input=input,
output=[
output_aesthetics, output_semantics,
conv_output_aesthetics, conv_output_semantics
])
else:
googlenet = Model(input=input,
output=[output_aesthetics, output_semantics])
if weights_path:
googlenet.load_weights(weights_path, by_name=True)
return googlenet
def modified_googlenet(WIDTH, HEIGHT):
# creates GoogLeNet a.k.a. Inception v1 (Szegedy, 2015)
input = Input(shape=(3, WIDTH, HEIGHT))
conv1_7x7_s2 = Convolution2D(64,
7,
7,
subsample=(2, 2),
border_mode='same',
activation='relu',
name='conv1/7x7_s2',
W_regularizer=l2(0.0002))(input)
conv1_zero_pad = ZeroPadding2D(padding=(1, 1))(conv1_7x7_s2)
pool1_helper = PoolHelper()(conv1_zero_pad)
pool1_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool1/3x3_s2')(pool1_helper)
pool1_norm1 = LRN(name='pool1/norm1')(pool1_3x3_s2)
conv2_3x3_reduce = Convolution2D(64,
1,
1,
border_mode='same',
activation='relu',
name='conv2/3x3_reduce',
W_regularizer=l2(0.0002))(pool1_norm1)
conv2_3x3 = Convolution2D(192,
3,
3,
border_mode='same',
activation='relu',
name='conv2/3x3',
W_regularizer=l2(0.0002))(conv2_3x3_reduce)
conv2_norm2 = LRN(name='conv2/norm2')(conv2_3x3)
conv2_zero_pad = ZeroPadding2D(padding=(1, 1))(conv2_norm2)
pool2_helper = PoolHelper()(conv2_zero_pad)
pool2_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool2/3x3_s2')(pool2_helper)
inception_3a_1x1 = Convolution2D(64,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/1x1',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3_reduce = Convolution2D(
96,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/3x3_reduce',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_3x3 = Convolution2D(
128,
3,
3,
border_mode='same',
activation='relu',
name='inception_3a/3x3',
W_regularizer=l2(0.0002))(inception_3a_3x3_reduce)
inception_3a_5x5_reduce = Convolution2D(
16,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/5x5_reduce',
W_regularizer=l2(0.0002))(pool2_3x3_s2)
inception_3a_5x5 = Convolution2D(
32,
5,
5,
border_mode='same',
activation='relu',
name='inception_3a/5x5',
W_regularizer=l2(0.0002))(inception_3a_5x5_reduce)
inception_3a_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_3a/pool')(pool2_3x3_s2)
inception_3a_pool_proj = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_3a/pool_proj',
W_regularizer=l2(0.0002))(inception_3a_pool)
inception_3a_output = merge([
inception_3a_1x1, inception_3a_3x3, inception_3a_5x5,
inception_3a_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_3a/output')
inception_3b_1x1 = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/1x1',
W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3_reduce = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/3x3_reduce',
W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_3x3 = Convolution2D(
192,
3,
3,
border_mode='same',
activation='relu',
name='inception_3b/3x3',
W_regularizer=l2(0.0002))(inception_3b_3x3_reduce)
inception_3b_5x5_reduce = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/5x5_reduce',
W_regularizer=l2(0.0002))(inception_3a_output)
inception_3b_5x5 = Convolution2D(
96,
5,
5,
border_mode='same',
activation='relu',
name='inception_3b/5x5',
W_regularizer=l2(0.0002))(inception_3b_5x5_reduce)
inception_3b_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_3b/pool')(inception_3a_output)
inception_3b_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_3b/pool_proj',
W_regularizer=l2(0.0002))(inception_3b_pool)
inception_3b_output = merge([
inception_3b_1x1, inception_3b_3x3, inception_3b_5x5,
inception_3b_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_3b/output')
inception_3b_output_zero_pad = ZeroPadding2D(
padding=(1, 1))(inception_3b_output)
pool3_helper = PoolHelper()(inception_3b_output_zero_pad)
pool3_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool3/3x3_s2')(pool3_helper)
inception_4a_1x1 = Convolution2D(192,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/1x1',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3_reduce = Convolution2D(
96,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/3x3_reduce',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_3x3 = Convolution2D(
208,
3,
3,
border_mode='same',
activation='relu',
name='inception_4a/3x3',
W_regularizer=l2(0.0002))(inception_4a_3x3_reduce)
inception_4a_5x5_reduce = Convolution2D(
16,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/5x5_reduce',
W_regularizer=l2(0.0002))(pool3_3x3_s2)
inception_4a_5x5 = Convolution2D(
48,
5,
5,
border_mode='same',
activation='relu',
name='inception_4a/5x5',
W_regularizer=l2(0.0002))(inception_4a_5x5_reduce)
inception_4a_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4a/pool')(pool3_3x3_s2)
inception_4a_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_4a/pool_proj',
W_regularizer=l2(0.0002))(inception_4a_pool)
inception_4a_output = merge([
inception_4a_1x1, inception_4a_3x3, inception_4a_5x5,
inception_4a_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4a/output')
loss1_ave_pool = AveragePooling2D(
pool_size=(5, 5), strides=(3, 3),
name='loss1/ave_pool')(inception_4a_output)
loss1_conv = Convolution2D(128,
1,
1,
border_mode='same',
activation='relu',
name='loss1/conv',
W_regularizer=l2(0.0002))(loss1_ave_pool)
loss1_flat = Flatten()(loss1_conv)
loss1_fc = Dense(1024,
activation='relu',
name='loss1/fc',
W_regularizer=l2(0.0002))(loss1_flat)
loss1_drop_fc = Dropout(0.7)(loss1_fc)
loss1_classifier = Dense(1,
name='loss1/classifier',
W_regularizer=l2(0.0002))(loss1_drop_fc)
loss1_classifier_act = Activation('sigmoid')(loss1_classifier)
inception_4b_1x1 = Convolution2D(
160,
1,
1,
border_mode='same',
activation='relu',
name='inception_4b/1x1',
W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3_reduce = Convolution2D(
112,
1,
1,
border_mode='same',
activation='relu',
name='inception_4b/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_3x3 = Convolution2D(
224,
3,
3,
border_mode='same',
activation='relu',
name='inception_4b/3x3',
W_regularizer=l2(0.0002))(inception_4b_3x3_reduce)
inception_4b_5x5_reduce = Convolution2D(
24,
1,
1,
border_mode='same',
activation='relu',
name='inception_4b/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4a_output)
inception_4b_5x5 = Convolution2D(
64,
5,
5,
border_mode='same',
activation='relu',
name='inception_4b/5x5',
W_regularizer=l2(0.0002))(inception_4b_5x5_reduce)
inception_4b_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4b/pool')(inception_4a_output)
inception_4b_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_4b/pool_proj',
W_regularizer=l2(0.0002))(inception_4b_pool)
inception_4b_output = merge([
inception_4b_1x1, inception_4b_3x3, inception_4b_5x5,
inception_4b_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4b_output')
inception_4c_1x1 = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_4c/1x1',
W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3_reduce = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_4c/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_3x3 = Convolution2D(
256,
3,
3,
border_mode='same',
activation='relu',
name='inception_4c/3x3',
W_regularizer=l2(0.0002))(inception_4c_3x3_reduce)
inception_4c_5x5_reduce = Convolution2D(
24,
1,
1,
border_mode='same',
activation='relu',
name='inception_4c/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4b_output)
inception_4c_5x5 = Convolution2D(
64,
5,
5,
border_mode='same',
activation='relu',
name='inception_4c/5x5',
W_regularizer=l2(0.0002))(inception_4c_5x5_reduce)
inception_4c_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4c/pool')(inception_4b_output)
inception_4c_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_4c/pool_proj',
W_regularizer=l2(0.0002))(inception_4c_pool)
inception_4c_output = merge([
inception_4c_1x1, inception_4c_3x3, inception_4c_5x5,
inception_4c_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4c/output')
inception_4d_1x1 = Convolution2D(
112,
1,
1,
border_mode='same',
activation='relu',
name='inception_4d/1x1',
W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3_reduce = Convolution2D(
144,
1,
1,
border_mode='same',
activation='relu',
name='inception_4d/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_3x3 = Convolution2D(
288,
3,
3,
border_mode='same',
activation='relu',
name='inception_4d/3x3',
W_regularizer=l2(0.0002))(inception_4d_3x3_reduce)
inception_4d_5x5_reduce = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_4d/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4c_output)
inception_4d_5x5 = Convolution2D(
64,
5,
5,
border_mode='same',
activation='relu',
name='inception_4d/5x5',
W_regularizer=l2(0.0002))(inception_4d_5x5_reduce)
inception_4d_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4d/pool')(inception_4c_output)
inception_4d_pool_proj = Convolution2D(
64,
1,
1,
border_mode='same',
activation='relu',
name='inception_4d/pool_proj',
W_regularizer=l2(0.0002))(inception_4d_pool)
inception_4d_output = merge([
inception_4d_1x1, inception_4d_3x3, inception_4d_5x5,
inception_4d_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4d/output')
loss2_ave_pool = AveragePooling2D(
pool_size=(5, 5), strides=(3, 3),
name='loss2/ave_pool')(inception_4d_output)
loss2_conv = Convolution2D(128,
1,
1,
border_mode='same',
activation='relu',
name='loss2/conv',
W_regularizer=l2(0.0002))(loss2_ave_pool)
loss2_flat = Flatten()(loss2_conv)
loss2_fc = Dense(1024,
activation='relu',
name='loss2/fc',
W_regularizer=l2(0.0002))(loss2_flat)
loss2_drop_fc = Dropout(0.7)(loss2_fc)
loss2_classifier = Dense(1,
name='loss2/classifier',
W_regularizer=l2(0.0002))(loss2_drop_fc)
loss2_classifier_act = Activation('sigmoid')(loss2_classifier)
inception_4e_1x1 = Convolution2D(
256,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/1x1',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3_reduce = Convolution2D(
160,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/3x3_reduce',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_3x3 = Convolution2D(
320,
3,
3,
border_mode='same',
activation='relu',
name='inception_4e/3x3',
W_regularizer=l2(0.0002))(inception_4e_3x3_reduce)
inception_4e_5x5_reduce = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/5x5_reduce',
W_regularizer=l2(0.0002))(inception_4d_output)
inception_4e_5x5 = Convolution2D(
128,
5,
5,
border_mode='same',
activation='relu',
name='inception_4e/5x5',
W_regularizer=l2(0.0002))(inception_4e_5x5_reduce)
inception_4e_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_4e/pool')(inception_4d_output)
inception_4e_pool_proj = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_4e/pool_proj',
W_regularizer=l2(0.0002))(inception_4e_pool)
inception_4e_output = merge([
inception_4e_1x1, inception_4e_3x3, inception_4e_5x5,
inception_4e_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_4e/output')
inception_4e_output_zero_pad = ZeroPadding2D(
padding=(1, 1))(inception_4e_output)
pool4_helper = PoolHelper()(inception_4e_output_zero_pad)
pool4_3x3_s2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
border_mode='valid',
name='pool4/3x3_s2')(pool4_helper)
inception_5a_1x1 = Convolution2D(256,
1,
1,
border_mode='same',
activation='relu',
name='inception_5a/1x1',
W_regularizer=l2(0.0002))(pool4_3x3_s2)
inception_5a_3x3_reduce = Convolution2D(
160,
1,
1,
border_mode='same',
activation='relu',
name='inception_5a/3x3_reduce',
W_regularizer=l2(0.0002))(pool4_3x3_s2)
inception_5a_3x3 = Convolution2D(
320,
3,
3,
border_mode='same',
activation='relu',
name='inception_5a/3x3',
W_regularizer=l2(0.0002))(inception_5a_3x3_reduce)
inception_5a_5x5_reduce = Convolution2D(
32,
1,
1,
border_mode='same',
activation='relu',
name='inception_5a/5x5_reduce',
W_regularizer=l2(0.0002))(pool4_3x3_s2)
inception_5a_5x5 = Convolution2D(
128,
5,
5,
border_mode='same',
activation='relu',
name='inception_5a/5x5',
W_regularizer=l2(0.0002))(inception_5a_5x5_reduce)
inception_5a_pool = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_5a/pool')(pool4_3x3_s2)
inception_5a_pool_proj = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_5a/pool_proj',
W_regularizer=l2(0.0002))(inception_5a_pool)
inception_5a_output = merge([
inception_5a_1x1, inception_5a_3x3, inception_5a_5x5,
inception_5a_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_5a/output')
inception_5b_1x1 = Convolution2D(
384,
1,
1,
border_mode='same',
activation='relu',
name='inception_5b/1x1',
W_regularizer=l2(0.0002))(inception_5a_output)
inception_5b_3x3_reduce = Convolution2D(
192,
1,
1,
border_mode='same',
activation='relu',
name='inception_5b/3x3_reduce',
W_regularizer=l2(0.0002))(inception_5a_output)
inception_5b_3x3 = Convolution2D(
384,
3,
3,
border_mode='same',
activation='relu',
name='inception_5b/3x3',
W_regularizer=l2(0.0002))(inception_5b_3x3_reduce)
inception_5b_5x5_reduce = Convolution2D(
48,
1,
1,
border_mode='same',
activation='relu',
name='inception_5b/5x5_reduce',
W_regularizer=l2(0.0002))(inception_5a_output)
inception_5b_5x5 = Convolution2D(
128,
5,
5,
border_mode='same',
activation='relu',
name='inception_5b/5x5',
W_regularizer=l2(0.0002))(inception_5b_5x5_reduce)
inception_5b_pool = MaxPooling2D(
pool_size=(3, 3),
strides=(1, 1),
border_mode='same',
name='inception_5b/pool')(inception_5a_output)
inception_5b_pool_proj = Convolution2D(
128,
1,
1,
border_mode='same',
activation='relu',
name='inception_5b/pool_proj',
W_regularizer=l2(0.0002))(inception_5b_pool)
inception_5b_output = merge([
inception_5b_1x1, inception_5b_3x3, inception_5b_5x5,
inception_5b_pool_proj
],
mode='concat',
concat_axis=1,
name='inception_5b/output')
pool5_7x7_s1 = AveragePooling2D(pool_size=(7, 7),
strides=(1, 1),
name='pool5/7x7_s2')(inception_5b_output)
loss3_flat = Flatten()(pool5_7x7_s1)
pool5_drop_7x7_s1 = Dropout(0.4)(loss3_flat)
loss3_classifier = Dense(1,
name='loss3/classifier',
W_regularizer=l2(0.0002))(pool5_drop_7x7_s1)
loss3_classifier_act = Activation('sigmoid', name='prob')(loss3_classifier)
googlenet = Model(input=input,
output=[
loss1_classifier_act, loss2_classifier_act,
loss3_classifier_act
])
sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
googlenet.compile(optimizer=sgd,
loss='binary_crossentropy',
metrics=['accuracy'])
return googlenet