def sam_vgg(data):
# conv_1
trainable = True
conv_1_out = Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv1',
trainable=trainable)(data)
conv_1_out = Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv2',
trainable=trainable)(conv_1_out)
ds_conv_1_out = MaxPooling2D((2, 2), strides=(2, 2),
name='block1_pool')(conv_1_out)
# conv_2
conv_2_out = Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv1',
trainable=trainable)(ds_conv_1_out)
conv_2_out = Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv2',
trainable=trainable)(conv_2_out)
ds_conv_2_out = MaxPooling2D((2, 2), strides=(2, 2),
name='block2_pool')(conv_2_out)
# conv_3
conv_3_out = Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv1',
trainable=trainable)(ds_conv_2_out)
conv_3_out = Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv2',
trainable=trainable)(conv_3_out)
conv_3_out = Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv3',
trainable=trainable)(conv_3_out)
ds_conv_3_out = MaxPooling2D((2, 2),
strides=(2, 2),
name='block3_pool',
padding='same')(conv_3_out)
# conv_4
conv_4_out = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv1',
trainable=trainable)(ds_conv_3_out)
conv_4_out = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv2',
trainable=trainable)(conv_4_out)
conv_4_out = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv3',
trainable=trainable)(conv_4_out)
ds_conv_4_out = MaxPooling2D((2, 2),
strides=(2, 2),
name='block4_pool',
padding='same')(conv_4_out)
# conv_5 #
conv_5_out = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv1',
trainable=trainable)(ds_conv_4_out)
conv_5_out = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv2',
trainable=trainable)(conv_5_out)
conv_5_out = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block5_conv3',
trainable=trainable)(conv_5_out)
conv_6_out = MaxPooling2D((2, 2),
strides=(2, 2),
name='block5_pool',
padding='same')(conv_5_out)
# salconv_6#
salconv_6_out = Conv2D(512, (5, 5),
padding='same',
activation='relu',
trainable=trainable)(conv_6_out)
salconv_6_out = Conv2D(512, (5, 5),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_6_out)
salconv_5_out = Conv2D(512, (3, 3),
padding='same',
activation='relu',
trainable=trainable)(conv_5_out)
salconv_5_out = Conv2D(512, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_5_out)
edgeconv_5_out = Conv2D(512, (3, 3),
padding='same',
activation='relu',
trainable=trainable)(conv_5_out)
edgeconv_5_out = Conv2D(512, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(edgeconv_5_out)
salconv_4_out = Conv2D(256, (3, 3),
padding='same',
activation='relu',
trainable=trainable)(conv_4_out)
salconv_4_out = Conv2D(256, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_4_out)
edgeconv_4_out = Conv2D(256, (3, 3),
padding='same',
activation='relu',
trainable=trainable)(conv_4_out)
edgeconv_4_out = Conv2D(256, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(edgeconv_4_out)
salconv_3_out = Conv2D(256, (3, 3),
padding='same',
activation='relu',
trainable=trainable)(conv_3_out)
salconv_3_out = Conv2D(256, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_3_out)
edgeconv_3_out = Conv2D(256, (3, 3),
padding='same',
activation='relu',
trainable=trainable)(conv_3_out)
edgeconv_3_out = Conv2D(256, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(edgeconv_3_out)
salconv_2_out = Conv2D(128, (3, 3),
padding='same',
activation='relu',
trainable=trainable)(conv_2_out)
salconv_2_out = Conv2D(128, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_2_out)
edgeconv_2_out = Conv2D(128, (3, 3),
padding='same',
activation='relu',
trainable=trainable)(conv_2_out)
edgeconv_2_out = Conv2D(128, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(edgeconv_2_out)
salconv_1_out = Conv2D(64, (3, 3),
padding='same',
activation='relu',
trainable=trainable)(conv_1_out)
salconv_1_out = Conv2D(64, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_1_out)
edgeconv_1_out = Conv2D(64, (3, 3),
padding='same',
activation='relu',
trainable=trainable)(conv_1_out)
edgeconv_1_out = Conv2D(64, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(edgeconv_1_out)
# saliency from conv_6 #
saliency6 = Conv2D(1, (1, 1),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_6_out)
saliency6_up = UpsampleLike()([saliency6, conv_1_out])
# saliency from conv_5 #
edge5 = Conv2D(1, (1, 1),
padding='same',
activation='sigmoid',
trainable=trainable)(edgeconv_5_out)
edge5_up = UpsampleLike()([edge5, conv_1_out])
salconv_5_out = Concatenate()(
[salconv_5_out,
UpsampleLike()([saliency6, salconv_5_out])])
salconv_5_out = Conv2D(256, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_5_out)
attention_5a = Conv2D(1, (1, 1), padding='same',
activation='sigmoid')(salconv_5_out)
attention_5a = Flatten()(attention_5a)
attention_5a = Softmax()(attention_5a)
attention_5b = Conv2D(1, (1, 1), padding='same', activation='sigmoid', dilation_rate=(3, 3))\
(MaxPooling2D((2, 2), strides=(2, 2), padding='same')(salconv_5_out))
attention_5b = UpsampleLike()([attention_5b, salconv_5_out])
attention_5b = Flatten()(attention_5b)
attention_5b = Softmax()(attention_5b)
attention_5 = Average()([attention_5a, attention_5b])
attention_5 = RepeatVector(256)(attention_5)
attention_5 = Permute((2, 1))(attention_5)
attention_5 = Reshape((14, 14, 256), name='att_conv5')(attention_5)
attention_5 = Multiply()([attention_5, salconv_5_out])
salconv_5_out = Add()([attention_5, salconv_5_out])
salconv_5_out = Concatenate()([salconv_5_out, edge5])
salconv_5_out = Conv2D(256, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_5_out)
saliency5 = Conv2D(1, (1, 1),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_5_out)
saliency5_up = UpsampleLike()([saliency5, conv_1_out])
# saliency from conv_4 #
edgeconv_4_out = Concatenate()(
[edgeconv_4_out,
UpsampleLike()([edge5, salconv_4_out])])
edgeconv_4_out = Conv2D(128, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(edgeconv_4_out)
edge4 = Conv2D(1, (1, 1),
padding='same',
activation='sigmoid',
trainable=trainable)(edgeconv_4_out)
edge4_up = UpsampleLike()([edge4, conv_1_out])
salconv_4_out = Concatenate()([
salconv_4_out,
UpsampleLike()([saliency6, salconv_4_out]),
UpsampleLike()([saliency5, salconv_4_out])
])
salconv_4_out = Conv2D(128, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_4_out)
attention_4a = Conv2D(1, (1, 1), padding='same',
activation='sigmoid')(salconv_4_out)
attention_4a = Flatten()(attention_4a)
attention_4a = Softmax()(attention_4a)
attention_4b = Conv2D(1, (1, 1), padding='same', activation='sigmoid', dilation_rate=(3, 3)) \
(MaxPooling2D((2, 2), strides=(2, 2), padding='same')(salconv_4_out))
attention_4b = UpsampleLike()([attention_4b, salconv_4_out])
attention_4b = Flatten()(attention_4b)
attention_4b = Softmax()(attention_4b)
attention_4c = Conv2D(1, (1, 1), padding='same', activation='sigmoid', dilation_rate=(3, 3)) \
(MaxPooling2D((4, 4), strides=(4, 4), padding='same')(salconv_4_out))
attention_4c = UpsampleLike()([attention_4c, salconv_4_out])
attention_4c = Flatten()(attention_4c)
attention_4c = Softmax()(attention_4c)
attention_4 = Average()([attention_4a, attention_4b, attention_4c])
attention_4 = RepeatVector(128)(attention_4)
attention_4 = Permute((2, 1))(attention_4)
attention_4 = Reshape((28, 28, 128), name='att_conv4')(attention_4)
attention_4 = Multiply()([attention_4, salconv_4_out])
salconv_4_out = Add()([attention_4, salconv_4_out])
salconv_4_out = Concatenate()([salconv_4_out, edge4])
salconv_4_out = Conv2D(128, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_4_out)
saliency4 = Conv2D(1, (1, 1),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_4_out)
saliency4_up = UpsampleLike()([saliency4, conv_1_out])
# saliency from conv_3 #
edgeconv_3_out = Concatenate()([
edgeconv_3_out,
UpsampleLike()([edge4, salconv_3_out]),
UpsampleLike()([edge5, salconv_3_out])
])
edgeconv_3_out = Conv2D(128, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(edgeconv_3_out)
edge3 = Conv2D(1, (1, 1),
padding='same',
activation='sigmoid',
trainable=trainable)(edgeconv_3_out)
edge3_up = UpsampleLike()([edge3, conv_1_out])
salconv_3_out = Concatenate()([
salconv_3_out,
UpsampleLike()([saliency6, salconv_3_out]),
UpsampleLike()([saliency5, salconv_3_out]),
UpsampleLike()([saliency4, salconv_3_out])
])
salconv_3_out = Conv2D(128, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_3_out)
attention_3a = Conv2D(1, (1, 1), padding='same',
activation='sigmoid')(salconv_3_out)
attention_3a = Flatten()(attention_3a)
attention_3a = Softmax()(attention_3a)
attention_3b = Conv2D(1, (1, 1), padding='same', activation='sigmoid', dilation_rate=(3, 3)) \
(MaxPooling2D((2, 2), strides=(2, 2), padding='same')(salconv_3_out))
attention_3b = UpsampleLike()([attention_3b, salconv_3_out])
attention_3b = Flatten()(attention_3b)
attention_3b = Softmax()(attention_3b)
attention_3c = Conv2D(1, (1, 1), padding='same', activation='sigmoid', dilation_rate=(3, 3)) \
(MaxPooling2D((4, 4), strides=(4, 4), padding='same')(salconv_3_out))
attention_3c = UpsampleLike()([attention_3c, salconv_3_out])
attention_3c = Flatten()(attention_3c)
attention_3c = Softmax()(attention_3c)
attention_3d = Conv2D(1, (1, 1), padding='same', activation='sigmoid', dilation_rate=(3, 3)) \
(MaxPooling2D((8, 8), strides=(8, 8), padding='same')(salconv_3_out))
attention_3d = UpsampleLike()([attention_3d, salconv_3_out])
attention_3d = Flatten()(attention_3d)
attention_3d = Softmax()(attention_3d)
attention_3 = Average()(
[attention_3a, attention_3b, attention_3c, attention_3d])
attention_3 = RepeatVector(128)(attention_3)
attention_3 = Permute((2, 1))(attention_3)
attention_3 = Reshape((56, 56, 128), name='att_conv3')(attention_3)
attention_3 = Multiply()([attention_3, salconv_3_out])
salconv_3_out = Add()([attention_3, salconv_3_out])
salconv_3_out = Concatenate()([salconv_3_out, edge3])
salconv_3_out = Conv2D(128, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_3_out)
saliency3 = Conv2D(1, (1, 1),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_3_out)
saliency3_up = UpsampleLike()([saliency3, conv_1_out])
# saliency from conv_2 #
edgeconv_2_out = Concatenate()([
edgeconv_2_out,
UpsampleLike()([edge5, salconv_2_out]),
UpsampleLike()([edge4, salconv_2_out]),
UpsampleLike()([edge3, salconv_2_out])
])
edgeconv_2_out = Conv2D(64, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(edgeconv_2_out)
edge2 = Conv2D(1, (1, 1),
padding='same',
activation='sigmoid',
trainable=trainable)(edgeconv_2_out)
edge2_up = UpsampleLike()([edge2, conv_1_out])
salconv_2_out = Concatenate()([
salconv_2_out,
UpsampleLike()([saliency6, salconv_2_out]),
UpsampleLike()([saliency5, salconv_2_out]),
UpsampleLike()([saliency4, salconv_2_out]),
UpsampleLike()([saliency3, salconv_2_out])
])
salconv_2_out = Conv2D(64, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_2_out)
attention_2a = Conv2D(1, (1, 1), padding='same',
activation='sigmoid')(salconv_2_out)
attention_2a = Flatten()(attention_2a)
attention_2a = Softmax()(attention_2a)
attention_2b = Conv2D(1, (1, 1), padding='same', activation='sigmoid', dilation_rate=(3, 3)) \
(MaxPooling2D((2, 2), strides=(2, 2), padding='same')(salconv_2_out))
attention_2b = UpsampleLike()([attention_2b, salconv_2_out])
attention_2b = Flatten()(attention_2b)
attention_2b = Softmax()(attention_2b)
attention_2c = Conv2D(1, (1, 1), padding='same', activation='sigmoid', dilation_rate=(3, 3)) \
(MaxPooling2D((4, 4), strides=(4, 4), padding='same')(salconv_2_out))
attention_2c = UpsampleLike()([attention_2c, salconv_2_out])
attention_2c = Flatten()(attention_2c)
attention_2c = Softmax()(attention_2c)
attention_2d = Conv2D(1, (1, 1), padding='same', activation='sigmoid', dilation_rate=(3, 3)) \
(MaxPooling2D((8, 8), strides=(8, 8), padding='same')(salconv_2_out))
attention_2d = UpsampleLike()([attention_2d, salconv_2_out])
attention_2d = Flatten()(attention_2d)
attention_2d = Softmax()(attention_2d)
attention_2e = Conv2D(1, (1, 1), padding='same', activation='sigmoid', dilation_rate=(3, 3)) \
(MaxPooling2D((16, 16), strides=(16, 16), padding='same')(salconv_2_out))
attention_2e = UpsampleLike()([attention_2e, salconv_2_out])
attention_2e = Flatten()(attention_2e)
attention_2e = Softmax()(attention_2e)
attention_2 = Average()(
[attention_2a, attention_2b, attention_2c, attention_2d, attention_2e])
attention_2 = RepeatVector(64)(attention_2)
attention_2 = Permute((2, 1))(attention_2)
attention_2 = Reshape((112, 112, 64), name='att_conv2')(attention_2)
attention_2 = Multiply()([attention_2, salconv_2_out])
salconv_2_out = Add()([attention_2, salconv_2_out])
salconv_2_out = Concatenate()([salconv_2_out, edge2])
salconv_2_out = Conv2D(64, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_2_out)
saliency2 = Conv2D(1, (1, 1),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_2_out)
saliency2_up = UpsampleLike()([saliency2, conv_1_out])
# saliency from conv_1 #
edgeconv_1_out = Concatenate()([
edgeconv_1_out,
UpsampleLike()([edge5, salconv_1_out]),
UpsampleLike()([edge4, salconv_1_out]),
UpsampleLike()([edge3, salconv_1_out]),
UpsampleLike()([edge2, salconv_1_out])
])
edgeconv_1_out = Conv2D(32, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(edgeconv_1_out)
edge1 = Conv2D(1, (1, 1),
padding='same',
activation='sigmoid',
trainable=trainable)(edgeconv_1_out)
salconv_1_out = Concatenate()([
salconv_1_out,
UpsampleLike()([saliency6, salconv_1_out]),
UpsampleLike()([saliency5, salconv_1_out]),
UpsampleLike()([saliency4, salconv_1_out]),
UpsampleLike()([saliency3, salconv_1_out]),
UpsampleLike()([saliency2, salconv_1_out])
])
salconv_1_out = Conv2D(32, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_1_out)
attention_1a = Conv2D(1, (1, 1), padding='same',
activation='sigmoid')(salconv_1_out)
attention_1a = Flatten()(attention_1a)
attention_1a = Softmax()(attention_1a)
attention_1b = Conv2D(1, (1, 1), padding='same', activation='sigmoid', dilation_rate=(3, 3)) \
(MaxPooling2D((2, 2), strides=(2, 2), padding='same')(salconv_1_out))
attention_1b = UpsampleLike()([attention_1b, salconv_1_out])
attention_1b = Flatten()(attention_1b)
attention_1b = Softmax()(attention_1b)
attention_1c = Conv2D(1, (1, 1), padding='same', activation='sigmoid', dilation_rate=(3, 3)) \
(MaxPooling2D((4, 4), strides=(4, 4), padding='same')(salconv_1_out))
attention_1c = UpsampleLike()([attention_1c, salconv_1_out])
attention_1c = Flatten()(attention_1c)
attention_1c = Softmax()(attention_1c)
attention_1d = Conv2D(1, (1, 1), padding='same', activation='sigmoid', dilation_rate=(3, 3)) \
(MaxPooling2D((8, 8), strides=(8, 8), padding='same')(salconv_1_out))
attention_1d = UpsampleLike()([attention_1d, salconv_1_out])
attention_1d = Flatten()(attention_1d)
attention_1d = Softmax()(attention_1d)
attention_1e = Conv2D(1, (1, 1), padding='same', activation='sigmoid', dilation_rate=(3, 3)) \
(MaxPooling2D((16, 16), strides=(16, 16), padding='same')(salconv_1_out))
attention_1e = UpsampleLike()([attention_1e, salconv_1_out])
attention_1e = Flatten()(attention_1e)
attention_1e = Softmax()(attention_1e)
attention_1f = Conv2D(1, (1, 1), padding='same', activation='sigmoid', dilation_rate=(3, 3)) \
(MaxPooling2D((32, 32), strides=(32, 32), padding='same')(salconv_1_out))
attention_1f = UpsampleLike()([attention_1f, salconv_1_out])
attention_1f = Flatten()(attention_1f)
attention_1f = Softmax()(attention_1f)
attention_1 = Average()([
attention_1a, attention_1b, attention_1c, attention_1d, attention_1e,
attention_1f
])
attention_1 = RepeatVector(32)(attention_1)
attention_1 = Permute((2, 1))(attention_1)
attention_1 = Reshape((224, 224, 32), name='att_conv1')(attention_1)
attention_1 = Multiply()([attention_1, salconv_1_out])
salconv_1_out = Add()([attention_1, salconv_1_out])
salconv_1_out = Concatenate()([salconv_1_out, edge1])
salconv_1_out = Conv2D(32, (3, 3),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_1_out)
saliency1 = Conv2D(1, (1, 1),
padding='same',
activation='sigmoid',
trainable=trainable)(salconv_1_out)
return [
saliency6_up, saliency5_up, edge5_up, saliency4_up, edge4_up,
saliency3_up, edge3_up, saliency2_up, edge2_up, saliency1, edge1
]
def build_extractor_29layers_v2(self, name, block, layers):
in_img = Input(shape=(*self.in_size_hw, 1))
X = self._mfm(in_img,
name=name + '_mfm1',
out_channels=48,
kernel_size=5,
strides=1)
X = Average()([
MaxPooling2D(pool_size=2, padding='same')(X),
AveragePooling2D(pool_size=2, padding='same')(X)
])
X = self._make_layer(X,
name=name + '_layers1',
block=block,
num_blocks=layers[0],
out_channels=48)
X = self._group(X,
name=name + '_group1',
in_channels=48,
out_channels=96,
kernel_size=3,
strides=1)
X = Average()([
MaxPooling2D(pool_size=2, padding='same')(X),
AveragePooling2D(pool_size=2, padding='same')(X)
])
X = self._make_layer(X,
name=name + '_layers2',
block=block,
num_blocks=layers[1],
out_channels=96)
X = self._group(X,
name=name + '_group2',
in_channels=96,
out_channels=192,
kernel_size=3,
strides=1)
X = Average()([
MaxPooling2D(pool_size=2, padding='same')(X),
AveragePooling2D(pool_size=2, padding='same')(X)
])
X = self._make_layer(X,
name=name + '_layers3',
block=block,
num_blocks=layers[2],
out_channels=192)
X = self._group(X,
name=name + '_group3',
in_channels=192,
out_channels=128,
kernel_size=3,
strides=1)
X = self._make_layer(X,
name=name + '_layers4',
block=block,
num_blocks=layers[3],
out_channels=128)
X = self._group(X,
name=name + '_group4',
in_channels=128,
out_channels=128,
kernel_size=3,
strides=1)
X = Average()([
MaxPooling2D(pool_size=2, padding='same')(X),
AveragePooling2D(pool_size=2, padding='same')(X)
])
feat = Dense(256,
name=name + '_dense1',
kernel_regularizer=regularizers.l2(0.0005))(Flatten()(X))
ret_extractor = Model(inputs=in_img, outputs=feat, name=name)
ret_extractor.summary()
return ret_extractor
weights=None)(embedded_sequences)
elif model_choice == 2:
gru_kata = Bidirectional(GRU(EMBEDDING_DIM,
return_sequences=True,
dropout=dropout,
recurrent_dropout=rec_dropout),
merge_mode=merge_m,
weights=None)(rtwo)
else:
combine = 1 # input('Enter 1 for Add, 2 for Subtract, 3 for Multiply, 4 for Average, 5 for Maximum: ')
if combine == 2:
merge = Subtract()([embedded_sequences, rtwo])
elif combine == 3:
merge = Multiply()([embedded_sequences, rtwo])
elif combine == 4:
merge = Average()([embedded_sequences, rtwo])
elif combine == 5:
merge = Maximum()([embedded_sequences, rtwo])
else:
merge = Add()([embedded_sequences, rtwo])
gru_kata = Bidirectional(GRU(EMBEDDING_DIM,
return_sequences=True,
dropout=dropout,
recurrent_dropout=rec_dropout,
trainable=gtrainable),
merge_mode=merge_m,
weights=None)(merge)
crf = CRF(len(label.index) + 1, learn_mode='marginal')(gru_kata)
preds = Dense(len(label.index) + 1, activation='softmax')(gru_kata)
y = mobilenet.mobilenet_by_me(
name='temporal',
inputs=input_y,
input_shape=(224,224,20),
classes=classes,
weight='weights/mobilenet_temporal{}_{}e.h5'.format(opt_size,tem_epochs))
else:
y = mobilenet.mobilenet_by_me(
name='temporal',
inputs=input_y,
input_shape=(224,224,20),
classes=classes)
# Fusion
if fusion == 'avg':
z = Average()([x, y])
elif fusion == 'max':
z = Maximum()([x, y])
elif fusion == 'concat':
z = Concatenate()([x, y])
elif fusion == 'conv':
z = Concatenate()([x, y])
z = Conv2D(256, (1, 1), use_bias=True)(z)
else:
z = Multiply()([x, y])
z = GlobalAveragePooling2D()(z)
if fusion == 'concat':
z = Reshape((1,1,2048))(z)
elif fusion == 'conv':
z = Reshape((1,1,256))(z)
def create_relationships(rel_type,
g_theta_model,
p1_joints,
p2_joints,
use_attention=False,
use_relations=True,
attention_proj_size=None,
return_attention=False):
g_theta_outs = []
if not use_relations:
for object_i in p1_joints:
g_theta_outs.append(g_theta_model([object_i]))
if use_attention:
# Output may be tuple if return_attention is true, second element is attention vector
return IRNAttention(
projection_size=attention_proj_size,
return_attention=return_attention)(g_theta_outs)
else:
rel_out = Average()(g_theta_outs)
return rel_out
if rel_type == 'inter' or rel_type == 'p1_p2_all_bidirectional':
# All joints from person1 connected to all joints of person2, and back
for object_i in p1_joints:
for object_j in p2_joints:
g_theta_outs.append(g_theta_model([object_i, object_j]))
for object_i in p2_joints:
for object_j in p1_joints:
g_theta_outs.append(g_theta_model([object_i, object_j]))
rel_out = Average()(g_theta_outs)
elif rel_type == 'intra' or rel_type == 'indivs':
indiv1_avg = create_relationships('p1_p1_all', g_theta_model,
p1_joints, p2_joints)
indiv2_avg = create_relationships('p2_p2_all', g_theta_model,
p1_joints, p2_joints)
rel_out = Concatenate()([indiv1_avg, indiv2_avg])
elif rel_type == 'inter_and_indivs':
# All joints from person1 connected to all joints of person2, and back
for object_i in p1_joints:
for object_j in p2_joints:
g_theta_outs.append(g_theta_model([object_i, object_j]))
for object_i in p2_joints:
for object_j in p1_joints:
g_theta_outs.append(g_theta_model([object_i, object_j]))
# All joints from person1 connected to all other joints of itself
for idx, object_i in enumerate(p1_joints):
for object_j in p1_joints[idx + 1:]:
# for object_j in p1_joints[idx:]:
g_theta_outs.append(g_theta_model([object_i, object_j]))
# All joints from person2 connected to all other joints of itself
for idx, object_i in enumerate(p2_joints):
for object_j in p2_joints[idx + 1:]:
# for object_j in p2_joints[idx:]:
g_theta_outs.append(g_theta_model([object_i, object_j]))
rel_out = Average()(g_theta_outs)
elif rel_type == 'p1_p2_all':
# All joints from person1 connected to all joints of person2
for object_i in p1_joints:
for object_j in p2_joints:
g_theta_outs.append(g_theta_model([object_i, object_j]))
rel_out = Average()(g_theta_outs)
elif rel_type == 'p1_p1_all':
# All joints from person1 connected to all other joints of itself
for idx, object_i in enumerate(p1_joints):
for object_j in p1_joints[idx + 1:]:
# for object_j in p1_joints[idx:]:
g_theta_outs.append(g_theta_model([object_i, object_j]))
if use_attention:
return IRNAttention(
projection_size=attention_proj_size,
return_attention=return_attention)(g_theta_outs)
else:
rel_out = Average()(g_theta_outs)
elif rel_type == 'p2_p2_all':
# All joints from person2 connected to all other joints of itself
rel_out = create_relationships('p1_p1_all', g_theta_model, p2_joints,
p1_joints)
elif rel_type == 'p1_p1_all_bidirectional':
# All joints from person1 connected to all other joints of itself, and back
rel_out = create_relationships('p1_p2_all_bidirectional',
g_theta_model, p1_joints, p1_joints)
elif rel_type == 'p2_p2_all_bidirectional':
# All joints from person2 connected to all other joints of itself, and back
rel_out = create_relationships('p1_p2_all_bidirectional',
g_theta_model, p2_joints, p2_joints)
elif rel_type == 'p1_p1_all-p2_p2_all':
# All joints from person1 connected to all other joints of itself
for idx, object_i in enumerate(p1_joints):
for object_j in p1_joints[idx + 1:]:
# for object_j in p1_joints[idx:]:
g_theta_outs.append(g_theta_model([object_i, object_j]))
for idx, object_i in enumerate(p2_joints):
for object_j in p2_joints[idx + 1:]:
# for object_j in p1_joints[idx:]:
g_theta_outs.append(g_theta_model([object_i, object_j]))
rel_out = Average()(g_theta_outs)
else:
raise ValueError("Invalid rel_type:", rel_type)
return rel_out
def main():
try:
os.mkdir('./img')
except OSError:
pass
# face_cascade = cv2.CascadeClassifier('lbpcascade_frontalface_improved.xml')
# detector = MTCNN()
# load model and weights
img_size = 64
stage_num = [3, 3, 3]
lambda_local = 1
lambda_d = 1
img_idx = 0
detected = '' #make this not local variable
time_detection = 0
time_network = 0
time_plot = 0
skip_frame = 1 # every 5 frame do 1 detection and network forward propagation
ad = 0.6
#Parameters
num_capsule = 3
dim_capsule = 16
routings = 2
stage_num = [3, 3, 3]
lambda_d = 1
num_classes = 3
image_size = 64
num_primcaps = 7 * 3
m_dim = 5
S_set = [num_capsule, dim_capsule, routings, num_primcaps, m_dim]
model1 = FSA_net_Capsule(image_size, num_classes, stage_num, lambda_d,
S_set)()
model2 = FSA_net_Var_Capsule(image_size, num_classes, stage_num, lambda_d,
S_set)()
num_primcaps = 8 * 8 * 3
S_set = [num_capsule, dim_capsule, routings, num_primcaps, m_dim]
model3 = FSA_net_noS_Capsule(image_size, num_classes, stage_num, lambda_d,
S_set)()
print('Loading models ...')
weight_file1 = '../pre-trained/300W_LP_models/fsanet_capsule_3_16_2_21_5/fsanet_capsule_3_16_2_21_5.h5'
model1.load_weights(weight_file1)
print('Finished loading model 1.')
weight_file2 = '../pre-trained/300W_LP_models/fsanet_var_capsule_3_16_2_21_5/fsanet_var_capsule_3_16_2_21_5.h5'
model2.load_weights(weight_file2)
print('Finished loading model 2.')
weight_file3 = '../pre-trained/300W_LP_models/fsanet_noS_capsule_3_16_2_192_5/fsanet_noS_capsule_3_16_2_192_5.h5'
model3.load_weights(weight_file3)
print('Finished loading model 3.')
inputs = Input(shape=(64, 64, 3))
x1 = model1(inputs) #1x1
x2 = model2(inputs) #var
x3 = model3(inputs) #w/o
avg_model = Average()([x1, x2, x3])
model = Model(inputs=inputs, outputs=avg_model)
# load our serialized face detector from disk
print("[INFO] loading face detector...")
protoPath = os.path.sep.join(["face_detector", "deploy.prototxt"])
modelPath = os.path.sep.join(
["face_detector", "res10_300x300_ssd_iter_140000.caffemodel"])
net = cv2.dnn.readNetFromCaffe(protoPath, modelPath)
# capture video
#cap = cv2.VideoCapture(0)
#cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1024*1)
#cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 768*1)
print('Start detecting pose ...')
detected_pre = np.empty((1, 1, 1))
while True:
# get video frame
#ret, input_img = cap.read()
#input_img = cv2.imread('HELEN_147110327_1_4.jpg')
photo_path = '/media/nuaa301/OS/师兄程序/资料整理/资料整理/论文程序/演示系统程序/python_test3/img1/103.jpg'
input_img = cv2.imread(photo_path)
img_idx = img_idx + 1
img_h, img_w, _ = np.shape(input_img)
if img_idx == 1 or img_idx % skip_frame == 0:
time_detection = 0
time_network = 0
time_plot = 0
# detect faces using LBP detector
gray_img = cv2.cvtColor(input_img, cv2.COLOR_BGR2GRAY)
# detected = face_cascade.detectMultiScale(gray_img, 1.1)
# detected = detector.detect_faces(input_img)
# pass the blob through the network and obtain the detections and
# predictions
blob = cv2.dnn.blobFromImage(cv2.resize(input_img,
(300, 300)), 1.0,
(300, 300), (104.0, 177.0, 123.0))
net.setInput(blob)
detected = net.forward()
if detected_pre.shape[2] > 0 and detected.shape[2] == 0:
detected = detected_pre
faces = np.empty((detected.shape[2], img_size, img_size, 3))
input_img = draw_results_ssd(detected, input_img, faces, ad,
img_size, img_w, img_h, model,
time_detection, time_network,
time_plot)
# cv2.imwrite('img/'+str(img_idx)+'.png',input_img)
else:
input_img = draw_results_ssd(detected, input_img, faces, ad,
img_size, img_w, img_h, model,
time_detection, time_network,
time_plot)
if detected.shape[2] > detected_pre.shape[2] or img_idx % (skip_frame *
3) == 0:
detected_pre = detected
key = cv2.waitKey(0)
def carnn(embedding_matrix,
config,
compare_out_size=CARNN_COMPARE_LAYER_OUTSIZE,
rnn_size=CARNN_RNN_SIZE,
rnn_dropout=CARNN_AGGREATION_DROPOUT):
q1 = Input(shape=(config['max_length'], ), dtype='int32', name='q1_input')
q2 = Input((config['max_length'], ), dtype='int32', name='q2_input')
activation = 'elu'
compare_dim = 500
compare_dropout = 0.2
embedding_layer = Embedding(embedding_matrix.shape[0],
embedding_matrix.shape[1],
trainable=config['embed_trainable'],
weights=[embedding_matrix]
# mask_zero=True
)
q1_embed = embedding_layer(q1)
q2_embed = embedding_layer(q2) # bsz, 1, emb_dims
q1_embed = BatchNormalization(axis=2)(q1_embed)
q2_embed = BatchNormalization(axis=2)(q2_embed)
q1_embed = SpatialDropout1D(config['spatial_dropout_rate'])(q1_embed)
q2_embed = SpatialDropout1D(config['spatial_dropout_rate'])(q2_embed)
highway_encoder = TimeDistributed(Highway(activation='relu'))
self_attention = SelfAttention(d_model=embedding_matrix.shape[1])
q1_encoded = highway_encoder(q1_embed, )
q2_encoded = highway_encoder(q2_embed, )
s1_encoded = self_attention(q1, q1_encoded)
s2_encoded = self_attention(q2, q2_encoded)
# Attention
q1_aligned, q2_aligned = soft_attention_alignment(q1_encoded, q2_encoded)
# Compare
q1_combined1 = Concatenate()([
q1_encoded,
q2_aligned,
interaction(q1_encoded, q2_aligned),
])
q1_combined2 = Concatenate()([
q2_aligned,
q1_encoded,
interaction(q1_encoded, q2_aligned),
])
q2_combined1 = Concatenate()([
q2_encoded,
q1_aligned,
interaction(q2_encoded, q1_aligned),
])
q2_combined2 = Concatenate()([
q1_aligned,
q2_encoded,
interaction(q2_encoded, q1_aligned),
])
s1_combined1 = Concatenate()([
q1_encoded,
s1_encoded,
interaction(q1_encoded, s1_encoded),
])
s1_combined2 = Concatenate()([
s1_encoded,
q1_encoded,
interaction(q1_encoded, s1_encoded),
])
s2_combined1 = Concatenate()([
q2_encoded,
s2_encoded,
interaction(q2_encoded, s2_encoded),
])
s2_combined2 = Concatenate()([
s2_encoded,
q2_encoded,
interaction(q2_encoded, s2_encoded),
])
compare_layers_d = [
Dense(compare_dim, activation=activation),
Dropout(compare_dropout),
Dense(compare_out_size, activation=activation),
Dropout(compare_dropout),
]
compare_layers_g = [
Dense(compare_dim, activation=activation),
Dropout(compare_dropout),
Dense(compare_out_size, activation=activation),
Dropout(compare_dropout),
]
# NOTE these can be optimized
q1_compare1 = time_distributed(q1_combined1, compare_layers_d)
q1_compare2 = time_distributed(q1_combined2, compare_layers_d)
q1_compare = Average()([q1_compare1, q1_compare2])
q2_compare1 = time_distributed(q2_combined1, compare_layers_d)
q2_compare2 = time_distributed(q2_combined2, compare_layers_d)
q2_compare = Average()([q2_compare1, q2_compare2])
s1_compare1 = time_distributed(s1_combined1, compare_layers_g)
s1_compare2 = time_distributed(s1_combined2, compare_layers_g)
s1_compare = Average()([s1_compare1, s1_compare2])
s2_compare1 = time_distributed(s2_combined1, compare_layers_g)
s2_compare2 = time_distributed(s2_combined2, compare_layers_g)
s2_compare = Average()([s2_compare1, s2_compare2])
# Aggregate
q1_encoded = Concatenate()([q1_encoded, q1_compare, s1_compare])
q2_encoded = Concatenate()([q2_encoded, q2_compare, s2_compare])
aggreate_rnn = CuDNNGRU(rnn_size, return_sequences=True)
q1_aggreated = aggreate_rnn(q1_encoded)
q1_aggreated = Dropout(rnn_dropout)(q1_aggreated)
q2_aggreated = aggreate_rnn(q2_encoded)
q2_aggreated = Dropout(rnn_dropout)(q2_aggreated)
# Pooling
q1_rep = apply_multiple(q1_aggreated, [
GlobalAvgPool1D(),
GlobalMaxPool1D(),
])
q2_rep = apply_multiple(q2_aggreated, [
GlobalAvgPool1D(),
GlobalMaxPool1D(),
])
q_diff = Lambda(lambda x: K.abs(x[0] - x[1]))([q1_rep, q2_rep])
q_multi = Lambda(lambda x: x[0] * x[1])([q1_rep, q2_rep])
feature_input = Input(shape=(config['feature_length'], ))
feature_dense = BatchNormalization()(feature_input)
feature_dense = Dense(config['dense_dim'],
activation='relu')(feature_dense)
h_all1 = Concatenate()([q1_rep, q2_rep, q_diff, q_multi, feature_dense])
h_all2 = Concatenate()([q2_rep, q1_rep, q_diff, q_multi, feature_dense])
h_all1 = Dropout(0.5)(h_all1)
h_all2 = Dropout(0.5)(h_all2)
dense = Dense(256, activation='relu')
h_all1 = dense(h_all1)
h_all2 = dense(h_all2)
h_all = Average()([h_all1, h_all2])
predictions = Dense(1, activation='sigmoid')(h_all)
model = Model(inputs=[q1, q2, feature_input], outputs=predictions)
opt = optimizers.get(config['optimizer'])
K.set_value(opt.lr, config['learning_rate'])
model.compile(optimizer=opt, loss='binary_crossentropy', metrics=[f1])
return model
def make_model(arch):
n = 2 # 9 will give 58 weight layers (downsample separate)
x_in = Input(shape=x_train.shape[1:], name='Input')
# Convolution before first block.
x = Conv2D(16, (3, 3), padding='same', use_bias=False,
name='Conv_Start')(x_in)
x = BatchNormalization(name='BN_Start')(x)
x = Activation('relu', name='ReLU_Start')(x)
# Block A
# Make feature map depth match
x_ = Conv2D(128, (1, 1), name='Conv_1x1_A')(x)
for i in range(0, n):
# First conv
x = Conv2D(128, (3, 3),
padding='same',
use_bias=False,
name='Conv_A{}'.format(2 * i + 1))(x)
x = BatchNormalization(name='BN_A{}'.format(2 * i + 1))(x)
x = Activation('relu', name='ReLU_A{}'.format(2 * i + 1))(x)
x = Dropout(0.25, name='Dropout_A{}'.format(2 * i + 1))(x)
# Second conv
x = Conv2D(128, (3, 3),
padding='same',
use_bias=False,
name='Conv_A{}'.format(2 * i + 2))(x)
x = BatchNormalization(name='BN_A{}'.format(2 * i + 2))(x)
# Merge
x = Add(name='Merge_A{}'.format(2 * i + 2))([x, x_])
x = Activation('relu', name='ReLU_A{}'.format(2 * i + 2))(x)
x = Dropout(0.25, name='Dropout_A{}'.format(2 * i + 2))(x)
# First branch (differance) classifier
if arch != 'baseline':
y_1 = Flatten(name='Flatten_Alpha')(x)
if arch == 'digest' or arch == 'avg' or arch == 'aux':
y_1 = Dense(512, name='Dense_Alpha')(y_1)
y_1 = BatchNormalization(name='BN_Alpha')(y_1)
y_1 = Activation('relu', name='ReLU_Alpha')(y_1)
y_1 = Dropout(0.5, name='Dropout_Alpha')(y_1)
y_1 = Dense(num_classes, activation='softmax',
name='Classifier_Alpha')(y_1)
# Block B
# Downsample with convolution of stride 2
x = Conv2D(256, (3, 3),
strides=2,
padding='same',
use_bias=False,
name='Downsample_Conv_B')(x)
for i in range(0, n):
x_ = x
x = Conv2D(256, (3, 3),
padding='same',
use_bias=False,
name='Conv_B{}'.format(2 * i + 1))(x)
x = BatchNormalization(name='BN_B{}'.format(2 * i + 1))(x)
x = Activation('relu', name='ReLU_B{}'.format(2 * i + 1))(x)
x = Dropout(0.25, name='Dropout_B{}'.format(2 * i + 1))(x)
x = Conv2D(256, (3, 3),
padding='same',
use_bias=False,
name='Conv_B{}'.format(2 * i + 2))(x)
x = BatchNormalization(name='BN_B{}'.format(2 * i + 2))(x)
x = Add(name='Merge_B{}'.format(2 * i + 2))([x, x_])
x = Activation('relu', name='ReLU_B{}'.format(2 * i + 2))(x)
x = Dropout(0.25, name='Dropout_B{}'.format(2 * i + 2))(x)
# Second branch (differance) classifier
if arch != 'baseline':
y_2 = Flatten(name='Flatten_Beta')(x)
if arch == 'digest' or arch == 'avg' or arch == 'aux':
y_2 = Dense(512, name='Dense_Beta')(y_2)
y_2 = BatchNormalization(name='BN_Beta')(y_2)
y_2 = Activation('relu', name='ReLU_Beta')(y_2)
y_2 = Dropout(0.5, name='Dropout_Beta')(y_2)
y_2 = Dense(num_classes, activation='softmax',
name='Classifier_Beta')(y_2)
# Block C
# Downsample with convolution of stride 2
x = Conv2D(512, (3, 3),
strides=2,
padding='same',
use_bias=False,
name='Downsample_Conv_C')(x)
for i in range(0, n):
x_ = x
x = Conv2D(512, (3, 3),
padding='same',
use_bias=False,
name='Conv_C{}'.format(2 * i + 1))(x)
x = BatchNormalization(name='BN_C{}'.format(2 * i + 1))(x)
x = Activation('relu', name='ReLU_C{}'.format(2 * i + 1))(x)
x = Dropout(0.25, name='Dropout_C{}'.format(2 * i + 1))(x)
x = Conv2D(512, (3, 3),
padding='same',
use_bias=False,
name='Conv_C{}'.format(2 * i + 2))(x)
x = BatchNormalization(name='BN_C{}'.format(2 * i + 2))(x)
x = Add(name='Merge_C{}'.format(2 * i + 2))([x, x_])
x = Activation('relu', name='ReLU_C{}'.format(2 * i + 2))(x)
x = Dropout(0.25, name='Dropout_C{}'.format(2 * i + 2))(x)
# Top
x = GlobalAveragePooling2D(name='Global_Average')(x)
# Third branch (differance) classifier
if arch != 'baseline':
y_3 = Flatten(name='Flatten_Gamma')(x)
if arch == 'digest' or arch == 'avg':
y_3 = Dense(512, name='Dense_Gamma')(y_3)
y_3 = BatchNormalization(name='BN_Gamma')(y_3)
y_3 = Activation('relu', name='ReLU_Gamma')(y_3)
y_3 = Dropout(0.5, name='Dropout_Gamma')(y_3)
y_3 = Dense(num_classes, activation='softmax',
name='Classifier_Gamma')(y_3)
# Main classifier
if arch == 'digest':
x = Concatenate(name='Classifications_Differance')([y_1, y_2, y_3])
x = Dense(512, name='Dense_Gestalt')(x)
x = BatchNormalization(name='BN_Gestalt')(x)
x = Activation('relu', name='ReLU_Gestalt')(x)
x = Dropout(0.5, name='Dropout_Gestalt')(x)
y = Dense(num_classes, activation='softmax',
name='Classifier_Gestalt')(x)
elif arch == 'avg':
y = Average(name='Classification_Average')([y_1, y_2, y_3])
elif arch == 'aux':
x = Dense(512, name='Dense_FC')(y_3)
x = BatchNormalization(name='BN_FC')(x)
x = Activation('relu', name='ReLU_FC')(x)
x = Dropout(0.5, name='Dropout_FC')(x)
y = Dense(num_classes, activation='softmax', name='Classifier_Main')(x)
elif arch == 'skip':
x = Concatenate(name='Concat_Layer')([y_1, y_2, y_3])
x = Dense(512, name='Dense_Top')(x)
x = BatchNormalization(name='BN_Top')(x)
x = Activation('relu', name='ReLU_Top')(x)
x = Dropout(0.5, name='Dropout_Top')(x)
y = Dense(num_classes, activation='softmax', name='Classifier')(x)
elif arch == 'baseline':
#x = Flatten(name="Flatten_Top")(x)
#x = Dense(512, name='Dense_Top')(x)
#x = BatchNormalization(name='BN_Top')(x)
#x = Activation('relu', name='ReLU_Top')(x)
#x = Dropout(0.5, name='Dropout_Top')(x)
y = Dense(num_classes, activation='softmax', name='Classifier')(x)
if arch == 'digest' or arch == 'avg':
model = Model(inputs=x_in, outputs=[y, y_3, y_2, y_1])
elif arch == 'aux':
model = Model(inputs=x_in, outputs=[y, y_2, y_1])
else:
model = Model(inputs=x_in, outputs=y)
# Halve learning rate every 10 epochs
#decay = 1 / ((train_samples * augment_factor / batch_size) * 10)
optimizer = keras.optimizers.SGD(lr=0.1, momentum=0.9, nesterov=True)
return model, optimizer
def ensembleModels(models, model_input):
# taken from https://medium.com/@twt446/ensemble-and-store-models-in-keras-2-x-b881a6d7693f
yModels = [model(model_input) for model in models]
yAvg = Average()(yModels)
modelEns = Model(inputs=model_input, outputs=yAvg, name='ensemble')
return modelEns
def run(dataset_path, vocab_path, model_dir):
print('Load vocabulary')
vocab = Dictionary.load(vocab_path)
vocab_size = len(vocab.token2id)
print('vocab_size:', vocab_size)
print('Load', dataset_path)
f = h5py.File(dataset_path, 'r')
X_train = f['x_train'].value
y_train = f['y_train'].value
print('X_train.shape:', X_train.shape)
print('y_train.shape:', y_train.shape)
max_train_size = 10000000
print('Cutoff train data to %d examples' % max_train_size)
X_train = X_train[:max_train_size, :]
y_train = y_train[:max_train_size]
print('X_train.shape:', X_train.shape)
print('y_train.shape:', y_train.shape)
print('Shuffle dataset')
indices = np.arange(X_train.shape[0])
np.random.shuffle(indices)
X_train = X_train[indices]
y_train = y_train[indices]
vec_dims = 100
aggregation_type = 'concat' # 'average', `sum` or 'concat'
win_size = X_train.shape[1]
print('win_size:', win_size)
# Define input shape
inputs = Input(shape=(win_size, ), dtype='int32')
# Embedding layer maps each tokenId in the window to a `vec_dim` dim.
# vector. Output shape: (-1, win_size, vec_dim)
word_vectors = Embedding(vocab_size, vec_dims)(inputs)
# The embedding layer output is fed into `win_size` lambda layers that
# each outputs a word vector.
sliced_word_vector = [
Lambda(lambda x: x[:, i, :], output_shape=(vec_dims, ))(word_vectors)
for i in range(win_size)
]
# Aggregate the word vectors
if aggregation_type == 'concat':
h = Concatenate()(sliced_word_vector)
elif aggregation_type == 'average':
h = Average()(sliced_word_vector)
elif aggregation_type == 'sum':
h = Add()(sliced_word_vector)
else:
raise ValueError('Invalid row aggregation')
# Feed the aggregated word vectors into a dense layer that returns
# a probability for each word in the vocabulary
probs = Dense(vocab_size, activation='softmax')(h)
model = Model(inputs, probs)
# Compile the model with SGD optimizer and cross entropy loss
model.compile(optimizer=SGD(lr=0.05),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
epochs_per_fit = 1
for i in range(100):
start = current_time_ms()
h = model.fit(X_train,
y_train,
batch_size=256,
epochs=epochs_per_fit,
verbose=0)
mean_epoch_time = (current_time_ms() - start) / epochs_per_fit
epoch = i * epochs_per_fit
loss = h.history['loss'][-1]
acc = h.history['acc'][-1]
print('epoch %d: loss=%f acc=%f time=%f' %
(epoch, loss, acc, mean_epoch_time))
checkpoint_dir = os.path.join(model_dir, ts_rand())
os.makedirs(checkpoint_dir, exist_ok=True)
checkpoint_path = os.path.join(checkpoint_dir, 'w2v_model.h5')
print('Save model:', checkpoint_path)
model.save(checkpoint_path)
model1.load_weights(weight_file1)
print('Finished loading model 1.')
weight_file2 = '/content/FSA-Net/pre-trained/300W_LP_models/fsanet_var_capsule_3_16_2_21_5/fsanet_var_capsule_3_16_2_21_5.h5'
model2.load_weights(weight_file2)
print('Finished loading model 2.')
weight_file3 = '/content/FSA-Net/pre-trained/300W_LP_models/fsanet_noS_capsule_3_16_2_192_5/fsanet_noS_capsule_3_16_2_192_5.h5'
model3.load_weights(weight_file3)
print('Finished loading model 3.')
inputs = Input(shape=(64,64,3))
x1 = model1(inputs) #1x1
x2 = model2(inputs) #var
x3 = model3(inputs) #w/o
avg_model = Average()([x1,x2,x3])
model = Model(inputs=inputs, outputs=avg_model)
detector = face_detection.build_detector(
"RetinaNetResNet50", confidence_threshold=.5, nms_iou_threshold=.3)
"""
SORT: A Simple, Online and Realtime Tracker
Copyright (C) 2016 Alex Bewley [email protected]
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
def __init__(self,fsanet_model_path,option=None):
if option is None:
self.option = {
'image_size' : 64,
'num_capsule' : 3,
'dim_capsule' : 16,
'routings' : 2,
'stage_num' : [3,3,3],
'lambda_d' : 1,
'num_classes' : 3,
'image_size' : 64,
'num_primcaps' : 7*3,
'm_dim' : 5,
'ad' : 0.6
}
else:
self.option = option
self.image_size = self.option['image_size']
self.channels = 3
fsanet_model_path = fsanet_model_path
S_set = [self.option['num_capsule'],
self.option['dim_capsule'],
self.option['routings'],
self.option['num_primcaps'],
self.option['m_dim']
]
model1 = FSA_net_Capsule(self.option['image_size'], self.option['num_classes'], self.option['stage_num'], self.option['lambda_d'], S_set)()
model2 = FSA_net_Var_Capsule(self.option['image_size'], self.option['num_classes'], self.option['stage_num'], self.option['lambda_d'], S_set)()
self.option['num_primcaps'] = 8*8*3
S_set = [self.option['num_capsule'],
self.option['dim_capsule'],
self.option['routings'],
self.option['num_primcaps'],
self.option['m_dim']
]
model3 = FSA_net_noS_Capsule(self.image_size, self.option['num_classes'], self.option['stage_num'], self.option['lambda_d'], S_set)()
print('Loading models ...')
weight_file1 = fsanet_model_path[0]
model1.load_weights(weight_file1)
print('Finished loading model 1.')
weight_file2 = fsanet_model_path[1]
model2.load_weights(weight_file2)
print('Finished loading model 2.')
weight_file3 = fsanet_model_path[2]
model3.load_weights(weight_file3)
print('Finished loading model 3.')
inputs = Input(shape=(self.image_size,self.image_size,self.channels))
x1 = model1(inputs) #1x1
x2 = model2(inputs) #var
x3 = model3(inputs) #w/o
avg_model = Average()([x1,x2,x3])
self.model = Model(inputs=inputs, outputs=avg_model)
def __init__(self, num=8, img_size=1080):
self.num = num
self.img_size = img_size
self.inputs = []
self.poses = []
self.ts = []
images = [
Input(shape=(self.img_size, self.img_size, 3),
name='image_{}'.format(i)) for i in range(self.num)
]
Js = [
Input(shape=(25, 3), name='J_2d_{}'.format(i))
for i in range(self.num)
]
self.inputs.extend(images)
self.inputs.extend(Js)
pose_raw = np.load(
os.path.join(os.path.dirname(__file__),
'../assets/mean_a_pose.npy'))
pose_raw[:3] = 0.
pose = tf.reshape(
batch_rodrigues(pose_raw.reshape(-1, 3).astype(np.float32)),
(-1, ))
trans = np.array([0., 0.2, -2.3])
batch_size = tf.shape(images[0])[0]
conv2d_0 = Conv2D(8, (3, 3),
strides=(2, 2),
activation='relu',
kernel_initializer='he_normal',
trainable=False)
maxpool_0 = MaxPool2D((2, 2))
conv2d_1 = Conv2D(16, (3, 3),
activation='relu',
kernel_initializer='he_normal',
trainable=False)
maxpool_1 = MaxPool2D((2, 2))
conv2d_2 = Conv2D(32, (3, 3),
activation='relu',
kernel_initializer='he_normal',
trainable=False)
maxpool_2 = MaxPool2D((2, 2))
conv2d_3 = Conv2D(64, (3, 3),
activation='relu',
kernel_initializer='he_normal',
trainable=False)
maxpool_3 = MaxPool2D((2, 2))
conv2d_4 = Conv2D(128, (3, 3), trainable=False)
maxpool_4 = MaxPool2D((2, 2))
flat = Flatten()
self.image_features = flat
latent_code = Dense(20, name='latent_shape')
pose_trans = tf.tile(
tf.expand_dims(tf.concat((trans, pose), axis=0), 0),
(batch_size, 1))
posetrans_init = Input(tensor=pose_trans, name='posetrans_init')
self.inputs.append(posetrans_init)
J_flat = Flatten()
concat_pose = Concatenate()
latent_pose_from_I = Dense(200,
name='latent_pose_from_I',
activation='relu',
trainable=False)
latent_pose_from_J = Dense(200,
name='latent_pose_from_J',
activation='relu',
trainable=False)
latent_pose = Dense(100, name='latent_pose')
posetrans_res = Dense(24 * 3 * 3 + 3,
name='posetrans_res',
kernel_initializer=RandomNormal(stddev=0.01),
trainable=False)
posetrans = Add(name='posetrans')
dense_layers = []
for i, (J, image) in enumerate(zip(Js, images)):
conv2d_0_i = conv2d_0(image)
maxpool_0_i = maxpool_0(conv2d_0_i)
conv2d_1_i = conv2d_1(maxpool_0_i)
maxpool_1_i = maxpool_1(conv2d_1_i)
conv2d_2_i = conv2d_2(maxpool_1_i)
maxpool_2_i = maxpool_2(conv2d_2_i)
conv2d_3_i = conv2d_3(maxpool_2_i)
maxpool_3_i = maxpool_3(conv2d_3_i)
conv2d_4_i = conv2d_4(maxpool_3_i)
maxpool_4_i = maxpool_4(conv2d_4_i)
# shape
flat_i = flat(maxpool_4_i)
latent_code_i = latent_code(flat_i)
dense_layers.append(latent_code_i)
# pose
J_flat_i = J_flat(J)
latent_pose_from_I_i = latent_pose_from_I(flat_i)
latent_pose_from_J_i = latent_pose_from_J(J_flat_i)
concat_pose_i = concat_pose(
[latent_pose_from_I_i, latent_pose_from_J_i])
latent_pose_i = latent_pose(concat_pose_i)
posetrans_res_i = posetrans_res(latent_pose_i)
posetrans_i = posetrans([posetrans_res_i, posetrans_init])
self.poses.append(
Lambda(lambda x: tf.reshape(x[:, 3:], (-1, 24, 3, 3)),
name='pose_{}'.format(i))(posetrans_i))
self.ts.append(
Lambda(lambda x: x[:, :3],
name='trans_{}'.format(i))(posetrans_i))
if self.num > 1:
self.dense_merged = Average(
name='merged_latent_shape')(dense_layers)
else:
self.dense_merged = NameLayer(name='merged_latent_shape')(
dense_layers[0])
# betas
self.betas = Dense(10, name='betas',
trainable=False)(self.dense_merged)
with open(
os.path.join(os.path.dirname(__file__),
'../assets/smpl_sampling.pkl'), 'rb') as f:
sampling = pkl.load(f)
M = sampling['meshes']
U = sampling['up']
D = sampling['down']
A = sampling['adjacency']
self.faces = M[0].f.astype(np.int32)
low_res = D[-1].shape[0]
tf_U = [sparse_to_tensor(u) for u in U]
tf_A = [map(sparse_to_tensor, chebyshev_polynomials(a, 3)) for a in A]
shape_features_dense = Dense(
low_res * 64,
kernel_initializer=RandomNormal(stddev=0.003),
name='shape_features_flat')(self.dense_merged)
shape_features = Reshape((low_res, 64),
name="shape_features")(shape_features_dense)
conv_l3 = GraphConvolution(32,
tf_A[3],
activation='relu',
name='conv_l3',
trainable=False)(shape_features)
unpool_l2 = Lambda(lambda v: sparse_dot_adj_batch(tf_U[2], v),
name='unpool_l2')(conv_l3)
conv_l2 = GraphConvolution(16,
tf_A[2],
activation='relu',
name='conv_l2',
trainable=False)(unpool_l2)
unpool_l1 = Lambda(lambda v: sparse_dot_adj_batch(tf_U[1], v),
name='unpool_l1')(conv_l2)
conv_l1 = GraphConvolution(16,
tf_A[1],
activation='relu',
name='conv_l1',
trainable=False)(unpool_l1)
unpool_l0 = Lambda(lambda v: sparse_dot_adj_batch(tf_U[0], v),
name='unpool_l0')(conv_l1)
conv_l0 = GraphConvolution(3,
tf_A[0],
activation='tanh',
name='offsets_pre')(unpool_l0)
self.offsets = Lambda(lambda x: x / 10., name='offsets')(conv_l0)
smpl = SmplTPoseLayer(theta_in_rodrigues=False,
theta_is_perfect_rotmtx=False)
smpls = [
NameLayer('smpl_{}'.format(i))(smpl(
[p, self.betas, t, self.offsets]))
for i, (p, t) in enumerate(zip(self.poses, self.ts))
]
self.vertices = [
Lambda(lambda s: s[0], name='vertices_{}'.format(i))(smpl)
for i, smpl in enumerate(smpls)
]
# we only need one instance per batch for laplace
self.vertices_tposed = Lambda(lambda s: s[1],
name='vertices_tposed')(smpls[0])
vertices_naked = Lambda(lambda s: s[2],
name='vertices_naked')(smpls[0])
self.laplacian = Lambda(
lambda (v0, v1): compute_laplacian_diff(v0, v1, self.faces),
name='laplacian')([self.vertices_tposed, vertices_naked])
self.symmetry = NameLayer('symmetry')(self.vertices_tposed)
l = SmplBody25FaceLayer(theta_in_rodrigues=False,
theta_is_perfect_rotmtx=False)
kps = [
NameLayer('kps_{}'.format(i))(l([p, self.betas, t]))
for i, (p, t) in enumerate(zip(self.poses, self.ts))
]
self.Js = [
Lambda(lambda jj: jj[:, :25], name='J_reproj_{}'.format(i))(j)
for i, j in enumerate(kps)
]
self.face_kps = [
Lambda(lambda jj: jj[:, 25:], name='face_reproj_{}'.format(i))(j)
for i, j in enumerate(kps)
]
self.repr_loss = reprojection([self.img_size, self.img_size],
[self.img_size / 2., self.img_size / 2.],
self.img_size, self.img_size)
renderer = RenderLayer(self.img_size,
self.img_size,
1,
np.ones((6890, 1)),
np.zeros(1),
self.faces, [self.img_size, self.img_size],
[self.img_size / 2., self.img_size / 2.],
name='render_layer')
self.rendered = [
NameLayer('rendered_{}'.format(i))(renderer(v))
for i, v in enumerate(self.vertices)
]
self.inference_model = Model(
inputs=self.inputs,
outputs=[self.vertices_tposed] + self.vertices +
[self.betas, self.offsets] + self.poses + self.ts)
self.opt_pose_model = Model(inputs=self.inputs, outputs=self.Js)
opt_pose_loss = {
'J_reproj_{}'.format(i): self.repr_loss
for i in range(self.num)
}
self.opt_pose_model.compile(loss=opt_pose_loss, optimizer='adam')
self.opt_shape_model = Model(inputs=self.inputs,
outputs=self.Js + self.face_kps +
self.rendered +
[self.symmetry, self.laplacian])
opt_shape_loss = {
'laplacian': laplace_mse,
'symmetry': symmetry_mse,
}
opt_shape_weights = {
'laplacian': 100. * self.num,
'symmetry': 50. * self.num,
}
for i in range(self.num):
opt_shape_loss['rendered_{}'.format(i)] = 'mse'
opt_shape_weights['rendered_{}'.format(i)] = 1.
opt_shape_loss['J_reproj_{}'.format(i)] = self.repr_loss
opt_shape_weights['J_reproj_{}'.format(i)] = 50.
opt_shape_loss['face_reproj_{}'.format(i)] = self.repr_loss
opt_shape_weights['face_reproj_{}'.format(i)] = 10. * self.num
self.opt_shape_model.compile(loss=opt_shape_loss,
loss_weights=opt_shape_weights,
optimizer='adam')
def __init__(self,
dim,
encode_dim,
num_clones,
initial_growth=0.10,
shrink=0.75,
**kwargs):
self.layer_dims = []
self.encoded_layer = 0
encode_data = Input(shape=(dim, ))
decode_data = Input(shape=(encode_dim, ))
################################################
###build the encoder and decoder architecture###
################################################
#initial expansion layer
start_dim = int(dim * (1.0 + initial_growth))
encoded = Dense(start_dim, **kwargs)(encode_data)
self.layer_dims.append(start_dim)
self.encoded_layer += 1
#compression layers
current_dim = int(start_dim * shrink)
while current_dim > encode_dim:
encoded = Dense(current_dim, **kwargs)(encoded)
self.layer_dims.append(current_dim)
self.encoded_layer += 1
current_dim = int(current_dim * shrink)
#final encoding layer
#encoded = Dense(encode_dim, **kwargs)(encoded)
encoded = Dense(encode_dim, activation='linear')(encoded)
self.layer_dims.append(encode_dim)
self.encoded_layer += 1
#first expansion layer
reversed_dims = self.layer_dims[::-1][1:]
decoded = Dense(reversed_dims[0], **kwargs)(decode_data)
self.layer_dims.append(reversed_dims[0])
self.encoded_layer += 1
#remaining expansion layers
for current_dim in reversed_dims[1:]:
decoded = Dense(current_dim, **kwargs)(decoded)
self.layer_dims.append(current_dim)
#final linear layer
decoded = Dense(dim, activation='linear')(decoded)
self.layer_dims.append(dim)
#create encoder and decoder models
self.encoder = Model(encode_data, encoded)
self.decoder = Model(decode_data, decoded)
#create the joint model
_encoded_ = self.encoder(encode_data)
_decoded_ = self.decoder(_encoded_)
self.autoencoder = Model(encode_data, _decoded_)
#########################################
###build the coupled autoencoder model###
#########################################
#################
###the encoder###
#################
#read in joint data and split into the columns
joint_encode_data = Input(shape=(dim * num_clones, ))
split_encode_data = []
for i in range(num_clones):
split_encode_data.append(
Lambda(lambda x: x[:, i * dim:(i + 1) * dim],
output_shape=(dim, ))(joint_encode_data))
#output from tied encoders
split_encoded = [self.encoder(input) for input in split_encode_data]
#average the output from each encoder
if num_clones == 1:
average_encoded = split_encoded[0]
else:
average_encoded = Average()(split_encoded)
#create the full encoder
self.joint_encoder = Model(joint_encode_data, average_encoded)
#################
###the decoder###
#################
average_encoded = Input(shape=(encode_dim, ))
#spit out the results and concatenate
split_decoded = [
self.decoder(average_encoded) for i in range(num_clones)
]
if num_clones == 1:
joint_decode_data = split_decoded[0]
else:
joint_decode_data = concatenate(split_decoded, axis=-1)
#final model
self.joint_decoder = Model(average_encoded, joint_decode_data)
#####################
###the autoencoder###
#####################
encoded = self.joint_encoder(joint_encode_data)
joint_decode_data = self.joint_decoder(encoded)
self.joint_autoencoder = Model(joint_encode_data, joint_decode_data)
return None
def make_preds(X_test, y_test):
img_rows, img_cols = 28, 28
f = 0
input_a = Input(shape=(img_rows, img_cols, 1))
input_b = Input(shape=(img_rows, img_cols, 1))
input_c = Input(shape=(img_rows, img_cols, 1))
# because we re-use the same instance `base_network`,
# the weights of the network
# will be shared across the two branches
#vec_a = create_base_network2(input_a)
#vec_b = create_base_network2(input_b)
#vec_c = create_base_network2(input_c)
cnet = convnet((img_rows, img_cols, 1))
vec_a = cnet(input_a)
vec_b = cnet(input_b)
vec_c = cnet(input_c)
#vec_a = Activation('selu')(vec_a)
#vec_b = Activation('selu')(vec_b)
#vec_c = Activation('selu')(vec_c)
#vec_a2 = GaussianNoise(50.0)(vec_a)
#vec_b2 = GaussianNoise(50.0)(vec_b)
#vec_c2 = GaussianNoise(50.0)(vec_c)
#vec_a2 = Lambda(lambda v: K.tanh(v[0] - v[1]))([vec_b, vec_c])
#vec_b2 = Lambda(lambda v: K.tanh(v[0] - v[1]))([vec_a, vec_c])
#vec_c2 = Lambda(lambda v: K.tanh(v[0] - v[1]))([vec_a, vec_b])
#vec_a2 = GaussianNoise(0.01)(vec_a2)
#vec_b2 = GaussianNoise(0.01)(vec_b2)
#vec_c2 = GaussianNoise(0.01)(vec_c2)
#vec_a2 = Lambda(lambda v: K.clip(v, -1.0, 1.0))(vec_a2)
#vec_b2 = Lambda(lambda v: K.clip(v, -1.0, 1.0))(vec_b2)
#vec_c2 = Lambda(lambda v: K.clip(v, -1.0, 1.0))(vec_c2)
#vec_a3 = Dot(axes=-1, normalize=False)([vec_b, vec_c])
#vec_b3 = Dot(axes=-1, normalize=False)([vec_c, vec_a])
#vec_c3 = Dot(axes=-1, normalize=False)([vec_a, vec_b])
#x = Lambda(lambda v : K.stack([K.abs(v[1]-v[2]), K.abs(v[2]-v[0]), K.abs(v[0]-v[1])], axis=-1))([vec_a, vec_b, vec_c])
#x = Lambda(lambda v : K.stack([K.mean(K.square(v[1]-v[2]), axis=-1), K.mean(K.square(v[2]-v[0]), axis=-1), K.mean(K.square(v[0]-v[1]), axis=-1)], axis=-1))([vec_a, vec_b, vec_c])
#x = Lambda(lambda v : K.stack([K.log(K.epsilon()+K.mean(K.square(v[1]-v[2]), axis=-1)), K.log(K.mean(K.epsilon()+K.square(v[2]-v[0]), axis=-1)), K.log(K.mean(K.epsilon()+K.square(v[0]-v[1]), axis=-1))], axis=-1))([vec_a, vec_b, vec_c])
#x = Lambda(lambda v : K.stack([K.mean(K.square(v[1]-v[2]), axis=-1), K.mean(K.square(v[2]-v[0]), axis=-1), K.mean(K.square(v[0]-v[1]), axis=-1)], axis=-1))([vec_a, vec_b, vec_c])
#x = Lambda(lambda v : K.stack([v[1]*v[2], v[2]*v[0], v[0]*v[1]], axis=-1))([vec_a, vec_b, vec_c])
#x = Lambda(lambda v : K.stack(v, axis=-1))([vec_a3, vec_b3, vec_c3])
#x = Activation('selu')(x)
#x = TimeDistributed(Dense(3, activation='selu'))(x)
#x = TimeDistributed(Dense(3, activation='sigmoid'))(x)
#x = Flatten()(x)
#x = Dense(3*nb_filters, activation='selu')(x)
#x = Dense(nb_filters, activation='selu')(x)
#x = Convolution1D(2*nb_filters, 3, strides=1, activation='selu', padding='same')(x)
#x = Convolution1D(2*nb_filters, 3, strides=2, activation='selu', padding='same')(x)
#x = Convolution1D(3, 1, strides=1, activation='sigmoid', padding='same')(x)
#x = GlobalAveragePooling1D()(x)
#probs = Activation('softmax')(x)
#x = Convolution1D(int(nb_filters/2), 2, strides=2, activation='selu', padding='same')(x)
#x = Reshape((-1,1))(x)
#x = TimeDistributed(Dense(nb_filters, activation='selu'))(x)
#x = TimeDistributed(Dense(nb_filters, activation='selu'))(x)
#x = TimeDistributed(Dense(1, activation='sigmoid'))(x)
#x = Flatten()(x)
#probs = Lambda(lambda x : x/(K.repeat_elements(K.expand_dims(K.sum(x, axis=-1)), 3, axis=-1)+K.epsilon()))(x)
#x = Reshape((-1,1))(x)
#x = TimeDistributed(Dense(1, activation='sigmoid'))(x)
#x = Flatten()(x)
#x = Dense(int(nb_filters/2), activation='selu')(x)
#x = Dense(int(nb_filters/2), activation='selu')(x)
#probs = Dense(3, activation='softmax')(x)
#input_u = Input(shape=(3,))
#y = Dense(int(nb_filters/2), activation='selu')(input_u)
#y = Dense(int(nb_filters/2), activation='selu')(y)
#p = Dense(3, activation='softmax')(y)
#unit = Model(inputs=input_u, outputs=p)
#probs = unit(x)
#x = Reshape((1,-1))(x)
#x = Convolution1D(2*nb_filters, 1, strides=1, activation='selu', padding='same')(x)
#x = Convolution1D(2*nb_filters, 1, strides=1, activation='selu', padding='same')(x)
#x = Convolution1D(3, 1, strides=1, activation='softmax', padding='same')(x)
#probs = Flatten()(x)
#probs = Lambda(lambda x : x/(K.repeat_elements(K.expand_dims(K.sum(x, axis=-1)), 3, axis=-1)+K.epsilon()))(x)
#probs = Lambda(lambda x : 1.0-x/(K.repeat_elements(K.expand_dims(K.sum(x, axis=-1)), 3, axis=-1)+K.epsilon()))(x)
input_u = Input(shape=(3, ))
y = Dense(6, activation='selu')(input_u)
y = Dense(6, activation='selu')(y)
p = Dense(3, activation='softmax')(y)
ntop = int(nb_filters / 2)
unit = Model(inputs=input_u, outputs=p)
x0 = Lambda(lambda v: K.stack([
K.mean(K.square(v[1] - v[2]), axis=-1),
K.mean(K.square(v[2] - v[0]), axis=-1),
K.mean(K.square(v[0] - v[1]), axis=-1)
],
axis=-1))([vec_a, vec_b, vec_c])
x1 = Lambda(lambda v: K.stack([
K.mean(K.square(v[2] - v[0]), axis=-1),
K.mean(K.square(v[0] - v[1]), axis=-1),
K.mean(K.square(v[1] - v[2]), axis=-1)
],
axis=-1))([vec_a, vec_b, vec_c])
x2 = Lambda(lambda v: K.stack([
K.mean(K.square(v[0] - v[1]), axis=-1),
K.mean(K.square(v[1] - v[2]), axis=-1),
K.mean(K.square(v[2] - v[0]), axis=-1)
],
axis=-1))([vec_a, vec_b, vec_c])
#x0 = Lambda(lambda v : K.stack([K.mean(tf.nn.top_k(K.square(v[1]-v[2]), k=ntop)[0], axis=-1), K.mean(tf.nn.top_k(K.square(v[2]-v[0]), k=ntop)[0], axis=-1), K.mean(tf.nn.top_k(K.square(v[0]-v[1]), k=ntop)[0], axis=-1)], axis=-1))([vec_a, vec_b, vec_c])
#x1 = Lambda(lambda v : K.stack([K.mean(tf.nn.top_k(K.square(v[2]-v[0]), k=ntop)[0], axis=-1), K.mean(tf.nn.top_k(K.square(v[0]-v[1]), k=ntop)[0], axis=-1), K.mean(tf.nn.top_k(K.square(v[1]-v[2]), k=ntop)[0], axis=-1)], axis=-1))([vec_a, vec_b, vec_c])
#x2 = Lambda(lambda v : K.stack([K.mean(tf.nn.top_k(K.square(v[0]-v[1]), k=ntop)[0], axis=-1), K.mean(tf.nn.top_k(K.square(v[1]-v[2]), k=ntop)[0], axis=-1), K.mean(tf.nn.top_k(K.square(v[2]-v[0]), k=ntop)[0], axis=-1)], axis=-1))([vec_a, vec_b, vec_c])
x0 = unit(x0)
x1 = unit(x1)
x2 = unit(x2)
x1 = Lambda(lambda v: K.stack([v[:, 2], v[:, 0], v[:, 1]], axis=-1))(x1)
x2 = Lambda(lambda v: K.stack([v[:, 1], v[:, 2], v[:, 0]], axis=-1))(x2)
probs = Average()([x0, x1, x2])
#x = Reshape((-1,1))(x)
#x = TimeDistributed(Dense(nb_filters, activation='selu'))(x)
#x = TimeDistributed(Dense(nb_filters, activation='selu'))(x)
#x = TimeDistributed(Dense(1, activation='linear'))(x)
#x = Flatten()(x)
#probs = Lambda(lambda y : K.exp(-y)/(K.repeat_elements(K.expand_dims(K.epsilon() + K.sum(K.exp(-y), axis=-1)), 3, axis=-1)))(x)
model = Model(inputs=[input_a, input_b, input_c], outputs=probs)
optimizer = Adam(lr=0.0003) #RMSprop()
model.compile(loss=pick_best_loss,
optimizer=optimizer,
metrics=['accuracy'])
model.load_weights('autoenc' + str(f) + '.h5')
#model = [model.layers[0], model.layers[-2]]
batch_size = 256
test_gen = create_triplets(X_test, None, 1.0, batch_size, mode='val')
print('log loss, accuracy')
print(model.evaluate_generator(test_gen, steps=20))
input = Input(shape=(img_rows, img_cols, 1))
output = model.layers[3](input)
model2 = Model(inputs=input, outputs=output)
vecs = model2.predict(1.0 * X_test)
print(np.std(vecs, axis=0))
pca = PCA(n_components=3, svd_solver='arpack', copy=True, whiten=True)
pca.fit(vecs[:, :])
pca_vecs = pca.transform(vecs[:, :])
kmeans_dims = 11
clf = KMeans(n_clusters=kmeans_dims, random_state=0,
max_iter=1000).fit(vecs)
preds = clf.predict(vecs)
#preds = DBSCAN(eps=5.0).fit_predict(vecs)
#preds = MeanShift().fit_predict(pca_vecs)
#preds = KMeans(n_clusters=kmeans_dims, random_state=0).fit_predict(vecs)
#gmm = BayesianGaussianMixture(n_components=4).fit(pca_vecs)
#preds = gmm.predict(pca_vecs)
#preds = AgglomerativeClustering(n_clusters=kmeans_dims, linkage='average', affinity='euclidean').fit_predict(vecs)
print(len(set(preds)))
print('0')
idx = np.where(y_test == 0)
print(preds[idx][0:40])
print('1')
idx = np.where(y_test == 1)
print(preds[idx][0:40])
print('2')
idx = np.where(y_test == 2)
print(preds[idx][0:40])
print('3')
idx = np.where(y_test == 3)
print(preds[idx][0:40])
print('4')
idx = np.where(y_test == 4)
print(preds[idx][0:40])
print('5')
idx = np.where(y_test == 5)
print(preds[idx][0:40])
print('6')
idx = np.where(y_test == 6)
print(preds[idx][0:40])
print('7')
idx = np.where(y_test == 7)
print(preds[idx][0:40])
print('8')
idx = np.where(y_test == 8)
print(preds[idx][0:40])
print('9')
idx = np.where(y_test == 9)
print(preds[idx][0:40])
plt.close('all')
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(xs=pca_vecs[y_test == 1][:, 0],
ys=pca_vecs[y_test == 1][:, 1],
zs=pca_vecs[y_test == 1][:, 2],
zdir='z',
s=5,
c='k',
depthshade=True,
label='1')
ax.scatter(xs=pca_vecs[y_test == 7][:, 0],
ys=pca_vecs[y_test == 7][:, 1],
zs=pca_vecs[y_test == 7][:, 2],
zdir='z',
s=5,
c='r',
depthshade=True,
label='7')
ax.scatter(xs=pca_vecs[y_test == 4][:, 0],
ys=pca_vecs[y_test == 4][:, 1],
zs=pca_vecs[y_test == 4][:, 2],
zdir='z',
s=5,
c='b',
depthshade=True,
label='4')
ax.scatter(xs=pca_vecs[y_test == 9][:, 0],
ys=pca_vecs[y_test == 9][:, 1],
zs=pca_vecs[y_test == 9][:, 2],
zdir='z',
s=5,
c='g',
depthshade=True,
label='9')
ax.scatter(xs=pca_vecs[y_test == 0][:, 0],
ys=pca_vecs[y_test == 0][:, 1],
zs=pca_vecs[y_test == 0][:, 2],
zdir='z',
s=5,
c='m',
depthshade=True,
label='0')
plt.legend()
'''
count = count + 1
modelVGG02_all.get_layer(name='batch_normalization_1_input').name='VGG02_all_norm'
count = 0
for layer in modelVGG_CNN.layers:
layer.name='VGG_CNN_'+ str(count)
count = count + 1
modelVGG_CNN.get_layer(name='batch_normalization_1_input').name='VGG_CNN_norm'
# join the models with an average layer
#inputs = Input(shape=(timesteps, data_dim))
X_1 = modelVGG02_all.output
X_2 = modelVGG.output
X_3 = modelVGG02.output
X_4 = modelVGG_CNN.output
out = Average()([X_1, X_2, X_3, X_4])
ensemble_VGG = Model(inputs=[modelVGG02_all.input, modelVGG.input, modelVGG02.input, modelVGG_CNN.input], outputs=out)
ensemble_VGG.compile(loss='binary_crossentropy',
optimizer='Adam',
metrics=['binary_accuracy', f1])
# evaluate on the mock test set
X, Y, Fscore = ensemble_VGG.evaluate([X_test, X_test, X_test, X_test], Y_test)
print(X, Y, Fscore)
# save the ensemble model
# save the model to the cloud storage bucket
filename = 'ensemble_Audio'
path = 'gs://mediaeval_data_storage/ensembles'
def test_delete_channels_merge_others(channel_index, data_format):
layer_test_helper_merge_2d(Add(), channel_index, data_format)
layer_test_helper_merge_2d(Multiply(), channel_index, data_format)
layer_test_helper_merge_2d(Average(), channel_index, data_format)
layer_test_helper_merge_2d(Maximum(), channel_index, data_format)
weights=[trans_embedding_matrix_fr],
input_length=max_words,
trainable=False)(trans_input)
e = Embedding(vocab_size,
300,
weights=[embedding_matrix],
input_length=max_words,
trainable=False)(a)
f = Embedding(vocab_size_fr,
300,
weights=[embedding_matrix_fr],
input_length=max_words,
trainable=False)(b)
merge = (Average()([e, f, ef]))
x = merge
x = Conv1D(filters=64, kernel_size=3, padding='valid', activation='relu')(x)
x = MaxPooling1D(pool_size=5)(x)
x = LSTM(80, dropout=0.5)(x)
x = concatenate([x, meta_input])
x = Dense(32, activation="relu")(x)
x = Dropout(0.50)(x)
x = Dense(16, activation="relu")(x)
x = Dropout(0.50)(x)
x = Dense(4, activation="relu")(x)
x = Dropout(0.50)(x)
x = Dense(2, activation="relu")(x)
def get_extract_model():
"""
构建事件抽取模型结构，加载模型参数，返回模型对象
1、使用bert输出预测动词下标
2、使用bert输出融合动词下标预测事件时间、地点、主语、宾语、否定词
:return: 各个部分的模型对象
"""
with extract_sess.as_default():
with extract_sess.graph.as_default():
# 构建bert模型主体
bert_model = build_transformer_model(
config_path=bert_config.config_path,
return_keras_model=False,
model=bert_config.model_type
)
# 搭建模型
# 动词输入
trigger_start_in = Input(shape=(None,))
trigger_end_in = Input(shape=(None,))
# 动词下标输入
trigger_index_start_in = Input(shape=(1,))
trigger_index_end_in = Input(shape=(1,))
# 宾语输入
object_start_in = Input(shape=(None,))
object_end_in = Input(shape=(None,))
# 主语输入
subject_start_in = Input(shape=(None,))
subject_end_in = Input(shape=(None,))
# 地点输入
loc_start_in = Input(shape=(None,))
loc_end_in = Input(shape=(None,))
# 时间输入
time_start_in = Input(shape=(None,))
time_end_in = Input(shape=(None,))
# 否定词输入
negative_start_in = Input(shape=(None,))
negative_end_in = Input(shape=(None,))
# 将模型外传入的下标赋值给模型内部变量(只是为了将模型中应用与构建Model的输入区分开来)
trigger_index_start, trigger_index_end = trigger_index_start_in, trigger_index_end_in
trigger_start_out = Dense(1, activation='sigmoid')(bert_model.model.output)
trigger_end_out = Dense(1, activation='sigmoid')(bert_model.model.output)
# 预测trigger动词的模型
trigger_model = Model(bert_model.model.inputs, [trigger_start_out, trigger_end_out])
# 按照动词下标采集字向量
k1v = Lambda(seq_gather)([bert_model.model.output, trigger_index_start])
k2v = Lambda(seq_gather)([bert_model.model.output, trigger_index_end])
kv = Average()([k1v, k2v])
# 使用归一化融合动词词向量与句子张量
t = LayerNormalization(conditional=True)([bert_model.model.output, kv])
# 宾语模型输出
object_start_out = Dense(1, activation='sigmoid')(t)
object_end_out = Dense(1, activation='sigmoid')(t)
# 主语模型输出
subject_start_out = Dense(1, activation='sigmoid')(t)
subject_end_out = Dense(1, activation='sigmoid')(t)
# 地点模型输出
loc_start_out = Dense(1, activation='sigmoid')(t)
loc_end_out = Dense(1, activation='sigmoid')(t)
# 时间模型输出
time_start_out = Dense(1, activation='sigmoid')(t)
time_end_out = Dense(1, activation='sigmoid')(t)
# 否定词模型输出
negative_start_out = Dense(1, activation='sigmoid')(t)
negative_end_out = Dense(1, activation='sigmoid')(t)
# 输入text和trigger，预测object
object_model = Model(bert_model.model.inputs + [trigger_index_start_in, trigger_index_end_in],
[object_start_out, object_end_out])
# 输入text和trigger，预测subject
subject_model = Model(bert_model.model.inputs + [trigger_index_start_in, trigger_index_end_in],
[subject_start_out, subject_end_out])
# 输入text和trigger，预测loc
loc_model = Model(bert_model.model.inputs + [trigger_index_start_in, trigger_index_end_in],
[loc_start_out, loc_end_out])
# 输入text和trigger，预测time
time_model = Model(bert_model.model.inputs + [trigger_index_start_in, trigger_index_end_in],
[time_start_out, time_end_out])
# 输入text和trigger，预测否定词negative
negative_model = Model(bert_model.model.inputs + [trigger_index_start_in, trigger_index_end_in],
[negative_start_out, negative_end_out])
# 主模型
train_model = Model(
bert_model.model.inputs + [trigger_start_in, trigger_end_in, trigger_index_start_in, trigger_index_end_in,
object_start_in, object_end_in, subject_start_in, subject_end_in, loc_start_in,
loc_end_in, time_start_in, time_end_in, negative_start_in, negative_end_in],
[trigger_start_out, trigger_end_out, object_start_out, object_end_out, subject_start_out, subject_end_out,
loc_start_out, loc_end_out, time_start_out, time_end_out, negative_start_out, negative_end_out])
# 加载事件抽取模型参数
logger.info("开始加载事件抽取模型参数。。。")
train_model.load_weights(pre_config.event_extract_model_path)
logger.info("事件抽取模型参数加载完成！")
return trigger_model, object_model, subject_model, loc_model, time_model, negative_model
def create_layer(self):
if self.type == "Dense":
self.block = Dense(units=int(self.gff("units")), activation=self.gff("activation"))
elif self.type == "Input":
self.block = Input(shape=self.get_int(self.gff("shape")))
elif self.type == "Dropout":
self.block = Dropout(rate=float(self.gff("rate")))
elif self.type == "Flatten":
self.block = Flatten()
elif self.type == "Reshape":
self.block = Reshape(target_shape=int(self.gff("rate")))
elif self.type == "Permute":
self.block = Permute(dims=self.get_int(self.gff("dims")))
elif self.type == "RepeatVector":
self.block = RepeatVector(n=int(self.gff("dims")))
#CONVOLUTIONAL
elif self.type == "Conv1D":
self.block = Conv1D(filters=int(self.gff("filters")), kernel_size=self.get_int(self.gff("kernel_size")),
activation=self.get_act(self.gff("activation")), strides=self.get_int(self.gff("strides")),
padding=self.gff("padding"))
elif self.type == "Conv2D":
self.block = Conv2D(filters=int(self.gff("filters")), kernel_size=self.get_int(self.gff("kernel_size")),
activation=self.get_act(self.gff("activation")), strides=self.get_int(self.gff("strides")),
padding=self.gff("padding"))
elif self.type == "Conv2DTranspose":
self.block = Conv2DTranspose(filters=int(self.gff("filters")), kernel_size=self.get_int(self.gff("kernel_size")),
activation=self.get_act(self.gff("activation")), strides=self.get_int(self.gff("strides")),
padding=self.gff("padding"))
elif self.type == "Conv3D":
self.block = Conv3D(filters=int(self.gff("filters")), kernel_size=self.get_int(self.gff("kernel_size")),
activation=self.get_act(self.gff("activation")), strides=self.get_int(self.gff("strides")),
padding=self.gff("padding"))
elif self.type == "UpSampling1D":
self.block = UpSampling1D(size=self.get_int(self.gff("size")))
elif self.type == "UpSampling2D":
self.block = UpSampling2D(size=self.get_int(self.gff("size")))
elif self.type == "UpSampling3D":
self.block = UpSampling3D(size=self.get_int(self.gff("size")))
#POOLING
elif self.type == "MaxPooling1D":
self.block = MaxPooling1D(pool_size=self.get_int(self.gff("pool_size")),
strides=self.get_int(self.gff("strides")), padding=self.gff("padding"))
elif self.type == "MaxPooling2D":
self.block = MaxPooling2D(pool_size=self.get_int(self.gff("pool_size")),
strides=self.get_int(self.gff("strides")), padding=self.gff("padding"))
elif self.type == "MaxPooling3D":
self.block = MaxPooling3D(pool_size=self.get_int(self.gff("pool_size")),
strides=self.get_int(self.gff("strides")), padding=self.gff("padding"))
elif self.type == "AveragePooling1D":
self.block = AveragePooling1D(pool_size=self.get_int(self.gff("pool_size")),
strides=self.get_int(self.gff("strides")), padding=self.gff("padding"))
elif self.type == "AveragePooling2D":
self.block = AveragePooling2D(pool_size=self.get_int(self.gff("pool_size")),
strides=self.get_int(self.gff("strides")), padding=self.gff("padding"))
elif self.type == "AveragePooling3D":
self.block = AveragePooling3D(pool_size=self.get_int(self.gff("pool_size")),
strides=self.get_int(self.gff("strides")), padding=self.gff("padding"))
elif self.type == "GlobalMaxPooling1D":
self.block = GlobalMaxPooling1D()
elif self.type == "GlobalMaxPooling2D":
self.block = GlobalMaxPooling2D()
elif self.type == "GlobalAveragePooling1D":
self.block = GlobalAveragePooling1D()
elif self.type == "GlobalAveragePooling2D":
self.block = GlobalAveragePooling2D()
#Locally Connected
elif self.type == "LocallyConnected1D":
self.block = LocallyConnected1D(filters=int(self.gff("filters")), kernel_size=self.get_int(self.gff("kernel_size")),
activation=self.get_act(self.gff("activation")), strides=self.get_int(self.gff("strides")),
padding=self.gff("padding"))
elif self.type == "LocallyConnected2D":
self.block = LocallyConnected2D(filters=int(self.gff("filters")), kernel_size=self.get_int(self.gff("kernel_size")),
activation=self.get_act(self.gff("activation")), strides=self.get_int(self.gff("strides")),
padding=self.gff("padding"))
#MERGE LAYERS
elif self.type == "Add":
self.block = Add()
elif self.type == "Subtract":
self.block = Subtract()
elif self.type == "Multiply":
self.block = Multiply()
elif self.type == "Average":
self.block = Average()
elif self.type == "Maximum":
self.block = Maximum()
elif self.type == "Concatenate":
self.block = Concatenate()
elif self.type == "Dot":
self.block = Dot()
#NORMALISATION LAYER
elif self.type == "BatchNormalization":
self.block = BatchNormalization(axis=int(self.gff("axis")), center=bool(self.gff("center")),
momentum=float(self.gff("momentum")), epsilon=float(self.gff("epsilon")),
scale=bool(self.gff("scale")))
#NOISE LAYERS
elif self.type == "GaussianNoise":
self.block = GaussianNoise(stddev=self.get_float(self.gff("stddev")))
#NOISE LAYERS
elif self.type == "GaussianDropout":
self.block = GaussianDropout(rate=self.get_float(self.gff("rate")))
elif self.type == "AlphaDropout":
self.block = AlphaDropout(rate=self.get_float(self.gff("rate")), seed=int(self.gff("rate")))
# model
X_input = Input(shape=(X.shape[1], X.shape[2]))
graph_conv_filters_input = Input(shape=(graph_conv_filters.shape[1],
graph_conv_filters.shape[2]))
layer_gcnn1 = MultiGraphCNN(
8, num_filters, activation='elu')([X_input, graph_conv_filters_input])
# layer_gcnn1 = Dropout(0.2)(layer_gcnn1)
layer_gcnn2 = MultiGraphCNN(
8, num_filters, activation='elu')([layer_gcnn1, graph_conv_filters_input])
# layer_gcnn2 = Dropout(0.2)(layer_gcnn2)
layer_gcnn3 = MultiGraphCNN(
8, num_filters, activation='elu')([layer_gcnn2, graph_conv_filters_input])
# layer_gcnn3 = Dropout(0.2)(layer_gcnn3)
layer_gcnn4 = Average()([layer_gcnn1, layer_gcnn2, layer_gcnn3])
# add new Graph layer with cnn
layer_gcnn4 = MultiGraphCNN(
1, num_filters, activation='elu')([layer_gcnn4, graph_conv_filters_input])
# layer_gcnn3 = Dropout(0.2)(layer_gcnn3)
# layer_gcnn5 = Reshape((layer_gcnn4.shape[1]*layer_gcnn4.shape[2],))(layer_gcnn4)
layer_gcnn5 = Flatten()(layer_gcnn4)
# layer_gcnn5 = Dropout(0.2)(layer_gcnn5)
# # layer_conv5 = AveragePooling2D(pool_size=(2, 1), strides=None, padding='valid', data_format=None)(layer_conv5)
layer_dense1 = Dense(V, activation='linear')(layer_gcnn5)
model = Model(inputs=[X_input, graph_conv_filters_input], outputs=layer_dense1)
model.summary()
# model = cnn_tfmodel.egrmodel2(A, X, graph_conv_filters,2)
gru_kata = Bidirectional(GRU(EMBEDDING_DIM * 3,
return_sequences=True,
dropout=dropout_gru,
recurrent_dropout=rec_dropout),
merge_mode=merge_m,
weights=None)(merge)
else:
combine = input(
'Enter 1 for Add, 2 for Subtract, 3 for Multiply, 4 for Average, '
'5 for Maximum, 6 for Concatenate: ')
if combine == 2:
merge = Subtract()([dropout, rtwo])
elif combine == 3:
merge = Multiply()([dropout, rtwo])
elif combine == 4:
merge = Average()([dropout, rtwo])
elif combine == 5:
merge = Maximum()([dropout, rtwo])
elif combine == 6:
merge = Concatenate()([dropout, rtwo])
else:
merge = Add()([dropout, rtwo])
if combine == 6:
gru_kata = Bidirectional(GRU(EMBEDDING_DIM * 2,
return_sequences=True,
dropout=dropout_gru,
recurrent_dropout=rec_dropout),
merge_mode=merge_m,
weights=None)(merge)
else:
gru_kata = Bidirectional(GRU(EMBEDDING_DIM,
def safe_average(list_inputs):
if len(list_inputs) == 1:
return list_inputs[0]
return Average()(list_inputs)
def main():
try:
os.mkdir('./img')
except OSError:
pass
# face_cascade = cv2.CascadeClassifier('lbpcascade_frontalface_improved.xml')
# detector = MTCNN()
# load model and weights
img_size = 64
stage_num = [3, 3, 3]
lambda_local = 1
lambda_d = 1
img_idx = 0
detected = '' #make this not local variable
time_detection = 0
time_network = 0
time_plot = 0
skip_frame = 1 # 每5帧做1次检测和网络前向传播
ad = 0.6
#Parameters
num_capsule = 3
dim_capsule = 16
routings = 2
stage_num = [3, 3, 3]
lambda_d = 1
num_classes = 3
image_size = 64
num_primcaps = 7 * 3
m_dim = 5
S_set = [num_capsule, dim_capsule, routings, num_primcaps, m_dim]
model1 = FSA_net_Capsule(image_size, num_classes, stage_num, lambda_d,
S_set)()
model2 = FSA_net_Var_Capsule(image_size, num_classes, stage_num, lambda_d,
S_set)()
num_primcaps = 8 * 8 * 3
# 分别构建3种模型
S_set = [num_capsule, dim_capsule, routings, num_primcaps, m_dim]
model3 = FSA_net_noS_Capsule(image_size, num_classes, stage_num, lambda_d,
S_set)()
print('Loading models ...')
# 1x1的卷积模型
weight_file1 = '../pre-trained/300W_LP_models/fsanet_capsule_3_16_2_21_5/fsanet_capsule_3_16_2_21_5.h5'
model1.load_weights(weight_file1)
print('Finished loading model 1.')
# 方差模型
weight_file2 = '../pre-trained/300W_LP_models/fsanet_var_capsule_3_16_2_21_5/fsanet_var_capsule_3_16_2_21_5.h5'
model2.load_weights(weight_file2)
print('Finished loading model 2.')
# 无卷积和方差操作模型
weight_file3 = '../pre-trained/300W_LP_models/fsanet_noS_capsule_3_16_2_192_5/fsanet_noS_capsule_3_16_2_192_5.h5'
model3.load_weights(weight_file3)
print('Finished loading model 3.')
# 把3个模型整合到一起，取均值变成一个模型
inputs = Input(shape=(64, 64, 3))
x1 = model1(inputs) #1x1
x2 = model2(inputs) #var
x3 = model3(inputs) #w/o
avg_model = Average()([x1, x2, x3])
model = Model(inputs=inputs, outputs=avg_model)
# 加载序列化人脸检测器
print("[INFO] loading face detector...")
protoPath = os.path.sep.join(["face_detector", "deploy.prototxt"])
modelPath = os.path.sep.join(
["face_detector", "res10_300x300_ssd_iter_140000.caffemodel"])
net = cv2.dnn.readNetFromCaffe(protoPath, modelPath)
# capture video
# cap = cv2.VideoCapture(0)
cap = cv2.VideoCapture("LFPW_image_test_0005_0.jpg")
print(cap)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1024 * 1)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 768 * 1)
print('Start detecting pose ...')
detected_pre = np.empty((1, 1, 1))
while True:
# get video frame
ret, input_img = cap.read()
img_idx = img_idx + 1
img_h, img_w, _ = np.shape(input_img)
if img_idx == 1 or img_idx % skip_frame == 0:
time_detection = 0
time_network = 0
time_plot = 0
# 使用LBP检测器检测人脸
gray_img = cv2.cvtColor(input_img, cv2.COLOR_BGR2GRAY)
# detected = face_cascade.detectMultiScale(gray_img, 1.1)
# detected = detector.detect_faces(input_img)
# 将blob通过网络并获得检测结果
# 预测
blob = cv2.dnn.blobFromImage(cv2.resize(input_img,
(300, 300)), 1.0,
(300, 300), (104.0, 177.0, 123.0))
net.setInput(blob)
detected = net.forward()
if detected_pre.shape[2] > 0 and detected.shape[2] == 0:
detected = detected_pre
faces = np.empty((detected.shape[2], img_size, img_size, 3))
input_img = draw_results_ssd(detected, input_img, faces, ad,
img_size, img_w, img_h, model,
time_detection, time_network,
time_plot)
cv2.imwrite('img/' + str(img_idx) + '.png', input_img)
else:
input_img = draw_results_ssd(detected, input_img, faces, ad,
img_size, img_w, img_h, model,
time_detection, time_network,
time_plot)
if detected.shape[2] > detected_pre.shape[2] or img_idx % (skip_frame *
3) == 0:
detected_pre = detected
key = cv2.waitKey(1)
def fuse_rn(output_size,
new_arch,
train_kwargs,
models_kwargs,
weights_filepaths,
freeze_g_theta=False,
fuse_at_fc1=False,
avg_at_end=False):
prunned_models = []
for model_kwargs, weights_filepath in zip(models_kwargs,
weights_filepaths):
model = get_model(output_size=output_size, **model_kwargs)
if weights_filepath != []:
model.load_weights(weights_filepath)
if not fuse_at_fc1 and not avg_at_end:
for layer in model.layers[::
-1]: # reverse looking for last pool layer
if layer.name.startswith(
('average', 'concatenate', 'irn_attention')):
out_pool = layer.output
break
prunned_model = Model(inputs=model.input, outputs=out_pool)
elif fuse_at_fc1: # Prune keeping dropout + f_phi_fc1
for layer in model.layers[::
-1]: # reverse looking for last f_phi_fc1 layer
if layer.name.startswith(('f_phi_fc1')):
out_f_phi_fc1 = layer.output
break
prunned_model = Model(inputs=model.input, outputs=out_f_phi_fc1)
elif avg_at_end:
prunned_model = Model(inputs=model.input, outputs=model.output)
if freeze_g_theta:
for layer in prunned_model.layers: # Freezing model
layer.trainable = False
prunned_models.append(prunned_model)
### Train params
drop_rate = train_kwargs.get('drop_rate', 0.1)
kernel_init_type = train_kwargs.get('kernel_init_type', 'TruncatedNormal')
kernel_init_param = train_kwargs.get('kernel_init_param', 0.045)
kernel_init_seed = train_kwargs.get('kernel_init_seed')
kernel_init = get_kernel_init(kernel_init_type,
param=kernel_init_param,
seed=kernel_init_seed)
if new_arch:
# Building bottom
joint_stream_objects = []
temp_stream_objects = []
for i in range(models_kwargs[0]['num_objs']):
obj_joint = Input(shape=models_kwargs[0]['object_shape'],
name="joint_stream_object" + str(i))
joint_stream_objects.append(obj_joint)
for i in range(models_kwargs[1]['num_objs']):
obj_temp = Input(shape=models_kwargs[1]['object_shape'],
name="temp_stream_object" + str(i))
temp_stream_objects.append(obj_temp)
inputs = joint_stream_objects + temp_stream_objects
models_outs = [
prunned_models[0](joint_stream_objects),
prunned_models[1](temp_stream_objects)
]
else:
# Building bottom
person1_joints = []
person2_joints = []
for i in range(model_kwargs['num_objs']):
object_i = Input(shape=models_kwargs[0]['object_shape'],
name="person1_object" + str(i))
object_j = Input(shape=models_kwargs[0]['object_shape'],
name="person2_object" + str(i))
person1_joints.append(object_i)
person2_joints.append(object_j)
inputs = person1_joints + person2_joints
models_outs = [m(inputs) for m in prunned_models]
if not avg_at_end:
x = Concatenate()(models_outs)
# Building top and Model
top_kwargs = get_relevant_kwargs(model_kwargs, create_top)
x = create_top(x, kernel_init, **top_kwargs)
out_rn = Dense(output_size,
activation='softmax',
kernel_initializer=kernel_init,
name='softmax')(x)
else:
# Model outputs already include individual soft max
out_rn = Average()(models_outs)
model = Model(inputs=inputs, outputs=out_rn, name="fused_rel_net")
return model
reshapedPoolForSentence = Lambda(lambda x: K.reshape(x, shape=newShape),
name='switch_axis_' + 'sentence' +
str(i + 1) + 'winSize' +
str(j + windowMin))(mergedPoolForSentence)
densePoolForSentence = Dense(numDensePool,
bias_regularizer=regularizers.l2(eta),
kernel_regularizer=regularizers.l2(eta),
activation='softmax',
use_bias=True)(reshapedPoolForSentence)
densePoolForSentence = Dropout(dr, name='DropDense' +
str(i))(densePoolForSentence)
maxPooledPerDoc.append(densePoolForSentence)
#Naive Approach
averaged = Average()(maxPooledPerDoc)
averaged = Lambda(lambda x: K.reshape(
x, shape=(-1, int(averaged.shape[1]) * int(averaged.shape[2]))),
name='attend_output')(averaged)
out_avg = Dense(1, activation='sigmoid', use_bias=True)(averaged)
#Apply Attention
mergedPoolPerDoc = Concatenate(axis=1)(maxPooledPerDoc)
biRnn_ = Bidirectional(GRU(dimGRU, return_sequences=True),
merge_mode='concat')(mergedPoolPerDoc)
newShape = (-1, int(mergedPoolPerDoc.shape[1]), int(biRnn_.shape[2]))
biRnn = Lambda(lambda x: K.reshape(x, shape=newShape),
name='biRnn_TF_Reminder1')(biRnn_)
#biRnn2 = Lambda(lambda x: K.reshape(x,shape=newShape), name ='biRnn_TF_Reminder2')(biRnn_[1])
#QIITA
def background_est_cnn(image_frame):
"""
implements:
background estimation using temporal depth reduction
inputs:
image_fame: history image frame for background computation
return:
background estimation model
"""
tdr_layer_1_1 = Conv2D(32,
kernel_size=(1, 1),
strides=1,
padding='same',
name='tdr_layer_1_1',
data_format='channels_last')(image_frame)
tdr_layer_1_1 = Activation('relu')(tdr_layer_1_1)
tdr_layer_1_2 = Conv2D(32,
kernel_size=(3, 3),
strides=1,
padding='same',
name='tdr_layer_1_2',
data_format='channels_last')(image_frame)
tdr_layer_1_2 = Activation('relu')(tdr_layer_1_2)
tdr_layer_1_3 = Conv2D(32,
kernel_size=(5, 5),
strides=1,
padding='same',
name='tdr_layer_1_3',
data_format='channels_last')(image_frame)
tdr_layer_1_3 = Activation('relu')(tdr_layer_1_3)
tdr_layer_1 = Average()([tdr_layer_1_1, tdr_layer_1_2, tdr_layer_1_3])
tdr_layer_2_1 = Conv2D(16,
kernel_size=(1, 1),
strides=1,
padding='same',
name='tdr_layer_2_1',
data_format='channels_last')(tdr_layer_1)
tdr_layer_2_1 = Activation('relu')(tdr_layer_2_1)
tdr_layer_2_2 = Conv2D(16,
kernel_size=(3, 3),
strides=1,
padding='same',
name='tdr_layer_2_2',
data_format='channels_last')(tdr_layer_1)
tdr_layer_2_2 = Activation('relu')(tdr_layer_2_2)
tdr_layer_2_3 = Conv2D(16,
kernel_size=(5, 5),
strides=1,
padding='same',
name='tdr_layer_2_3',
data_format='channels_last')(tdr_layer_1)
tdr_layer_2_3 = Activation('relu')(tdr_layer_2_3)
tdr_layer_2 = Average()([tdr_layer_2_1, tdr_layer_2_2, tdr_layer_2_3])
tdr_layer_3_1 = Conv2D(8,
kernel_size=(1, 1),
strides=1,
padding='same',
name='tdr_layer_3_1',
data_format='channels_last')(tdr_layer_2)
tdr_layer_3_1 = Activation('relu')(tdr_layer_3_1)
tdr_layer_3_2 = Conv2D(8,
kernel_size=(3, 3),
strides=1,
padding='same',
name='tdr_layer_3_2',
data_format='channels_last')(tdr_layer_2)
tdr_layer_3_2 = Activation('relu')(tdr_layer_3_2)
tdr_layer_3_3 = Conv2D(8,
kernel_size=(5, 5),
strides=1,
padding='same',
name='tdr_layer_3_3',
data_format='channels_last')(tdr_layer_2)
tdr_layer_3_3 = Activation('relu')(tdr_layer_3_3)
tdr_layer_3 = Average()([tdr_layer_3_1, tdr_layer_3_2, tdr_layer_3_3])
tdr_layer_4 = Conv2D(1,
kernel_size=(3, 3),
padding='same',
name="TDR_block",
data_format='channels_last')(tdr_layer_3)
model = Activation('relu')(tdr_layer_4)
return model
def train(X_train, X_test):
batch_size = 512
same_frac = 0.125 # 0.0625 #0.25 # 0.125 #
epochs = 40
train_steps_per_epoch = 50
f = 0
# the data, shuffled and split between train and test sets
# input image dimensions
img_rows, img_cols = 28, 28
#input_dim = 784
input_dim = (img_rows, img_cols, 1)
nb_epoch = 12
# network definition
input_a = Input(shape=(img_rows, img_cols, 1))
input_b = Input(shape=(img_rows, img_cols, 1))
input_c = Input(shape=(img_rows, img_cols, 1))
# because we re-use the same instance `base_network`,
# the weights of the network
# will be shared across the two branches
#vec_a = create_base_network2(input_a)
#vec_b = create_base_network2(input_b)
#vec_c = create_base_network2(input_c)
cnet = convnet((img_rows, img_cols, 1))
vec_a = cnet(input_a)
vec_b = cnet(input_b)
vec_c = cnet(input_c)
#vec_a = Activation('selu')(vec_a)
#vec_b = Activation('selu')(vec_b)
#vec_c = Activation('selu')(vec_c)
#vec_a2 = GaussianNoise(50.0)(vec_a)
#vec_b2 = GaussianNoise(50.0)(vec_b)
#vec_c2 = GaussianNoise(50.0)(vec_c)
#vec_a2 = Lambda(lambda v: K.tanh(v[0] - v[1]))([vec_b, vec_c])
#vec_b2 = Lambda(lambda v: K.tanh(v[0] - v[1]))([vec_a, vec_c])
#vec_c2 = Lambda(lambda v: K.tanh(v[0] - v[1]))([vec_a, vec_b])
#vec_a2 = GaussianNoise(0.01)(vec_a2)
#vec_b2 = GaussianNoise(0.01)(vec_b2)
#vec_c2 = GaussianNoise(0.01)(vec_c2)
#vec_a2 = Lambda(lambda v: K.clip(v, -1.0, 1.0))(vec_a2)
#vec_b2 = Lambda(lambda v: K.clip(v, -1.0, 1.0))(vec_b2)
#vec_c2 = Lambda(lambda v: K.clip(v, -1.0, 1.0))(vec_c2)
#vec_a3 = Dot(axes=-1, normalize=False)([vec_b, vec_c])
#vec_b3 = Dot(axes=-1, normalize=False)([vec_c, vec_a])
#vec_c3 = Dot(axes=-1, normalize=False)([vec_a, vec_b])
#x = Lambda(lambda v : K.stack([K.abs(v[1]-v[2]), K.abs(v[2]-v[0]), K.abs(v[0]-v[1])], axis=-1))([vec_a, vec_b, vec_c])
#x = Lambda(lambda v : K.stack([K.mean(K.square(v[1]-v[2]), axis=-1), K.mean(K.square(v[2]-v[0]), axis=-1), K.mean(K.square(v[0]-v[1]), axis=-1)], axis=-1))([vec_a, vec_b, vec_c])
#x = Lambda(lambda v : K.stack([K.log(K.mean(K.square(v[1]-v[2]), axis=-1)), K.log(K.mean(K.square(v[2]-v[0]), axis=-1)), K.log(K.mean(K.square(v[0]-v[1]), axis=-1))], axis=-1))([vec_a, vec_b, vec_c])
#x = Lambda(lambda v : K.stack([K.log(K.epsilon()+K.mean(K.square(v[1]-v[2]), axis=-1)), K.log(K.mean(K.epsilon()+K.square(v[2]-v[0]), axis=-1)), K.log(K.mean(K.epsilon()+K.square(v[0]-v[1]), axis=-1))], axis=-1))([vec_a, vec_b, vec_c])
#x = Lambda(lambda v : K.stack([v[1]*v[2], v[2]*v[0], v[0]*v[1]], axis=-1))([vec_a, vec_b, vec_c])
#x = Lambda(lambda v : K.stack(v, axis=-1))([vec_a3, vec_b3, vec_c3])
#x = Activation('selu')(x)
#x = TimeDistributed(Dense(3, activation='selu'))(x)
#x = TimeDistributed(Dense(3, activation='sigmoid'))(x)
#x = Flatten()(x)
#x = Dense(3*nb_filters, activation='selu')(x)
#x = Dense(nb_filters, activation='selu')(x)
#x = Convolution1D(2*nb_filters, 3, strides=1, activation='selu', padding='same')(x)
#x = Convolution1D(2*nb_filters, 3, strides=2, activation='selu', padding='same')(x)
#x = Convolution1D(3, 1, strides=1, activation='sigmoid', padding='same')(x)
#x = GlobalAveragePooling1D()(x)
#probs = Activation('softmax')(x)
#x = Convolution1D(int(nb_filters/2), 2, strides=2, activation='selu', padding='same')(x)
#x = Reshape((-1,1))(x)
#x = TimeDistributed(Dense(nb_filters, activation='selu'))(x)
#x = TimeDistributed(Dense(nb_filters, activation='selu'))(x)
#x = TimeDistributed(Dense(1, activation='linear'))(x)
#x = Flatten()(x)
#probs = Lambda(lambda y : K.exp(-y)/(K.repeat_elements(K.expand_dims(K.epsilon() + K.sum(K.exp(-y), axis=-1)), 3, axis=-1)))(x)
input_u = Input(shape=(3, ))
y = Dense(6, activation='selu')(input_u)
y = Dense(6, activation='selu')(y)
p = Dense(3, activation='softmax')(y)
ntop = int(nb_filters / 2)
unit = Model(inputs=input_u, outputs=p)
x0 = Lambda(lambda v: K.stack([
K.mean(K.square(v[1] - v[2]), axis=-1),
K.mean(K.square(v[2] - v[0]), axis=-1),
K.mean(K.square(v[0] - v[1]), axis=-1)
],
axis=-1))([vec_a, vec_b, vec_c])
x1 = Lambda(lambda v: K.stack([
K.mean(K.square(v[2] - v[0]), axis=-1),
K.mean(K.square(v[0] - v[1]), axis=-1),
K.mean(K.square(v[1] - v[2]), axis=-1)
],
axis=-1))([vec_a, vec_b, vec_c])
x2 = Lambda(lambda v: K.stack([
K.mean(K.square(v[0] - v[1]), axis=-1),
K.mean(K.square(v[1] - v[2]), axis=-1),
K.mean(K.square(v[2] - v[0]), axis=-1)
],
axis=-1))([vec_a, vec_b, vec_c])
# x0 = Lambda(lambda v : K.stack([K.mean(tf.nn.top_k(K.square(v[1]-v[2]), k=ntop)[0], axis=-1), K.mean(tf.nn.top_k(K.square(v[2]-v[0]), k=ntop)[0], axis=-1), K.mean(tf.nn.top_k(K.square(v[0]-v[1]), k=ntop)[0], axis=-1)], axis=-1))([vec_a, vec_b, vec_c])
# x1 = Lambda(lambda v : K.stack([K.mean(tf.nn.top_k(K.square(v[2]-v[0]), k=ntop)[0], axis=-1), K.mean(tf.nn.top_k(K.square(v[0]-v[1]), k=ntop)[0], axis=-1), K.mean(tf.nn.top_k(K.square(v[1]-v[2]), k=ntop)[0], axis=-1)], axis=-1))([vec_a, vec_b, vec_c])
# x2 = Lambda(lambda v : K.stack([K.mean(tf.nn.top_k(K.square(v[0]-v[1]), k=ntop)[0], axis=-1), K.mean(tf.nn.top_k(K.square(v[1]-v[2]), k=ntop)[0], axis=-1), K.mean(tf.nn.top_k(K.square(v[2]-v[0]), k=ntop)[0], axis=-1)], axis=-1))([vec_a, vec_b, vec_c])
x0 = unit(x0)
x1 = unit(x1)
x2 = unit(x2)
x1 = Lambda(lambda v: K.stack([v[:, 2], v[:, 0], v[:, 1]], axis=-1))(x1)
x2 = Lambda(lambda v: K.stack([v[:, 1], v[:, 2], v[:, 0]], axis=-1))(x2)
probs = Average()([x0, x1, x2])
model = Model(inputs=[input_a, input_b, input_c], outputs=probs)
optimizer = Adam(lr=0.0003, clipnorm=0.1, clipvalue=0.1) #
model.compile(loss=pick_best_loss, optimizer=optimizer)
#overlaps = np.ones((X_train.shape[0], X_train.shape[0]))
train_gen = create_triplets(X_train,
None,
same_frac,
batch_size,
mode='train')
es_gen = create_triplets(X_test, None, same_frac, batch_size, mode='val')
es_steps = 100
callbacks = [
EarlyStopping(monitor='val_loss', patience=4, verbose=0),
ModelCheckpoint('autoenc' + str(f) + '.h5',
monitor='loss',
save_best_only=True,
verbose=0)
]
model.fit_generator(train_gen,
steps_per_epoch=train_steps_per_epoch,
epochs=epochs,
validation_data=es_gen,
validation_steps=es_steps,
callbacks=callbacks) #, max_q_size=10, workers=1
#model.fit_generator(train_gen, steps_per_epoch=train_steps_per_epoch, epochs=epochs, validation_data=val_gen, validation_steps=100) #, max_q_size=10, workers=1
model.load_weights('autoenc' + str(f) + '.h5')
#train_gen.close()
#es_gen.close()
'''
history.close()
equa += "{}\t{:.3f}".format(args.save1, acc)
model1 = keras.models.load_model(savename + "/check_" + str(epoch))
for i in range(len(model1.layers)):
model1.layers[i].name += "_1"
savename = "save/" + str(args.save2)
history = open(savename + '/history')
epoch = eval(history.readline()) + 1
acc = eval(history.readline())['val_acc'][epoch - 1]
history.close()
equa += "{}\t{:.3f}".format(args.save2, acc)
model2 = keras.models.load_model(savename + "/check_" + str(epoch))
for i in range(len(model2.layers)):
model2.layers[i].name += "_2"
out = Average()([model1.outputs[0], model2.outputs[0]])
imodel = Model(inputs=[model1.input, model2.input],
outputs=out,
name='ensemble')
#imodel=Model(inputs=model2.input,outputs=model2.outputs[0],name='ensemble')
rc = ""
for sha in imodel._feed_inputs:
if (len(sha._keras_shape) == 4):
rc += "c"
if (len(sha._keras_shape) == 3):
rc += "r"
train = wkiter([args.left + ".root", args.right + ".root"],
batch_size=batch_size,
end=args.end * 1.,
istrain=1,
K.greater_equal(x, 0.5),
Add(name="add_inner")
([xception_model_embeddings_current, xception_model_embeddings_previous]
), xception_model_embeddings_current),
name="add_conditional")(
similarity_prediction)
#diffrentiable_conditional_dot = Lambda( lambda x:K.where(x>=0.5, Dot()([xception_model_embeddings_current, xception_model_embeddings_previous]), xception_model_embeddings_current))(similarity_prediction)
diffrentiable_conditional_multiply = Lambda(lambda x: K.switch(
x >= 0.5,
Multiply()
([xception_model_embeddings_current, xception_model_embeddings_previous]
), xception_model_embeddings_current))(similarity_prediction)
diffrentiable_conditional_average = Lambda(lambda x: K.switch(
x >= 0.5,
Average()
([xception_model_embeddings_current, xception_model_embeddings_previous]
), xception_model_embeddings_current))(similarity_prediction)
#diffrentiable_conidtional_concatenate = Lambda( lambda x:K.where(x>=0.5,Concatenate()([xception_model_embeddings_current, xception_model_embeddings_previous]) , xception_model_embeddings_current))(similarity_prediction)
#Here we try convolution
#Be sure to initialize a different one with var input sizes for concat version
#predictor_convolution = xception_conv_final_predictor(diffrentiable_conditional_add)
final_predictor_1 = Flatten()(diffrentiable_conditional_add)
final_predictor_2 = Dense(1024, activation='relu')(final_predictor_1)
final_predictor_3 = Dense(512, activation='relu')(final_predictor_2)
final_predictor_4 = Dense(256, activation='relu')(final_predictor_3)
final_prediction_reg = Dense(2, name="final_pred_reg")(final_predictor_4)
final_prediction_class = Dense(3,
activation='softmax',
name="final_pred_class")(final_predictor_4)
