processed_and_indexed_training_examples)
outputs = np.array(outputs)
testing_preprocessed_and_indexed = processed_and_indexed_training_examples[
1300:]
testing_outputs = outputs[1300:]
#processed_and_indexed_training_examples = processed_and_indexed_training_examples[:1300]
#outputs = outputs[:1300]
API_sequence = layers.Input(shape=(API_SEQUENCE_MAX_LEN, ), dtype='int32')
embeded_sequence = layers.Embedding(len(all_api_calls),
EMBED_HIDDEN_SIZE)(API_sequence)
conv_layer = layers.Conv1D(NO_OF_CONV_FILTERS,
NO_OF_CONV_SIZE)(embeded_sequence)
activ_layer = layers.Activation('relu')(conv_layer)
global_max_pool = layers.GlobalMaxPooling1D()(activ_layer)
fc_layer = layers.Dense(256)(global_max_pool)
fc_relu = layers.Activation('relu')(fc_layer)
dropout_layer = layers.Dropout(0.5)(fc_relu)
final_layer = layers.Dense(2)(dropout_layer)
final_softmax_layer = layers.Activation('softmax')(final_layer)
model = Model(API_sequence, final_softmax_layer)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(processed_and_indexed_training_examples,
outputs,
epochs=100,
validation_split=0.0)
def sqnxt_unit(x,
in_channels,
out_channels,
strides,
name="sqnxt_unit"):
"""
SqueezeNext unit.
Parameters:
----------
x : keras.backend tensor/variable/symbol
Input tensor/variable/symbol.
in_channels : int
Number of input channels.
out_channels : int
Number of output channels.
strides : int or tuple/list of 2 int
Strides of the convolution.
name : str, default 'sqnxt_unit'
Block name.
Returns
-------
keras.backend tensor/variable/symbol
Resulted tensor/variable/symbol.
"""
if strides == 2:
reduction_den = 1
resize_identity = True
elif in_channels > out_channels:
reduction_den = 4
resize_identity = True
else:
reduction_den = 2
resize_identity = False
if resize_identity:
identity = conv1x1_block(
x=x,
in_channels=in_channels,
out_channels=out_channels,
strides=strides,
use_bias=True,
name=name + "/identity_conv")
else:
identity = x
x = conv1x1_block(
x=x,
in_channels=in_channels,
out_channels=(in_channels // reduction_den),
strides=strides,
use_bias=True,
name=name + "/conv1")
x = conv1x1_block(
x=x,
in_channels=(in_channels // reduction_den),
out_channels=(in_channels // (2 * reduction_den)),
use_bias=True,
name=name + "/conv2")
x = conv_block(
x=x,
in_channels=(in_channels // (2 * reduction_den)),
out_channels=(in_channels // reduction_den),
kernel_size=(1, 3),
strides=1,
padding=(0, 1),
use_bias=True,
name=name + "/conv3")
x = conv_block(
x=x,
in_channels=(in_channels // reduction_den),
out_channels=(in_channels // reduction_den),
kernel_size=(3, 1),
strides=1,
padding=(1, 0),
use_bias=True,
name=name + "/conv4")
x = conv1x1_block(
x=x,
in_channels=(in_channels // reduction_den),
out_channels=out_channels,
use_bias=True,
name=name + "/conv5")
x = nn.add([x, identity], name=name + "/add")
x = nn.Activation('relu', name=name + "/final_activ")(x)
return x
3,
activation='relu',
input_shape=[IMAGE_SIZE, IMAGE_SIZE, 1],
padding='same'))
M.add(KL.MaxPooling2D())
M.add(KL.Conv2D(32, 3, activation='relu', padding='same'))
M.add(KL.Conv2D(32, 3, activation='relu', padding='same'))
M.add(KL.MaxPooling2D())
M.add(KL.Conv2D(32, 3, padding='same', activation='relu'))
M.add(KL.Flatten())
M.add(
KL.Dense(512,
activation='relu',
kernel_regularizer=regularizers.l2(1e-5)))
M.add(KL.Dropout(0.5))
M.add(
KL.Dense(10, activation=None,
kernel_regularizer=regularizers.l2(1e-5)))
M.add(KL.Activation('softmax'))
dataset_train, dataset_test = get_data()
M = KerasModel(M, QueueInput(dataset_train))
M.compile(optimizer=tf.train.AdamOptimizer(1e-3),
loss='categorical_crossentropy',
metrics=['accuracy'])
M.fit(
validation_data=dataset_test,
steps_per_epoch=dataset_train.size(),
)
def out2d():
inp = layers.Input((352, 640, 3))
x = layers.Conv2D(32, 3, strides=(2, 2), padding='same')(inp)
x = layers.BatchNormalization()(x)
x = layers.Activation('relu')(x)
d1 = layers.Conv2D(32, 3, padding='same')(x)
x = layers.BatchNormalization()(d1)
x = layers.Activation('relu')(x)
x = layers.MaxPooling2D()(x)
x = layers.Conv2D(64, 3, padding='same')(x)
x = layers.BatchNormalization()(x)
x = layers.Activation('relu')(x)
d2 = layers.Conv2D(64, 3, padding='same')(x)
x = layers.BatchNormalization()(d2)
x = layers.Activation('relu')(x)
x = layers.MaxPooling2D()(x)
x = layers.Conv2D(128, 3, padding='same')(x)
x = layers.BatchNormalization()(x)
x = layers.Activation('relu')(x)
d3 = layers.Conv2D(128, 3, padding='same')(x)
x = layers.BatchNormalization()(d3)
x = layers.Activation('relu')(x)
x = layers.MaxPooling2D()(x)
x = layers.Conv2D(256, 3, padding='same')(x)
x = layers.BatchNormalization()(x)
x = layers.Activation('relu')(x)
d4 = layers.Conv2D(256, 3, padding='same')(x)
x = layers.BatchNormalization()(d4)
x = layers.Activation('relu')(x)
x = layers.MaxPooling2D()(x)
x = layers.Conv2D(512, 3, padding='same')(x)
x = layers.BatchNormalization()(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(512, 3, padding='same')(x)
x = layers.BatchNormalization()(x)
x = layers.Activation('relu')(x)
# x = layers.Conv2DTranspose(256, 3, strides=(2, 2), padding='same')(x)
# x = layers.BatchNormalization()(x)
# x = layers.Activation('relu')(x)
# x = layers.Conv2D(256, 3, padding='same')(x)
# # x = layers.concatenate([x, d4])
# x = layers.BatchNormalization()(x)
# x = layers.Activation('relu')(x)
# x = layers.Conv2DTranspose(128, 3, strides=(2, 2), padding='same')(x)
# x = layers.BatchNormalization()(x)
# x = layers.Activation('relu')(x)
# x = layers.Conv2D(128, 3, padding='same')(x)
# # x = layers.concatenate([x, d3])
# x = layers.BatchNormalization()(x)
# x = layers.Activation('relu')(x)
# x = layers.Conv2DTranspose(64, 3, strides=(2, 2), padding='same')(x)
# x = layers.BatchNormalization()(x)
# x = layers.Activation('relu')(x)
# x = layers.Conv2D(64, 3, padding='same')(x)
# # x = layers.concatenate([x, d2])
# x = layers.BatchNormalization()(x)
# x = layers.Activation('relu')(x)
# x = layers.Conv2DTranspose(32, 3, strides=(2, 2), padding='same')(x)
# x = layers.BatchNormalization()(x)
# x = layers.Activation('relu')(x)
# x = layers.Conv2D(32, 3, padding='same')(x)
# # x = layers.concatenate([x, d1])
# x = layers.BatchNormalization()(x)
# x = layers.Activation('relu')(x)
# x = layers.Conv2D(32, 3, padding='same')(x)
# x = layers.BatchNormalization()(x)
# x = layers.Activation('relu')(x)
x = layers.Conv2D(1, 3, padding='same', activation='sigmoid')(x)
model = models.Model(inp, x)
model.compile(optimizer=optimizers.Adam(),
loss='binary_crossentropy',
metrics=['acc'])
return model
def build_model(self):
"""Build a critic (value) network that maps (state, action) pairs -> Q-values."""
# Define input layers
states = layers.Input(shape=(self.state_size, ), name='states')
actions = layers.Input(shape=(self.action_size, ), name='actions')
# Add hidden layer(s) for state pathway
net_states = layers.Dense(
units=32,
use_bias=False,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(states)
net_states = layers.BatchNormalization()(net_states)
net_states = layers.Activation('relu')(net_states)
net_states = layers.Dropout(0.5)(net_states)
net_states = layers.Dense(
units=64,
use_bias=False,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(states)
net_states = layers.BatchNormalization()(net_states)
net_states = layers.Activation('relu')(net_states)
net_states = layers.Dropout(0.5)(net_states)
net_states = layers.Dense(
units=128,
use_bias=False,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(states)
net_states = layers.BatchNormalization()(net_states)
net_states = layers.Activation('relu')(net_states)
net_states = layers.Dropout(0.5)(net_states)
# Add hidden layer(s) for action pathway
net_actions = layers.Dense(
units=32,
use_bias=False,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(actions)
net_actions = layers.BatchNormalization()(net_actions)
net_actions = layers.Activation('relu')(net_actions)
net_actions = layers.Dropout(0.5)(net_actions)
net_actions = layers.Dense(
units=64,
use_bias=False,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(actions)
net_actions = layers.BatchNormalization()(net_actions)
net_actions = layers.Activation('relu')(net_actions)
net_actions = layers.Dropout(0.5)(net_actions)
net_actions = layers.Dense(
units=128,
use_bias=False,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(actions)
net_actions = layers.BatchNormalization()(net_actions)
net_actions = layers.Activation('relu')(net_actions)
net_actions = layers.Dropout(0.5)(net_actions)
# Try different layer sizes, activations, add batch normalization, regularizers, etc.
# Combine state and action pathways
net = layers.Add()([net_states, net_actions])
net = layers.Activation('relu')(net)
# Add more layers to the combined network if needed
# Add final output layer to prduce action values (Q values)
Q_values = layers.Dense(units=1, name='q_values')(net)
# Create Keras model
self.model = models.Model(inputs=[states, actions], outputs=Q_values)
# Define optimizer and compile model for training with built-in loss function
optimizer = optimizers.Adam()
self.model.compile(optimizer=optimizer, loss='mse')
# Compute action gradients (derivative of Q values w.r.t. to actions)
action_gradients = K.gradients(Q_values, actions)
# Define an additional function to fetch action gradients (to be used by actor model)
self.get_action_gradients = K.function(
inputs=[*self.model.input, K.learning_phase()],
outputs=action_gradients)
def build_model(self):
"""Build an actor (policy) network that maps states -> actions."""
# Define input layer (states)
states = layers.Input(shape=(self.state_size,), name="states")
net = layers.Dense(units=400)(states)
# net = layers.BatchNormalization()(net)
net = layers.Activation("relu")(net)
net = layers.Dense(units=300)(net)
# net = layers.BatchNormalization()(net)
net = layers.Activation("relu")(net)
# # # Add final output layer with sigmoid activation
# raw_actions = layers.Dense(
# units=self.action_size,
# activation="sigmoid",
# name="raw_actions",
# kernel_initializer=layers.initializers.RandomUniform(
# minval=-0.003, maxval=0.003
# ),
# )(net)
# # Try different layer sizes, activations, add batch normalization, regularizers, etc.
# # Add final output layer with sigmoid activation
# # raw_actions = layers.Dense(
# # units=self.action_size, activation="sigmoid", name="raw_actions"
# # )(net)
# # Scale [0, 1] output for each action dimension to proper range
# # Create temp variable for storing self variables before using in lambda
# # Without this, you get recursion error when saving Keras model: https://github.com/keras-team/keras/issues/12081
# action_range = self.action_range
# action_low = self.action_low
# actions = layers.Lambda(
# lambda x: (x * action_range) + action_low, name="actions"
# )(raw_actions)
# Use tanh activation function with no scaling, since actions are in [-1,1]
actions = layers.Dense(
units=self.action_size,
activation="tanh",
name="actions",
kernel_initializer=layers.initializers.RandomUniform(
minval=-0.003, maxval=0.003
),
)(net)
# Create Keras model
self.model = models.Model(inputs=states, outputs=actions)
# Define loss function using action value (Q value) gradients
action_gradients = layers.Input(shape=(self.action_size,))
loss = K.mean(-action_gradients * actions)
# Incorporate any additional losses here (e.g. from regularizers)
# Define optimizer and training function
optimizer = optimizers.Adam(lr=1e-4)
updates_op = optimizer.get_updates(
params=self.model.trainable_weights, loss=loss
)
self.train_fn = K.function(
inputs=[self.model.input, action_gradients, K.learning_phase()],
outputs=[loss],
updates=updates_op,
)
def KitModel(weight_file=None):
weights_dict = load_weights_from_file(
weight_file) if not weight_file == None else None
data = layers.Input(name='data', shape=(
96,
96,
3,
))
conv1_1_input = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(data)
conv1_1 = convolution(weights_dict,
name='conv1_1',
input=conv1_1_input,
group=1,
conv_type='layers.Conv2D',
filters=64,
kernel_size=(3, 3),
strides=(1, 1),
dilation_rate=(1, 1),
padding='valid',
use_bias=True)
relu1_1 = layers.Activation(name='relu1_1', activation='relu')(conv1_1)
conv1_2_input = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(relu1_1)
conv1_2 = convolution(weights_dict,
name='conv1_2',
input=conv1_2_input,
group=1,
conv_type='layers.Conv2D',
filters=64,
kernel_size=(3, 3),
strides=(1, 1),
dilation_rate=(1, 1),
padding='valid',
use_bias=True)
relu1_2 = layers.Activation(name='relu1_2', activation='relu')(conv1_2)
pool1_input = layers.ZeroPadding2D(padding=((0, 1), (0, 1)))(relu1_2)
pool1 = layers.MaxPooling2D(name='pool1',
pool_size=(2, 2),
strides=(2, 2),
padding='valid')(pool1_input)
conv2_1_input = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(pool1)
conv2_1 = convolution(weights_dict,
name='conv2_1',
input=conv2_1_input,
group=1,
conv_type='layers.Conv2D',
filters=128,
kernel_size=(3, 3),
strides=(1, 1),
dilation_rate=(1, 1),
padding='valid',
use_bias=True)
relu2_1 = layers.Activation(name='relu2_1', activation='relu')(conv2_1)
conv2_2_input = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(relu2_1)
conv2_2 = convolution(weights_dict,
name='conv2_2',
input=conv2_2_input,
group=1,
conv_type='layers.Conv2D',
filters=128,
kernel_size=(3, 3),
strides=(1, 1),
dilation_rate=(1, 1),
padding='valid',
use_bias=True)
relu2_2 = layers.Activation(name='relu2_2', activation='relu')(conv2_2)
pool2_input = layers.ZeroPadding2D(padding=((0, 1), (0, 1)))(relu2_2)
pool2 = layers.MaxPooling2D(name='pool2',
pool_size=(2, 2),
strides=(2, 2),
padding='valid')(pool2_input)
conv3_1_input = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(pool2)
conv3_1 = convolution(weights_dict,
name='conv3_1',
input=conv3_1_input,
group=1,
conv_type='layers.Conv2D',
filters=256,
kernel_size=(3, 3),
strides=(1, 1),
dilation_rate=(1, 1),
padding='valid',
use_bias=True)
relu3_1 = layers.Activation(name='relu3_1', activation='relu')(conv3_1)
conv3_2_input = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(relu3_1)
conv3_2 = convolution(weights_dict,
name='conv3_2',
input=conv3_2_input,
group=1,
conv_type='layers.Conv2D',
filters=256,
kernel_size=(3, 3),
strides=(1, 1),
dilation_rate=(1, 1),
padding='valid',
use_bias=True)
relu3_2 = layers.Activation(name='relu3_2', activation='relu')(conv3_2)
conv3_3_input = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(relu3_2)
conv3_3 = convolution(weights_dict,
name='conv3_3',
input=conv3_3_input,
group=1,
conv_type='layers.Conv2D',
filters=256,
kernel_size=(3, 3),
strides=(1, 1),
dilation_rate=(1, 1),
padding='valid',
use_bias=True)
relu3_3 = layers.Activation(name='relu3_3', activation='relu')(conv3_3)
pool3_input = layers.ZeroPadding2D(padding=((0, 1), (0, 1)))(relu3_3)
pool3 = layers.MaxPooling2D(name='pool3',
pool_size=(2, 2),
strides=(2, 2),
padding='valid')(pool3_input)
conv4_1_input = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(pool3)
conv4_1 = convolution(weights_dict,
name='conv4_1',
input=conv4_1_input,
group=1,
conv_type='layers.Conv2D',
filters=512,
kernel_size=(3, 3),
strides=(1, 1),
dilation_rate=(1, 1),
padding='valid',
use_bias=True)
relu4_1 = layers.Activation(name='relu4_1', activation='relu')(conv4_1)
conv4_2_input = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(relu4_1)
conv4_2 = convolution(weights_dict,
name='conv4_2',
input=conv4_2_input,
group=1,
conv_type='layers.Conv2D',
filters=512,
kernel_size=(3, 3),
strides=(1, 1),
dilation_rate=(1, 1),
padding='valid',
use_bias=True)
relu4_2 = layers.Activation(name='relu4_2', activation='relu')(conv4_2)
conv4_3_input = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(relu4_2)
conv4_3 = convolution(weights_dict,
name='conv4_3',
input=conv4_3_input,
group=1,
conv_type='layers.Conv2D',
filters=512,
kernel_size=(3, 3),
strides=(1, 1),
dilation_rate=(1, 1),
padding='valid',
use_bias=True)
relu4_3 = layers.Activation(name='relu4_3', activation='relu')(conv4_3)
pool4_input = layers.ZeroPadding2D(padding=((0, 1), (0, 1)))(relu4_3)
pool4 = layers.MaxPooling2D(name='pool4',
pool_size=(2, 2),
strides=(2, 2),
padding='valid')(pool4_input)
conv5_1_input = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(pool4)
conv5_1 = convolution(weights_dict,
name='conv5_1',
input=conv5_1_input,
group=1,
conv_type='layers.Conv2D',
filters=512,
kernel_size=(3, 3),
strides=(1, 1),
dilation_rate=(1, 1),
padding='valid',
use_bias=True)
relu5_1 = layers.Activation(name='relu5_1', activation='relu')(conv5_1)
conv5_2_input = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(relu5_1)
conv5_2 = convolution(weights_dict,
name='conv5_2',
input=conv5_2_input,
group=1,
conv_type='layers.Conv2D',
filters=512,
kernel_size=(3, 3),
strides=(1, 1),
dilation_rate=(1, 1),
padding='valid',
use_bias=True)
relu5_2 = layers.Activation(name='relu5_2', activation='relu')(conv5_2)
conv5_3_input = layers.ZeroPadding2D(padding=((1, 1), (1, 1)))(relu5_2)
conv5_3 = convolution(weights_dict,
name='conv5_3',
input=conv5_3_input,
group=1,
conv_type='layers.Conv2D',
filters=512,
kernel_size=(3, 3),
strides=(1, 1),
dilation_rate=(1, 1),
padding='valid',
use_bias=True)
relu5_3 = layers.Activation(name='relu5_3', activation='relu')(conv5_3)
conv5_blur = convolution(weights_dict,
name='conv5_blur',
input=relu5_3,
group=1,
conv_type='layers.Conv2D',
filters=1,
kernel_size=(1, 1),
strides=(1, 1),
dilation_rate=(1, 1),
padding='valid',
use_bias=True)
conv5_blur_up_input = layers.ZeroPadding2D(padding=((8, 8),
(8, 8)))(conv5_blur)
conv5_blur_up = convolution(weights_dict,
name='conv5_blur_up',
input=conv5_blur_up_input,
group=1,
conv_type='layers.Conv2DTranspose',
filters=1,
kernel_size=(32, 32),
strides=(16, 16),
dilation_rate=(1, 1),
padding='valid',
use_bias=False)
score = layers.Activation(name='score',
activation='sigmoid')(conv5_blur_up)
model = Model(inputs=[data], outputs=[score])
set_layer_weights(model, weights_dict)
return model
def xceptionv0(input_shape=(512, 512, 3),
input_tensor=None,
pretrained_weights_path=None,
output_stride=16):
""" the original xception model """
if input_tensor is None:
img_input = layers.Input(shape=input_shape, name='img_input')
else:
img_input = input_tensor
x = layers.BatchNormalization()(img_input)
channel_axis = -1
output_stride = output_stride
x = layers.Conv2D(32, (3, 3),
strides=(2, 2),
use_bias=False,
name='block1_conv1')(x)
x = layers.BatchNormalization(axis=channel_axis, name='block1_conv1_bn')(x)
x = layers.Activation('relu', name='block1_conv1_act')(x)
x = layers.Conv2D(64, (3, 3), use_bias=False, name='block1_conv2')(x)
x = layers.BatchNormalization(axis=channel_axis, name='block1_conv2_bn')(x)
x = layers.Activation('relu', name='block1_conv2_act')(x)
residual = layers.Conv2D(128, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = layers.BatchNormalization(axis=channel_axis)(residual)
x = layers.SeparableConv2D(128, (3, 3),
padding='same',
use_bias=False,
name='block2_sepconv1')(x)
x = layers.BatchNormalization(axis=channel_axis,
name='block2_sepconv1_bn')(x)
x = layers.Activation('relu', name='block2_sepconv2_act')(x)
x = layers.SeparableConv2D(128, (3, 3),
padding='same',
use_bias=False,
name='block2_sepconv2')(x)
x = layers.BatchNormalization(axis=channel_axis,
name='block2_sepconv2_bn')(x)
f1 = x
x = layers.MaxPooling2D((3, 3),
strides=(2, 2),
padding='same',
name='block2_pool')(x)
x = layers.add([x, residual])
residual = layers.Conv2D(256, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = layers.BatchNormalization(axis=channel_axis)(residual)
x = layers.Activation('relu', name='block3_sepconv1_act')(x)
x = layers.SeparableConv2D(256, (3, 3),
padding='same',
use_bias=False,
name='block3_sepconv1')(x)
x = layers.BatchNormalization(axis=channel_axis,
name='block3_sepconv1_bn')(x)
x = layers.Activation('relu', name='block3_sepconv2_act')(x)
x = layers.SeparableConv2D(256, (3, 3),
padding='same',
use_bias=False,
name='block3_sepconv2')(x)
x = layers.BatchNormalization(axis=channel_axis,
name='block3_sepconv2_bn')(x)
f2 = x
x = layers.MaxPooling2D((3, 3),
strides=(2, 2),
padding='same',
name='block3_pool')(x)
x = layers.add([x, residual])
residual = layers.Conv2D(728, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = layers.BatchNormalization(axis=channel_axis)(residual)
x = layers.Activation('relu', name='block4_sepconv1_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name='block4_sepconv1')(x)
x = layers.BatchNormalization(axis=channel_axis,
name='block4_sepconv1_bn')(x)
x = layers.Activation('relu', name='block4_sepconv2_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name='block4_sepconv2')(x)
x = layers.BatchNormalization(axis=channel_axis,
name='block4_sepconv2_bn')(x)
f3 = x
# simply remove the max pooling layer
if output_stride > 16:
x = layers.MaxPooling2D((3, 3),
strides=(2, 2),
padding='same',
name='block4_pool')(x)
x = layers.add([x, residual])
for i in range(8):
residual = x
prefix = 'block' + str(i + 5)
x = layers.Activation('relu', name=prefix + '_sepconv1_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name=prefix + '_sepconv1')(x)
x = layers.BatchNormalization(axis=channel_axis,
name=prefix + '_sepconv1_bn')(x)
x = layers.Activation('relu', name=prefix + '_sepconv2_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name=prefix + '_sepconv2')(x)
x = layers.BatchNormalization(axis=channel_axis,
name=prefix + '_sepconv2_bn')(x)
x = layers.Activation('relu', name=prefix + '_sepconv3_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name=prefix + '_sepconv3')(x)
x = layers.BatchNormalization(axis=channel_axis,
name=prefix + '_sepconv3_bn')(x)
x = layers.add([x, residual])
residual = layers.Conv2D(1024, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = layers.BatchNormalization(axis=channel_axis)(residual)
x = layers.Activation('relu', name='block13_sepconv1_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name='block13_sepconv1')(x)
x = layers.BatchNormalization(axis=channel_axis,
name='block13_sepconv1_bn')(x)
x = layers.Activation('relu', name='block13_sepconv2_act')(x)
x = layers.SeparableConv2D(1024, (3, 3),
padding='same',
use_bias=False,
name='block13_sepconv2')(x)
x = layers.BatchNormalization(axis=channel_axis,
name='block13_sepconv2_bn')(x)
f4 = x
if output_stride > 8:
x = layers.MaxPooling2D((3, 3),
strides=(2, 2),
padding='same',
name='block13_pool')(x)
x = layers.add([x, residual])
x = layers.SeparableConv2D(1536, (3, 3),
padding='same',
use_bias=False,
name='block14_sepconv1')(x)
x = layers.BatchNormalization(axis=channel_axis,
name='block14_sepconv1_bn')(x)
x = layers.Activation('relu', name='block14_sepconv1_act')(x)
x = layers.SeparableConv2D(2048, (3, 3),
padding='same',
use_bias=False,
name='block14_sepconv2')(x)
x = layers.BatchNormalization(axis=channel_axis,
name='block14_sepconv2_bn')(x)
x = layers.Activation('relu', name='block14_sepconv2_act')(x)
f5 = x
if pretrained_weights_path is not None and os.path.exists(
pretrained_weights_path):
log('load pretrained encoder weights from `{}`'.format(
pretrained_weights_path))
Model(img_input, x,
name='xceptionv0_encoder').load_weights(pretrained_weights_path)
if output_stride == 8:
features = [f1, f2, f5, f5, f5]
elif output_stride == 16:
features = [f1, f2, f3, f5, f5]
else:
features = [f1, f2, f3, f4, f5]
return img_input, features
def resnet50(input, classes_num=1000, layer_num=50, is_extractor=False, output_layer_name = None, is_transfer_learning=False):
"""
ResNet50
:param input: 输入Keras.Input
:param is_extractor: 是否用于特征提取
:param layer_num: 可选40、50，40用于训练frcnn的时候速度过慢的问题
:return:
"""
bn_axis = 3
x = layers.ZeroPadding2D(padding=(3, 3), name='conv1_pad')(input)
# conv1 [64]*1
x = layers.Conv2D(64, (7, 7), strides=(2, 2), padding='valid', name='conv1')(x)
x = layers.BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = layers.Activation('relu')(x)
c1 = x
# 池化
x = layers.MaxPooling2D((3, 3), strides=(2, 2))(x)
c2 = x
# conv2 [64,64,256]*3
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
c3 = x
# conv3 [128,128,512]*4
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')
c4 = x
# conv4 [256,256,1024]*6
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')
c5 = x
# conv5 [512,512,2048]*3
if layer_num == 50:
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
c6 = x
# 确定fine-turning层
# outputs = [c1, c2, c3, c4, c5]
outputs = [c1, c2, c3, c4]
model = Model(input, outputs=outputs)
return model
# 用作特征提取器做迁移学习
if is_extractor:
# 冻结参数，停止学习
for l in no_train_model.layers:
l.trainable = False
# if isinstance(l, layers.BatchNormalization):
# l.trainable = True
# else:
# l.trainable = False
if output_layer_name:
return no_train_model.get_layer(output_layer_name).output
return x
elif is_transfer_learning:
x = layers.AveragePooling2D()(x)
x = layers.Flatten()(x)
preds = layers.Dense(units=classes_num, activation='softmax', kernel_initializer='he_normal')(x)
model = Model(input, preds, name='resnet50')
# 3 4 6 3=16 * 3 前2个block 21层冻结
for layer in model.layers[:21]:
layer.trainable = False
return model
# 完整CNN模型
else:
# x = layers.MaxPooling2D(pool_size=(7, 7))(x)
x = layers.AveragePooling2D()(x)
x = layers.Flatten()(x)
preds = layers.Dense(units=classes_num, activation='softmax', kernel_initializer='he_normal')(x)
return Model(input, preds, name='resnet50')
def build_model(self):
""" mapping of states to actions """
# defining input layer state
states = layers.Input(shape=(self.state_size, ), name='states')
# adding hidden layers
net = layers.Dense(units=32,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(states)
net = layers.BatchNormalization()(net)
net = layers.Activation('relu')(net)
net = layers.Dropout(0.4)(net)
net = layers.Dense(units=64,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net)
net = layers.BatchNormalization()(net)
net = layers.Activation('relu')(net)
net = layers.Dropout(0.4)(net)
net = layers.Dense(units=128,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net)
net = layers.BatchNormalization()(net)
net = layers.Activation('relu')(net)
net = layers.Dropout(0.4)(net)
net = layers.Dense(units=256,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net)
net = layers.BatchNormalization()(net)
net = layers.Activation('relu')(net)
net = layers.Dropout(0.4)(net)
net = layers.Dense(units=128,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net)
net = layers.BatchNormalization()(net)
net = layers.Activation('relu')(net)
net = layers.Dropout(0.4)(net)
# output_layer
raw_actions = layers.Dense(units=self.action_size,
activation='sigmoid',
name='raw_actions')(net)
actions = layers.Lambda(lambda x:
(x * self.action_range) + self.action_low,
name='actions')(raw_actions)
# keras model
self.model = models.Model(inputs=states, outputs=actions)
# loss function using action value (Q value) gradients
action_gradients = layers.Input(shape=(self.action_size, ))
loss = K.mean(-action_gradients * actions)
# Incorporate any additional losses here (e.g. from regularizers)
# Define optimizer and training function
optimizer = optimizers.Adam()
updates_op = optimizer.get_updates(params=self.model.trainable_weights,
loss=loss)
self.train_fn = K.function(
inputs=[self.model.input, action_gradients,
K.learning_phase()],
outputs=[],
updates=updates_op)
def xception(input_shape=(512, 512, 3),
input_tensor=None,
pretrained_weights_path=None,
output_stride=16):
img_input = input_tensor if input_tensor is not None else layers.Input(
shape=input_shape, name='img_input')
x = layers.BatchNormalization()(img_input)
if output_stride == 8:
entry_block3_stride = 1
middle_block_rate = 2 # ! Not mentioned in paper, but required
exit_block_rates = (2, 4)
exit_block1_stride = 1
elif output_stride == 16:
entry_block3_stride = 2
middle_block_rate = 1
exit_block_rates = (1, 2)
exit_block1_stride = 1
else:
entry_block3_stride = 2
middle_block_rate = 1
exit_block_rates = (1, 2)
exit_block1_stride = 2
x = layers.Conv2D(32, (3, 3),
strides=(2, 2),
name='entry_flow_conv1_1',
use_bias=False,
padding='same')(x)
x = layers.BatchNormalization(name='entry_flow_conv1_1_BN')(x)
x = layers.Activation('relu')(x)
x = _conv2d_same(x, 64, 'entry_flow_conv1_2', kernel_size=3, stride=1)
x = layers.BatchNormalization(name='entry_flow_conv1_2_BN')(x)
x = layers.Activation('relu')(x)
# entry flow
x, f1 = _xception_block(x, [128, 128, 128],
'entry_flow_block1',
skip_connection_type='conv',
stride=2,
depth_activation=False)
x, f2 = _xception_block(x, [256, 256, 256],
'entry_flow_block2',
skip_connection_type='conv',
stride=2,
depth_activation=False)
x, f3 = _xception_block(x, [728, 728, 728],
'entry_flow_block3',
skip_connection_type='conv',
stride=entry_block3_stride,
depth_activation=False)
# middle flow
for i in range(16):
x, _ = _xception_block(x, [728, 728, 728],
'middle_flow_unit_{}'.format(i + 1),
skip_connection_type='sum',
stride=1,
rate=middle_block_rate,
depth_activation=False)
# exit flow
x, f4 = _xception_block(x, [728, 1024, 1024],
'exit_flow_block1',
skip_connection_type='conv',
stride=exit_block1_stride,
rate=exit_block_rates[0],
depth_activation=False)
x, _ = _xception_block(x, [1536, 1536, 2048],
'exit_flow_block2',
skip_connection_type='none',
stride=1,
rate=exit_block_rates[1],
depth_activation=True)
f5 = x
if pretrained_weights_path is not None and os.path.exists(
pretrained_weights_path):
log('load pretrained encoder weights from `{}`'.format(
pretrained_weights_path))
Model(img_input, x,
name='xception_encoder').load_weights(pretrained_weights_path,
by_name=True,
skip_mismatch=True)
if output_stride == 8:
features = [f1, f2, f5, f5, f5]
elif output_stride == 16:
features = [f1, f2, f3, f5, f5]
else:
features = [f1, f2, f3, f4, f5]
return img_input, features
def EfficientNetV2(
width_coefficient,
depth_coefficient,
default_size,
dropout_rate=0.2,
drop_connect_rate=0.2,
depth_divisor=8,
min_depth=8,
bn_momentum=0.9,
activation="swish",
blocks_args="default",
model_name="efficientnetv2",
include_top=True,
weights="imagenet",
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
classifier_activation="softmax",
include_preprocessing=True,
):
"""Instantiates the EfficientNetV2 architecture using given scaling
coefficients.
Args:
width_coefficient: float, scaling coefficient for network width.
depth_coefficient: float, scaling coefficient for network depth.
default_size: integer, default input image size.
dropout_rate: float, dropout rate before final classifier layer.
drop_connect_rate: float, dropout rate at skip connections.
depth_divisor: integer, a unit of network width.
min_depth: integer, minimum number of filters.
bn_momentum: float. Momentum parameter for Batch Normalization layers.
activation: activation function.
blocks_args: list of dicts, parameters to construct block modules.
model_name: string, model name.
include_top: whether to include the fully-connected layer at the top of
the network.
weights: one of `None` (random initialization), `"imagenet"` (pre-training
on ImageNet), or the path to the weights file to be loaded.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) or
numpy array to use as image input for the model.
input_shape: optional shape tuple, only to be specified if `include_top`
is False. It should have exactly 3 inputs channels.
pooling: optional pooling mode for feature extraction when `include_top`
is `False`.
- `None` means that the output of the model will be the 4D tensor output
of the last convolutional layer.
- "avg" means that global average pooling will be applied to the output
of the last convolutional layer, and thus the output of the model will
be a 2D tensor.
- `"max"` means that global max pooling will be applied.
classes: optional number of classes to classify images into, only to be
specified if `include_top` is True, and if no `weights` argument is
specified.
classifier_activation: A string or callable. The activation function to
use on the `"top"` layer. Ignored unless `include_top=True`. Set
`classifier_activation=None` to return the logits of the `"top"` layer.
include_preprocessing: Boolean, whether to include the preprocessing layer
(`Rescaling`) at the bottom of the network. Defaults to `True`.
Returns:
A `keras.Model` instance.
Raises:
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
ValueError: if `classifier_activation` is not `"softmax"` or `None` when
using a pretrained top layer.
"""
if blocks_args == "default":
blocks_args = DEFAULT_BLOCKS_ARGS[model_name]
if not (weights in {"imagenet", None} or tf.io.gfile.exists(weights)):
raise ValueError("The `weights` argument should be either "
"`None` (random initialization), `imagenet` "
"(pre-training on ImageNet), "
"or the path to the weights file to be loaded."
f"Received: weights={weights}")
if weights == "imagenet" and include_top and classes != 1000:
raise ValueError(
"If using `weights` as `'imagenet'` with `include_top`"
" as true, `classes` should be 1000"
f"Received: classes={classes}")
# Determine proper input shape
input_shape = imagenet_utils.obtain_input_shape(
input_shape,
default_size=default_size,
min_size=32,
data_format=backend.image_data_format(),
require_flatten=include_top,
weights=weights,
)
if input_tensor is None:
img_input = layers.Input(shape=input_shape)
else:
if not backend.is_keras_tensor(input_tensor):
img_input = layers.Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
bn_axis = 3 if backend.image_data_format() == "channels_last" else 1
x = img_input
if include_preprocessing:
# Apply original V1 preprocessing for Bx variants
# if number of channels allows it
num_channels = input_shape[bn_axis - 1]
if model_name.split("-")[-1].startswith("b") and num_channels == 3:
x = layers.Rescaling(scale=1.0 / 255)(x)
x = layers.Normalization(
mean=[0.485, 0.456, 0.406],
variance=[0.229**2, 0.224**2, 0.225**2],
axis=bn_axis,
)(x)
else:
x = layers.Rescaling(scale=1.0 / 128.0, offset=-1)(x)
# Build stem
stem_filters = round_filters(
filters=blocks_args[0]["input_filters"],
width_coefficient=width_coefficient,
min_depth=min_depth,
depth_divisor=depth_divisor,
)
x = layers.Conv2D(
filters=stem_filters,
kernel_size=3,
strides=2,
kernel_initializer=CONV_KERNEL_INITIALIZER,
padding="same",
use_bias=False,
name="stem_conv",
)(x)
x = layers.BatchNormalization(
axis=bn_axis,
momentum=bn_momentum,
name="stem_bn",
)(x)
x = layers.Activation(activation, name="stem_activation")(x)
# Build blocks
blocks_args = copy.deepcopy(blocks_args)
b = 0
blocks = float(sum(args["num_repeat"] for args in blocks_args))
for (i, args) in enumerate(blocks_args):
assert args["num_repeat"] > 0
# Update block input and output filters based on depth multiplier.
args["input_filters"] = round_filters(
filters=args["input_filters"],
width_coefficient=width_coefficient,
min_depth=min_depth,
depth_divisor=depth_divisor,
)
args["output_filters"] = round_filters(
filters=args["output_filters"],
width_coefficient=width_coefficient,
min_depth=min_depth,
depth_divisor=depth_divisor,
)
# Determine which conv type to use:
block = {0: MBConvBlock, 1: FusedMBConvBlock}[args.pop("conv_type")]
repeats = round_repeats(repeats=args.pop("num_repeat"),
depth_coefficient=depth_coefficient)
for j in range(repeats):
# The first block needs to take care of stride and filter size
# increase.
if j > 0:
args["strides"] = 1
args["input_filters"] = args["output_filters"]
x = block(
activation=activation,
bn_momentum=bn_momentum,
survival_probability=drop_connect_rate * b / blocks,
name="block{}{}_".format(i + 1, chr(j + 97)),
**args,
)(x)
b += 1
# Build top
top_filters = round_filters(
filters=1280,
width_coefficient=width_coefficient,
min_depth=min_depth,
depth_divisor=depth_divisor,
)
x = layers.Conv2D(
filters=top_filters,
kernel_size=1,
strides=1,
kernel_initializer=CONV_KERNEL_INITIALIZER,
padding="same",
data_format="channels_last",
use_bias=False,
name="top_conv",
)(x)
x = layers.BatchNormalization(
axis=bn_axis,
momentum=bn_momentum,
name="top_bn",
)(x)
x = layers.Activation(activation=activation, name="top_activation")(x)
if include_top:
x = layers.GlobalAveragePooling2D(name="avg_pool")(x)
if dropout_rate > 0:
x = layers.Dropout(dropout_rate, name="top_dropout")(x)
imagenet_utils.validate_activation(classifier_activation, weights)
x = layers.Dense(
classes,
activation=classifier_activation,
kernel_initializer=DENSE_KERNEL_INITIALIZER,
bias_initializer=tf.constant_initializer(0),
name="predictions",
)(x)
else:
if pooling == "avg":
x = layers.GlobalAveragePooling2D(name="avg_pool")(x)
elif pooling == "max":
x = layers.GlobalMaxPooling2D(name="max_pool")(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = layer_utils.get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = training.Model(inputs, x, name=model_name)
# Load weights.
if weights == "imagenet":
if include_top:
file_suffix = ".h5"
file_hash = WEIGHTS_HASHES[model_name[-2:]][0]
else:
file_suffix = "_notop.h5"
file_hash = WEIGHTS_HASHES[model_name[-2:]][1]
file_name = model_name + file_suffix
weights_path = data_utils.get_file(
file_name,
BASE_WEIGHTS_PATH + file_name,
cache_subdir="models",
file_hash=file_hash,
)
model.load_weights(weights_path)
elif weights is not None:
model.load_weights(weights)
return model
def apply(inputs):
filters = input_filters * expand_ratio
if expand_ratio != 1:
x = layers.Conv2D(
filters,
kernel_size=kernel_size,
strides=strides,
kernel_initializer=CONV_KERNEL_INITIALIZER,
data_format="channels_last",
padding="same",
use_bias=False,
name=name + "expand_conv",
)(inputs)
x = layers.BatchNormalization(axis=bn_axis,
momentum=bn_momentum,
name=name + "expand_bn")(x)
x = layers.Activation(activation=activation,
name=name + "expand_activation")(x)
else:
x = inputs
# Squeeze and excite
if 0 < se_ratio <= 1:
filters_se = max(1, int(input_filters * se_ratio))
se = layers.GlobalAveragePooling2D(name=name + "se_squeeze")(x)
if bn_axis == 1:
se_shape = (filters, 1, 1)
else:
se_shape = (1, 1, filters)
se = layers.Reshape(se_shape, name=name + "se_reshape")(se)
se = layers.Conv2D(
filters_se,
1,
padding="same",
activation=activation,
kernel_initializer=CONV_KERNEL_INITIALIZER,
name=name + "se_reduce",
)(se)
se = layers.Conv2D(
filters,
1,
padding="same",
activation="sigmoid",
kernel_initializer=CONV_KERNEL_INITIALIZER,
name=name + "se_expand",
)(se)
x = layers.multiply([x, se], name=name + "se_excite")
# Output phase:
x = layers.Conv2D(
output_filters,
kernel_size=1 if expand_ratio != 1 else kernel_size,
strides=1 if expand_ratio != 1 else strides,
kernel_initializer=CONV_KERNEL_INITIALIZER,
padding="same",
use_bias=False,
name=name + "project_conv",
)(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=bn_momentum,
name=name + "project_bn")(x)
if expand_ratio == 1:
x = layers.Activation(activation=activation,
name=name + "project_activation")(x)
# Residual:
if strides == 1 and input_filters == output_filters:
if survival_probability:
x = layers.Dropout(
survival_probability,
noise_shape=(None, 1, 1, 1),
name=name + "drop",
)(x)
x = layers.add([x, inputs], name=name + "add")
return x
def get_model(params):
# Model definition
X_in = Input(shape=params["X_shape"])
A_in = Input(shape=params["A_shape"])
# E_in = Input(shape=params["E_shape"])
# aux_in = Input(shape=params["aux_shape"])
net = X_in
A_exp = layers.Lambda(lambda x: K.expand_dims(x, axis=-1))(A_in)
################################
# block
################################
net = RelationalDense(32)([net, A_exp])
# net = layers.BatchNormalization()(net)
net = layers.Activation("relu")(net)
net = MaxEdges()(net)
# net = EdgeConditionedConv(32)([X_in, A_in, E_in])
net = GraphConv(32)([net, A_in])
net = layers.BatchNormalization()(net)
net = layers.Activation("relu")(net)
################################
# block
################################
# net = RelationalDense(64)([net, A_exp])
# # net = layers.BatchNormalization()(net)
# net = layers.Activation("relu")(net)
# net = MaxEdges()(net)
# # net = EdgeConditionedConv(64)([net, A_in, E_in])
# net = GraphConv(128)([net, A_in])
# net = layers.BatchNormalization()(net)
# net = layers.Activation("relu")(net)
################################
# pooling
################################
net = GlobalAttentionPool(128)(net)
# net = GlobalMaxPool()(net)
net = layers.Dropout(0.5)(net)
################################
# block
################################
# concat = Concatenate()([dense1, aux_in])
net = Dense(64)(net)
net = layers.BatchNormalization()(net)
net = layers.Activation("relu")(net)
################################
# block
################################
output = Dense(1)(net)
################################
# model
################################
# Build model
# model = Model(inputs=[X_in, A_in, E_in], outputs=output)
model = Model(inputs=[X_in, A_in], outputs=output)
optimizer = Adam(lr=params["learning_rate"])
model.compile(
optimizer=optimizer,
loss="mse",
metrics=["mse", "mae", "mape"],
)
return model
def test_TimeDistributed():
# first, test with Dense layer
model = Sequential()
model.add(wrappers.TimeDistributed(layers.Dense(2), input_shape=(3, 4)))
model.add(layers.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.fit(np.random.random((10, 3, 4)),
np.random.random((10, 3, 2)),
epochs=1,
batch_size=10)
# test config
model.get_config()
# test when specifying a batch_input_shape
test_input = np.random.random((1, 3, 4))
test_output = model.predict(test_input)
weights = model.layers[0].get_weights()
reference = Sequential()
reference.add(
wrappers.TimeDistributed(layers.Dense(2), batch_input_shape=(1, 3, 4)))
reference.add(layers.Activation('relu'))
reference.compile(optimizer='rmsprop', loss='mse')
reference.layers[0].set_weights(weights)
reference_output = reference.predict(test_input)
assert_allclose(test_output, reference_output, atol=1e-05)
# test with Embedding
model = Sequential()
model.add(
wrappers.TimeDistributed(layers.Embedding(5, 6),
batch_input_shape=(10, 3, 4),
dtype='int32'))
model.compile(optimizer='rmsprop', loss='mse')
model.fit(np.random.randint(5, size=(10, 3, 4), dtype='int32'),
np.random.random((10, 3, 4, 6)),
epochs=1,
batch_size=10)
# compare to not using batch_input_shape
test_input = np.random.randint(5, size=(10, 3, 4), dtype='int32')
test_output = model.predict(test_input)
weights = model.layers[0].get_weights()
reference = Sequential()
reference.add(
wrappers.TimeDistributed(layers.Embedding(5, 6),
input_shape=(3, 4),
dtype='int32'))
reference.compile(optimizer='rmsprop', loss='mse')
reference.layers[0].set_weights(weights)
reference_output = reference.predict(test_input)
assert_allclose(test_output, reference_output, atol=1e-05)
# test with Conv2D
model = Sequential()
model.add(
wrappers.TimeDistributed(layers.Conv2D(5, (2, 2), padding='same'),
input_shape=(2, 4, 4, 3)))
model.add(layers.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.train_on_batch(np.random.random((1, 2, 4, 4, 3)),
np.random.random((1, 2, 4, 4, 5)))
model = model_from_json(model.to_json())
model.summary()
# test stacked layers
model = Sequential()
model.add(wrappers.TimeDistributed(layers.Dense(2), input_shape=(3, 4)))
model.add(wrappers.TimeDistributed(layers.Dense(3)))
model.add(layers.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.fit(np.random.random((10, 3, 4)),
np.random.random((10, 3, 3)),
epochs=1,
batch_size=10)
# test wrapping Sequential model
model = Sequential()
model.add(layers.Dense(3, input_dim=2))
outer_model = Sequential()
outer_model.add(wrappers.TimeDistributed(model, input_shape=(3, 2)))
outer_model.compile(optimizer='rmsprop', loss='mse')
outer_model.fit(np.random.random((10, 3, 2)),
np.random.random((10, 3, 3)),
epochs=1,
batch_size=10)
# test with functional API
x = Input(shape=(3, 2))
y = wrappers.TimeDistributed(model)(x)
outer_model = Model(x, y)
outer_model.compile(optimizer='rmsprop', loss='mse')
outer_model.fit(np.random.random((10, 3, 2)),
np.random.random((10, 3, 3)),
epochs=1,
batch_size=10)
# test with BatchNormalization
model = Sequential()
model.add(
wrappers.TimeDistributed(layers.BatchNormalization(center=True,
scale=True),
name='bn',
input_shape=(10, 2)))
model.compile(optimizer='rmsprop', loss='mse')
# Assert that mean and variance are 0 and 1.
td = model.layers[0]
assert np.array_equal(td.get_weights()[2], np.array([0, 0]))
assert np.array_equal(td.get_weights()[3], np.array([1, 1]))
# Train
model.train_on_batch(np.random.normal(loc=2, scale=2, size=(1, 10, 2)),
np.broadcast_to(np.array([0, 1]), (1, 10, 2)))
# Assert that mean and variance changed.
assert not np.array_equal(td.get_weights()[2], np.array([0, 0]))
assert not np.array_equal(td.get_weights()[3], np.array([1, 1]))
# Verify input_map has one mapping from inputs to reshaped inputs.
uid = _object_list_uid(model.inputs)
assert len(td._input_map.keys()) == 1
assert uid in td._input_map
assert K.int_shape(td._input_map[uid]) == (None, 2)
x_train /= 255
x_test /= 255
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
img_input = layers.Input(shape=input_shape)
model = layers.ZeroPadding2D(padding=(3,3))(img_input)
model = layers.Conv2D(64, kernel_size=(7,7), strides=(2,2), padding='valid', kernel_initializer='glorot_normal')(model)
model = layers.BatchNormalization(axis=3, momentum=0.9)(model)
model = layers.Activation('relu')(model)
model = layers.ZeroPadding2D(padding=(1,1))(model)
model = layers.MaxPooling2D((3,3), strides=(2,2))(model)
model_shortcut = model
# convulutional block
model = layers.Conv2D(64, kernel_size=(1,1), strides=(2,2), kernel_initializer=glorot_uniform(seed=None), padding='valid')(model)
model = layers.BatchNormalization(axis=3, momentum=0.9)(model)
model = layers.Activation('relu')(model)
model = layers.Conv2D(64, 3, strides=(1,1), kernel_initializer=glorot_uniform(seed=None), padding='same')(model)
model = layers.BatchNormalization(axis=3, momentum=0.9)(model)
model = layers.Activation('relu')(model)
model = layers.Conv2D(256, kernel_size=(1,1), strides=(1,1), kernel_initializer=glorot_uniform(seed=None), padding='valid')(model)
model = layers.BatchNormalization(axis=3, momentum=0.9)(model)
def build_mask_upsample_model(config):
"""Builds a Keras model of the Region Proposal Network.
It wraps the RPN graph so it can be used multiple times with shared
weights.
anchors_per_location: number of anchors per pixel in the feature map
anchor_stride: Controls the density of anchors. Typically 1 (anchors for
every pixel in the feature map), or 2 (every other pixel).
depth: Depth of the backbone feature map.
Returns a Keras Model object. The model outputs, when called, are:
rpn_class_logits: [batch, H * W * anchors_per_location, 2] Anchor classifier logits (before softmax)
rpn_probs: [batch, H * W * anchors_per_location, 2] Anchor classifier probabilities.
rpn_bbox: [batch, H * W * anchors_per_location, (dy, dx, log(dh), log(dw))] Deltas to be
applied to anchors.
"""
input_feature_maps = KL.Input(shape=[None, 1, 1, config.TOP_DOWN_PYRAMID_SIZE],
name="input_mask_feature_maps")
C1 = KL.Input(shape=[None, None, 64],
name="input_mask_refine_c1")
C2 = KL.Input(shape=[None, None, 256],
name="input_mask_refine_c2")
# deconv feature maps
x = KL.TimeDistributed(KL.Conv2D(256, (1, 1), strides=1),
name='mask_conv')(input_feature_maps)
x = KL.TimeDistributed(BatchNorm(),
name='mask_conv_bn')(x, training = config.TRAIN_BN)
x = KL.Activation('relu')(x)
deconv_features = KL.TimeDistributed(
KL.Conv2DTranspose(64, 32, strides = (32, 32)),
name = "mask_deconv")(x)
# U3
u3 = KL.Conv2D(64, (3, 3), padding = "same", activation = "relu",
name='mask_u3_conv1')(C2)
u3 = KL.Conv2D(32, (3, 3), padding = "same", activation = "relu",
name='mask_u3_conv2')(u3)
u3 = KL.Conv2D(16, (3, 3), padding = "same",
name='mask_u3_conv3')(u3)
u3 = KL.Lambda(
lambda t: tf.expand_dims(t, 1))(u3)
h3 = KL.TimeDistributed(
KL.Conv2D(16, (3, 3), padding = "same", activation = "relu"),
name='mask_h3_conv1')(deconv_features)
h3 = KL.TimeDistributed(
KL.Conv2D(16, (3, 3), padding = "same"),
name='mask_h3_conv2')(h3)
u3 = KL.Add(name = 'mask_u3_add')([u3, h3])
u3 = KL.Activation('relu')(u3)
u3 = KL.TimeDistributed(
#CustomUpscale2D([61, 61]),
KL.Conv2DTranspose(8, 3, strides = (2, 2),
padding="same"),
name = 'mask_u3_up')(u3)
# U4
u4 = KL.Conv2D(32, (3, 3), padding = "same", activation = "relu",
name='mask_u4_conv1')(C1)
u4 = KL.Conv2D(16, (3, 3), padding = "same", activation = "relu",
name='mask_u4_conv2')(u4)
u4 = KL.Conv2D(8, (3, 3), padding = "same",
name='mask_u4_conv3')(u4)
u4 = KL.Lambda(
lambda t: tf.expand_dims(t, 1))(u4)
h4 = KL.TimeDistributed(
KL.Conv2D(8, (3, 3), padding = "same", activation = "relu"),
name='mask_h4_conv1')(u3)
h4 = KL.TimeDistributed(
KL.Conv2D(8, (3, 3), padding = "same"),
name='mask_h4_conv2')(h4)
u4 = KL.Add(name = 'mask_u4_add')([u4, h4])
u4 = KL.Activation('relu')(u4)
u4 = KL.TimeDistributed(
#CustomUpscale2D([127, 127]),
KL.Conv2DTranspose(4, 7, strides = (2, 2),
padding="same"),
#output_padding = (1, 1)),
name = 'mask_u4_up')(u4)
rpn_masks = KL.TimeDistributed(
KL.Conv2D(1, (3, 3), strides=1,
padding = 'same',
activation="sigmoid"),
name="rpn_masks")(u4)
return KM.Model([input_feature_maps, C1, C2], rpn_masks, name="mask_unet_model")
encoded_document = layers.Bidirectional(
layers.LSTM(int(EMBEDDINGS_SIZE/2),
activation='tanh',
recurrent_activation='tanh',
return_sequences=True))\
(encoded_document)
# Size: 554
encoded_document = layers.Conv1D(filters=128,
kernel_size=16,
strides=2,
activation='relu')(encoded_document)
# Size: 270
encoded_document = layers.MaxPool1D(pool_size=2)(encoded_document)
encoded_document = layers.Activation('relu')(encoded_document)
# Size: 135
encoded_document = layers.Conv1D(filters=128,
kernel_size=16,
strides=2,
activation='relu')(encoded_document)
# Size: 60
encoded_document = layers.MaxPool1D(pool_size=2)(encoded_document)
encoded_document = layers.Activation('relu')(encoded_document)
# Size: 30
encoded_document = layers.Conv1D(filters=128,
kernel_size=4,
strides=1,
activation='relu')(encoded_document)
# Size: 27
encoded_document = layers.MaxPool1D(pool_size=2)(encoded_document)
if resume_model:
model = expr.load_weight_and_training_config_and_state()
expr.printdebug("Checkpoint found. Resuming model at %s" %
expr.dir_lasttime)
else:
###############################
# Architecture of the network #
###############################
# First channel: image
x_inputs = L.Input(shape=SHAPE)
x = x_inputs #inputs is used by the line "Model(inputs, ... )" below
conv11 = L.Conv2D(32, (8, 8), strides=4, padding='valid')
x = conv11(x)
x = L.Activation('relu')(x)
x = L.BatchNormalization()(x)
x = L.Dropout(dropout)(x)
conv12 = L.Conv2D(64, (4, 4), strides=2, padding='valid')
x = conv12(x)
x = L.Activation('relu')(x)
x = L.BatchNormalization()(x)
x = L.Dropout(dropout)(x)
conv13 = L.Conv2D(64, (3, 3), strides=1, padding='valid')
x = conv13(x)
x = L.Activation('relu')(x)
x = L.BatchNormalization()(x)
x = L.Dropout(dropout)(x)
def resnet18(input_shape, num_classes, dtype='float32'):
"""Instantiates the ResNet architecture.
Args:
input_shape: It should have exactly 3 inputs channels,
and width and height should be no smaller than 32.
E.g. `(200, 200, 3)` would be one valid value.
num_classes: `int` number of classes for image classification.
Returns:
A Keras model instance.
"""
# input_shape = (224, 224, 3)
img_input = layers.Input(shape=input_shape)
if backend.image_data_format() == 'channels_first':
x = layers.Lambda(lambda x: backend.permute_dimensions(x, (0, 3, 1, 2)),
name='transpose')(img_input)
bn_axis = 1
else: # channels_last
x = img_input
bn_axis = 3
x = layers.ZeroPadding2D(padding=(3, 3), name='conv1_pad')(x)
x = layers.Conv2D(64, (7, 7),
strides=(2, 2),
padding='valid', use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name='conv1')(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name='bn_conv1')(x)
x = layers.Activation('relu')(x)
x = layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
x = layers.GlobalAveragePooling2D(name='avg_pool')(x)
x = layers.Dense(
num_classes,
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
bias_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name='fc1000')(x)
x = backend.cast(x, 'float32')
x = layers.Activation('softmax')(x)
# Create model.
return models.Model(img_input, x, name='resnet18')
# RNN for each time step. Repeat 'DIGITS + 1' times as that's the maximum
# length of output, e.g., when DIGITS=3, max output is 999+999=1998.
model.add(layers.Dropout(0.5))
model.add(layers.RepeatVector(SENTLEN))
# The decoder RNN could be multiple layers stacked or a single layer.
for _ in range(LAYERS):
# By setting return_sequences to True, return not only the last output but
# all the outputs so far in the form of (num_samples, timesteps,
# output_dim). This is necessary as TimeDistributed in the below expects
# the first dimension to be the timesteps.
model.add(RNN(HIDDEN_SIZE, return_sequences=True))
# Apply a dense layer to the every temporal slice of an input. For each of step
# of the output sequence, decide which character should be chosen.
model.add(layers.TimeDistributed(layers.Dense(len(ctable_y.chars))))
model.add(layers.Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy', 'sparse_categorical_accuracy'])
model.summary()
# Train the model each generation and show predictions against the validation
# dataset.
for iteration in range(1, ITERATIONS + 1):
print()
print('-' * 50)
print('Iteration', iteration)
history = model.fit(x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=EPOCHS,
def conv_block(input_tensor,
kernel_size,
filters,
stage,
block,
strides=(2, 2)):
"""A block that has a conv layer at shortcut.
# Arguments
input_tensor: input tensor
kernel_size: default 3, the kernel size of
middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
strides: Strides for the second conv layer in the block.
# Returns
Output tensor for the block.
Note that from stage 3,
the second conv layer at main path is with strides=(2, 2)
And the shortcut should have strides=(2, 2) as well
"""
filters1, filters2, filters3 = filters
if backend.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
x = layers.Conv2D(filters1, (1, 1), use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2a')(input_tensor)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2a')(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(filters2, kernel_size, strides=strides, padding='same',
use_bias=False, kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2b')(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2b')(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(filters3, (1, 1), use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2c')(x)
x = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2c')(x)
shortcut = layers.Conv2D(filters3, (1, 1), strides=strides, use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '1')(input_tensor)
shortcut = layers.BatchNormalization(axis=bn_axis,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '1')(shortcut)
x = layers.add([x, shortcut])
x = layers.Activation('relu')(x)
return x
def get_model():
inpl = layers.Input((360, 640, 3))
inpr = layers.Input((360, 640, 3))
inpc = layers.Input((384, 640, 3))
# xl = layers.Conv2D(32, 3, strides=(2, 2))(inpl)
# xl = layers.BatchNormalization()(xl)
# xl = layers.Activation('relu')(xl)
# xl = layers.Conv2D(32, 3)(xl)
# xl = layers.BatchNormalization()(xl)
# xl = layers.Activation('relu')(xl)
# xl = layers.MaxPooling2D()(xl)
#
# xr = layers.Conv2D(32, 3, strides=(2, 2))(inpr)
# xr = layers.BatchNormalization()(xr)
# xr = layers.Activation('relu')(xr)
# xr = layers.Conv2D(32, 3)(xr)
# xr = layers.BatchNormalization()(xr)
# xr = layers.Activation('relu')(xr)
# xr = layers.MaxPooling2D()(xr)
xc = layers.Conv2D(32, 3, 2)(inpc)
xc = layers.BatchNormalization()(xc)
xc = layers.Activation('relu')(xc)
xc = layers.Conv2D(32, 3)(xc)
xc = layers.BatchNormalization()(xc)
xc = layers.Activation('relu')(xc)
x = layers.MaxPooling2D()(xc)
# x = layers.concatenate((xl, xc, xr))
# x = layers.concatenate([xl, xr])
x = layers.Conv2D(96, 3, strides=(2, 2))(x)
x = layers.BatchNormalization()(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(128, 3)(x)
x = layers.BatchNormalization()(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(128, 3)(x)
x = layers.BatchNormalization()(x)
x = layers.Activation('relu')(x)
x = layers.MaxPooling2D()(x)
x = layers.Conv2D(256, 3)(x)
x = layers.BatchNormalization()(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(256, 3)(x)
x = layers.BatchNormalization()(x)
x = layers.Activation('relu')(x)
x = layers.MaxPooling2D()(x)
x = layers.Conv2D(512, 3)(x)
x = layers.BatchNormalization()(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(512, 3)(x)
x = layers.BatchNormalization()(x)
x = layers.Activation('relu')(x)
# x = layers.GlobalAveragePooling2D()(x)
x = layers.Flatten()(x)
x = layers.Dense(2, activation='sigmoid')(x)
# model = models.Model([inpl, inpc, inpr], x)
# model = models.Model([inpl, inpr], x)
model = models.Model(inpc, x)
model.compile(optimizer='adam', loss='mae')
return model
def conv_block(input_tensor,
kernel_size,
filters,
stage,
block,
strides=(2, 2)):
"""A block that has a conv layer at shortcut.
# Arguments
input_tensor: input tensor
kernel_size: default 3, the kernel size of
middle conv layer at main path
filters: list of integers, the filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
strides: Strides for the first conv layer in the block.
# Returns
Output tensor for the block.
Note that from stage 3,
the first conv layer at main path is with strides=(2, 2)
And the shortcut should have strides=(2, 2) as well
"""
filters1, filters2, filters3 = filters
conv_name_base = 'conv' + str(stage) + '_' + str(block)
bn_name_base = 'bn' + str(stage) + '_' + str(block)
x = layers.Conv2D(filters1, (1, 1),
strides=strides,
kernel_initializer='he_normal',
name=conv_name_base + '_1')(input_tensor)
x = layers.BatchNormalization(epsilon=1e-5,
momentum=1,
name=bn_name_base + '_1')(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(filters2,
kernel_size,
padding='same',
kernel_initializer='he_normal',
name=conv_name_base + '_2')(x)
x = layers.BatchNormalization(epsilon=1e-5,
momentum=1,
name=bn_name_base + '_2')(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(filters3, (1, 1),
kernel_initializer='he_normal',
name=conv_name_base + '_3')(x)
x = layers.BatchNormalization(epsilon=1e-5,
momentum=1,
name=bn_name_base + '_3')(x)
shortcut = layers.Conv2D(filters3, (1, 1),
strides=strides,
kernel_initializer='he_normal',
name='relu_' + str(stage))(input_tensor)
shortcut = layers.BatchNormalization(epsilon=1e-5,
momentum=1,
name='relubn_' + str(stage))(shortcut)
x = layers.add([x, shortcut])
x = layers.Activation('relu')(x)
return x
def build_model(self):
"""Build an actor (policy) network that maps states -> actions."""
# Define input layer (states)
states = layers.Input(shape=(self.state_size, ), name='states')
# Add hidden layers
net = layers.Dense(units=32,
use_bias=False,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(states)
net = layers.BatchNormalization()(net)
net = layers.Activation('relu')(net)
net = layers.Dropout(0.5)(net)
net = layers.Dense(units=64,
use_bias=False,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net)
net = layers.BatchNormalization()(net)
net = layers.Activation('relu')(net)
net = layers.Dropout(0.5)(net)
net = layers.Dense(units=128,
use_bias=False,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net)
net = layers.BatchNormalization()(net)
net = layers.Activation('relu')(net)
net = layers.Dropout(0.5)(net)
net = layers.Dense(units=64,
use_bias=False,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net)
net = layers.BatchNormalization()(net)
net = layers.Activation('relu')(net)
net = layers.Dropout(0.5)(net)
# Try different layer sizes, activations, add batch normalization, regularizers, etc.
# Add final output layer with sigmoid activation
raw_actions = layers.Dense(units=self.action_size,
activation='sigmoid',
name='raw_actions')(net)
# Scale [0, 1] output for each action dimension to proper range
actions = layers.Lambda(lambda x:
(x * self.action_range) + self.action_low,
name='actions')(raw_actions)
# Create Keras model
self.model = models.Model(inputs=states, outputs=actions)
# Define loss function using action value (Q value) gradients
action_gradients = layers.Input(shape=(self.action_size, ))
loss = K.mean(-action_gradients * actions)
# Incorporate any additional losses here (e.g. from regularizers)
# Define optimizer and training function CHECK LEARNINING RATE NEEDED!!!!!
optimizer = optimizers.Adam()
updates_op = optimizer.get_updates(params=self.model.trainable_weights,
loss=loss)
self.train_fn = K.function(
inputs=[self.model.input, action_gradients,
K.learning_phase()],
outputs=[],
updates=updates_op)
def resnet_3d_graph(input_clip,
mode,
architecture,
stage5=False,
temporal=True):
# input_clip: [bs, timesteps, height, width, channel]
# TODO: add a param in config indicating which stages contain temporal layers
assert mode in ['training', 'inference']
training = True if mode == 'training' else False
assert architecture in ["resnet50", "resnet101"]
# Stage 1
x = KL.ZeroPadding3D((0, 3, 3))(input_clip)
x = KL.TimeDistributed(KL.Conv2D(64, (7, 7), strides=(2, 2),
use_bias=True),
name='conv1')(x)
x = TD(BatchNorm(axis=3), name='bn_conv1')(x)
x = KL.Activation('relu')(x)
C1 = x = KL.TimeDistributed(
KL.MaxPooling2D((3, 3), strides=(2, 2), padding='same'))(x)
# Stage 2
x = conv_block_3d(x,
3,
3, [64, 64, 256],
stage=2,
block='a',
training=training,
strides=(1, 1))
x = identity_block_3d(x,
3,
3, [64, 64, 256],
stage=2,
block='b',
training=training)
C2 = x = identity_block_3d(x,
3,
3, [64, 64, 256],
stage=2,
block='c',
training=training)
# Stage 3
x = conv_block_3d(x,
3,
3, [128, 128, 512],
stage=3,
block='a',
training=training)
x = identity_block_3d(x,
3,
3, [128, 128, 512],
stage=3,
block='b',
training=training)
x = identity_block_3d(x,
3,
3, [128, 128, 512],
stage=3,
block='c',
training=training)
C3 = x = identity_block_3d(x,
3,
3, [128, 128, 512],
stage=3,
block='d',
training=training)
# Stage 4
x = conv_block_3d(x,
3,
3, [256, 256, 1024],
stage=4,
block='a',
training=training)
block_count = {"resnet50": 5, "resnet101": 22}[architecture]
for i in range(block_count):
x = identity_block_3d(x,
3,
3, [256, 256, 1024],
stage=4,
block=chr(98 + i),
training=training,
temporal=temporal)
C4 = x
# Stage 5
if stage5:
x = conv_block_3d(x,
3,
3, [512, 512, 2048],
stage=5,
block='a',
training=training,
temporal=temporal)
x = identity_block_3d(x,
3,
3, [512, 512, 2048],
stage=5,
block='b',
training=training,
temporal=temporal)
C5 = x = identity_block_3d(x,
3,
3, [512, 512, 2048],
stage=5,
block='c',
training=training,
temporal=temporal)
else:
C5 = None
return [C1, C2, C3, C4, C5]
def Xception(include_top=False,
weights=None,
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
Div=8,
**kwargs):
"""Instantiates the Xception architecture.
Optionally loads weights pre-trained on ImageNet. This model can
only be used with the data format `(width, height, channels)`.
You should set `image_data_format='channels_last'` in your Keras config
located at ~/.keras/keras.json.
Note that the default input image size for this model is 299x299.
# Arguments
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization),
'imagenet' (pre-training on ImageNet),
or the path to the weights file to be loaded.
input_tensor: optional Keras tensor
(i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(299, 299, 3)`.
It should have exactly 3 inputs channels,
and width and height should be no smaller than 71.
E.g. `(150, 150, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True,
and if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
"""
#backend, layers, models, keras_utils = get_submodules_from_kwargs(kwargs)
strides1 = 2
strides2 = 2
strides3 = 2
if Div == 16:
strides1 = 1
elif Div == 8:
strides1 = 1
strides2 = 1
elif Div == 4:
strides1 = 1
strides2 = 1
strides3 = 1
if not (weights in {'imagenet', None} or os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError(
'If using `weights` as `"imagenet"` with `include_top`'
' as true, `classes` should be 1000')
if backend.image_data_format() != 'channels_last':
warnings.warn('The Xception model is only available for the '
'input data format "channels_last" '
'(width, height, channels). '
'However your settings specify the default '
'data format "channels_first" '
'(channels, width, height). '
'You should set `image_data_format="channels_last"` '
'in your Keras '
'config located at ~/.keras/keras.json. '
'The model being returned right now will expect inputs '
'to follow the "channels_last" data format.')
backend.set_image_data_format('channels_last')
old_data_format = 'channels_first'
else:
old_data_format = None
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=299,
min_size=71,
data_format=backend.image_data_format(),
require_flatten=False,
weights=weights)
if input_tensor is None:
img_input = layers.Input(shape=input_shape)
else:
if not backend.is_keras_tensor(input_tensor):
img_input = layers.Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
x = layers.Conv2D(32, (3, 3),
strides=(strides1, strides1),
use_bias=False,
padding='same',
name='block1_conv1')(img_input)
x = layers.BatchNormalization(name='block1_conv1_bn')(x)
x = layers.Activation('relu', name='block1_conv1_act')(x)
x = layers.Conv2D(64, (3, 3),
use_bias=False,
padding='same',
name='block1_conv2')(x)
x = layers.BatchNormalization(name='block1_conv2_bn')(x)
x = layers.Activation('relu', name='block1_conv2_act')(x)
residual = layers.Conv2D(128, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = layers.BatchNormalization()(residual)
x = layers.SeparableConv2D(128, (3, 3),
padding='same',
use_bias=False,
name='block2_sepconv1')(x)
x = layers.BatchNormalization(name='block2_sepconv1_bn')(x)
x = layers.Activation('relu', name='block2_sepconv2_act')(x)
x = layers.SeparableConv2D(128, (3, 3),
strides=(1, 1),
padding='same',
use_bias=False,
name='block2_sepconv2')(x)
x = layers.BatchNormalization(name='block2_sepconv2_bn')(x)
x = layers.MaxPooling2D((2, 2),
strides=(2, 2),
padding='same',
name='block2_pool')(x)
x = layers.add([x, residual])
residual = layers.Conv2D(256, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = layers.BatchNormalization()(residual)
x = layers.Activation('relu', name='block3_sepconv1_act')(x)
x = layers.SeparableConv2D(256, (3, 3),
padding='same',
use_bias=False,
name='block3_sepconv1')(x)
x = layers.BatchNormalization(name='block3_sepconv1_bn')(x)
x = layers.Activation('relu', name='block3_sepconv2_act')(x)
x = layers.SeparableConv2D(256, (3, 3),
strides=(1, 1),
padding='same',
use_bias=False,
name='block3_sepconv2')(x)
x = layers.BatchNormalization(name='block3_sepconv2_bn')(x)
x = layers.MaxPooling2D((2, 2),
strides=(2, 2),
padding='same',
name='block3_pool')(x)
x = layers.add([x, residual])
residual = layers.Conv2D(728, (1, 1),
strides=(strides3, strides3),
padding='same',
use_bias=False)(x)
residual = layers.BatchNormalization()(residual)
x = layers.Activation('relu', name='block4_sepconv1_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name='block4_sepconv1')(x)
x = layers.BatchNormalization(name='block4_sepconv1_bn')(x)
x = layers.Activation('relu', name='block4_sepconv2_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
strides=(1, 1),
padding='same',
use_bias=False,
name='block4_sepconv2')(x)
x = layers.BatchNormalization(name='block4_sepconv2_bn')(x)
#x = layers.MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='block4_pool')(x)
if strides3 == 1:
x = layers.MaxPooling2D((1, 1),
strides=(1, 1),
padding='same',
name='block4_pool')(x)
else:
x = layers.MaxPooling2D((2, 2),
strides=(2, 2),
padding='same',
name='block4_pool')(x)
x = layers.add([x, residual])
for i in range(8):
residual = x
prefix = 'block' + str(i + 5)
x = layers.Activation('relu', name=prefix + '_sepconv1_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name=prefix + '_sepconv1')(x)
x = layers.BatchNormalization(name=prefix + '_sepconv1_bn')(x)
x = layers.Activation('relu', name=prefix + '_sepconv2_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name=prefix + '_sepconv2')(x)
x = layers.BatchNormalization(name=prefix + '_sepconv2_bn')(x)
x = layers.Activation('relu', name=prefix + '_sepconv3_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name=prefix + '_sepconv3')(x)
x = layers.BatchNormalization(name=prefix + '_sepconv3_bn')(x)
x = layers.add([x, residual], name=prefix + '_add')
residual = layers.Conv2D(1024, (1, 1),
strides=(strides2, strides2),
padding='same',
use_bias=False)(x)
residual = layers.BatchNormalization()(residual)
x = layers.Activation('relu', name='block13_sepconv1_act')(x)
x = layers.SeparableConv2D(728, (3, 3),
padding='same',
use_bias=False,
name='block13_sepconv1')(x)
x = layers.BatchNormalization(name='block13_sepconv1_bn')(x)
x = layers.Activation('relu', name='block13_sepconv2_act')(x)
x = layers.SeparableConv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
use_bias=False,
name='block13_sepconv2')(x)
x = layers.BatchNormalization(name='block13_sepconv2_bn')(x)
if strides2 == 1:
x = layers.MaxPooling2D((1, 1),
strides=(1, 1),
padding='same',
name='block13_pool')(x)
else:
x = layers.MaxPooling2D((2, 2),
strides=(2, 2),
padding='same',
name='block13_pool')(x)
x = layers.add([x, residual], name='block13_add')
x = layers.SeparableConv2D(1536, (3, 3),
padding='same',
use_bias=False,
name='block14_sepconv1')(x)
x = layers.BatchNormalization(name='block14_sepconv1_bn')(x)
x = layers.Activation('relu', name='block14_sepconv1_act')(x)
x = layers.SeparableConv2D(2048, (3, 3),
padding='same',
use_bias=False,
name='block14_sepconv2')(x)
x = layers.BatchNormalization(name='block14_sepconv2_bn')(x)
x = layers.Activation('relu', name='block14_sepconv2_act')(x)
if include_top:
x = layers.GlobalAveragePooling2D(name='avg_pool')(x)
x = layers.Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = layers.GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = layers.GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = keras_utils.get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = models.Model(inputs, x, name='xception')
# Load weights.
if weights == 'imagenet':
if include_top:
weights_path = keras_utils.get_file(
'xception_weights_tf_dim_ordering_tf_kernels.h5',
TF_WEIGHTS_PATH,
cache_subdir='models',
file_hash='0a58e3b7378bc2990ea3b43d5981f1f6')
else:
weights_path = keras_utils.get_file(
'xception_weights_tf_dim_ordering_tf_kernels_notop.h5',
TF_WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
file_hash='b0042744bf5b25fce3cb969f33bebb97')
model.load_weights(weights_path)
print('you load imagenet weights...')
if backend.backend() == 'theano':
keras_utils.convert_all_kernels_in_model(model)
elif weights is not None:
model.load_weights(weights)
if old_data_format:
backend.set_image_data_format(old_data_format)
return model
def conv_block_3d(input_tensor,
s_kernel_size,
t_kernel_size,
filters,
stage,
block,
training,
strides=(2, 2),
use_bias=True,
temporal=False):
"""conv_block is the block that has a conv layer at shortcut
# Arguments
input_tensor: input tensor
s_kernel_size: defualt 3, the kernel size of middle spatial conv layer at main path
filters: list of integers, the nb_filters of 3 conv layer at main path
stage: integer, current stage label, used for generating layer names
block: 'a','b'..., current block label, used for generating layer names
Note that from stage 3, the first conv layer at main path is with subsample=(2,2)
And the shortcut should have subsample=(2,2) as well
"""
nb_filter1, nb_filter2, nb_filter3 = filters
spatial_conv_name_base = 'res' + str(stage) + block + '_branch'
temporal_conv_name_base = 'tres' + str(stage) + block + '_branch'
spatial_bn_name_base = 'bn' + str(stage) + block + '_branch'
temporal_bn_name_base = 'tbn' + str(stage) + block + '_branch'
# 1x1x1
x = KL.TimeDistributed(KL.Conv2D(nb_filter1, (1, 1),
strides=strides,
use_bias=use_bias),
name=spatial_conv_name_base + '2a')(input_tensor)
x = TD(BatchNorm(axis=3), name=spatial_bn_name_base + '2a')(x)
x = KL.Activation('relu')(x)
# 1xkxk
x = KL.TimeDistributed(KL.Conv2D(nb_filter2,
(s_kernel_size, s_kernel_size),
padding='same'),
name=spatial_conv_name_base + '2b')(x)
x = TD(BatchNorm(axis=3), name=spatial_bn_name_base + '2b')(x)
x = KL.Activation('relu')(x)
if temporal:
# kx1x1 # TODO: add back
x = KL.Conv3D(nb_filter2, (t_kernel_size, 1, 1),
padding='same',
name=temporal_conv_name_base + '2b')(x)
#x = KL.BatchNormalization(axis=4, name=temporal_bn_name_base + '2b')(x, training=training)
x = KL.Activation('relu')(x)
# 1x1x1
x = KL.TimeDistributed(KL.Conv2D(nb_filter3, (1, 1), use_bias=use_bias),
name=spatial_conv_name_base + '2c')(x)
x = TD(BatchNorm(axis=3), name=spatial_bn_name_base + '2c')(x)
# Add shortcut
shortcut = KL.TimeDistributed(KL.Conv2D(nb_filter3, (1, 1),
strides=strides,
use_bias=use_bias),
name=spatial_conv_name_base +
'1')(input_tensor)
shortcut = TD(BatchNorm(axis=3), name=spatial_bn_name_base + '1')(shortcut)
x = KL.Add()([x, shortcut])
x = KL.Activation('relu', name='res' + str(stage) + block + '_out')(x)
return x
def ResNet50(input_shape=(224, 224, 3), n_classes=1000,
l2_reg=5e-5, bn_mom=0.9):
"""Instantiates the ResNet50 architecture.
# Arguments
input_shape: input shape tuple. It should have 3 input channels.
n_classes: number of classes to classify images.
l2_reg: L2 weight regularization (weight decay)
bn_mom: batch-norm momentum
# Returns
A Keras model instance.
"""
img_input = layers.Input(shape=input_shape)
if backend.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
x = layers.ZeroPadding2D(padding=(3, 3), name='conv1_pad')(img_input)
x = layers.Conv2D(64, (7, 7), strides=(2, 2), padding='valid',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(l2_reg),
name='conv1')(x)
x = layers.BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = layers.Activation('relu')(x)
x = layers.ZeroPadding2D(padding=(1, 1), name='pool1_pad')(x)
x = layers.MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1),
l2_reg=l2_reg, bn_mom=bn_mom)
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b',
l2_reg=l2_reg, bn_mom=bn_mom)
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c',
l2_reg=l2_reg, bn_mom=bn_mom)
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a',
l2_reg=l2_reg, bn_mom=bn_mom)
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b',
l2_reg=l2_reg, bn_mom=bn_mom)
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c',
l2_reg=l2_reg, bn_mom=bn_mom)
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d',
l2_reg=l2_reg, bn_mom=bn_mom)
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a',
l2_reg=l2_reg, bn_mom=bn_mom)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b',
l2_reg=l2_reg, bn_mom=bn_mom)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c',
l2_reg=l2_reg, bn_mom=bn_mom)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d',
l2_reg=l2_reg, bn_mom=bn_mom)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e',
l2_reg=l2_reg, bn_mom=bn_mom)
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f',
l2_reg=l2_reg, bn_mom=bn_mom)
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a',
l2_reg=l2_reg, bn_mom=bn_mom)
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b',
l2_reg=l2_reg, bn_mom=bn_mom)
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c',
l2_reg=l2_reg, bn_mom=bn_mom)
x = layers.GlobalAveragePooling2D(name='avg_pool')(x)
x = layers.Dense(n_classes, activation='softmax',
kernel_regularizer=regularizers.l2(l2_reg),
name='fc1000')(x)
return models.Model(img_input, x, name='resnet50')
def construct_graph(self, input_tensor, stage5=True):
assert self.input_tensor is not None, "input_tensor can not be none!"
# Stage 1
x = KL.ZeroPadding2D((3, 3))(input_tensor)
x = KL.Conv2D(64, (7, 7), strides=(2, 2), name='conv1',
use_bias=True)(x)
x = BatchNorm(axis=3, name='bn_conv1')(x)
x = KL.Activation('relu')(x)
C1 = x = KL.MaxPooling2D((3, 3), strides=(2, 2), padding="same")(x)
# Stage 2
x = self.conv_block(x,
3, [64, 64, 256],
stage=2,
block='a',
strides=(1, 1))
x = self.identity_block(x, 3, [64, 64, 256], stage=2, block='b')
C2 = x = self.identity_block(x, 3, [64, 64, 256], stage=2, block='c')
# Stage 3
x = self.conv_block(x, 3, [128, 128, 512], stage=3, block='a')
x = self.identity_block(x, 3, [128, 128, 512], stage=3, block='b')
x = self.identity_block(x, 3, [128, 128, 512], stage=3, block='c')
C3 = x = self.identity_block(x, 3, [128, 128, 512], stage=3, block='d')
# Stage 4
x = self.conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
block_count = {"resnet50": 5, "resnet101": 22}[self.architecture]
for i in range(block_count):
x = self.identity_block(x,
3, [256, 256, 1024],
stage=4,
block=chr(98 + i))
C4 = x
# Stage 5
x = self.conv_block(x,
3, [256, 256, 256],
stage=5,
block='a',
dilated=2,
strides=(1, 1))
x = self.identity_block(x,
3, [256, 256, 256],
stage=5,
block='b',
dilated=2)
C5 = x = self.identity_block(x,
3, [256, 256, 256],
stage=5,
block='c',
dilated=2)
# Stage 6
x = self.conv_block(x,
3, [256, 256, 256],
stage=6,
block='a',
dilated=2,
strides=(1, 1))
x = self.identity_block(x,
3, [256, 256, 256],
stage=6,
block='b',
dilated=2)
C6 = x = self.identity_block(x,
3, [256, 256, 256],
stage=6,
block='c',
dilated=2)
P6 = KL.Conv2D(256, (1, 1), name='fpn_c6p6')(C6)
P5 = KL.Add(name="fpn_p5add")(
[P6, KL.Conv2D(256, (1, 1), name='fpn_c5p5')(C5)])
P4 = KL.Add(name="fpn_p4add")(
[P5, KL.Conv2D(256, (1, 1), name='fpn_c4p4')(C4)])
P3 = KL.Add(name="fpn_p3add")([
KL.UpSampling2D(size=(2, 2), name="fpn_p4upsampled")(P4),
KL.Conv2D(256, (1, 1), name='fpn_c3p3')(C3)
])
P2 = KL.Add(name="fpn_p2add")([
KL.UpSampling2D(size=(2, 2), name="fpn_p3upsampled")(P3),
KL.Conv2D(256, (1, 1), name='fpn_c2p2')(C2)
])
# Attach 3x3 conv to all P layers to get the final feature maps.
P2 = KL.Conv2D(256, (3, 3), padding="SAME", name="fpn_p2")(P2)
P3 = KL.Conv2D(256, (3, 3), padding="SAME", name="fpn_p3")(P3)
P4 = KL.Conv2D(256, (3, 3), padding="SAME", name="fpn_p4")(P4)
P5 = KL.Conv2D(256, (3, 3), padding="SAME", name="fpn_p5")(P5)
# P6 is used for the 5th anchor scale in RPN. Generated by
# subsampling from P5 with stride of 2.
P6 = KL.Conv2D(256, (3, 3), padding="SAME", name="fpn_p6")(P6)
self.output_layers = [P2, P3, P4, P5, P6]
