batch = K.shape(t_mean)[0]
print(batch)
dim = K.int_shape(t_mean)[1]
epsilon = K.random_normal(shape=(batch, dim))
return t_mean + K.exp(0.5 * t_log_var) * epsilon
#Encoder model
inputs = Input(shape=input_shape, name='encoder_input')
x = Conv2D(64,
4,
padding='same',
kernel_initializer='he_uniform',
use_bias=False)(inputs)
x = BatchNormalization()(x)
x = ELU(alpha=0.1)(x)
x = Conv2D(128,
4,
padding='same',
kernel_initializer='he_uniform',
use_bias=False,
strides=2)(x)
x = BatchNormalization()(x)
x = ELU(alpha=0.1)(x)
#x=Dropout(0.2)(x)
#x=MaxPooling2D((2,2))(x)
#x=Dropout(0.25)(x)
x = Conv2D(128,
def create_model():
model = Sequential()
# Normalization (-1 to 1)
model.add(Lambda(lambda x: x / 127.5 - 1.0, input_shape=(160, 320, 3)))
model.add(
Conv2D(24, (5, 5),
strides=(2, 2),
padding="valid",
kernel_initializer='he_normal'))
# model.add(MaxPooling2D(pool_size = (2,2)))
model.add(BatchNormalization())
model.add(ELU())
model.add(
Conv2D(36, (5, 5),
strides=(2, 2),
padding="valid",
kernel_initializer='he_normal'))
model.add(BatchNormalization())
model.add(ELU())
model.add(
Conv2D(48, (5, 5),
strides=(2, 2),
padding="valid",
kernel_initializer='he_normal'))
model.add(BatchNormalization())
model.add(ELU())
model.add(
Conv2D(64, (3, 3),
strides=(1, 1),
padding="valid",
kernel_initializer='he_normal'))
model.add(BatchNormalization())
model.add(ELU())
model.add(
Conv2D(64, (3, 3),
strides=(1, 1),
padding="valid",
kernel_initializer='he_normal'))
model.add(BatchNormalization())
model.add(ELU())
model.add(Flatten())
model.add(Dropout(0.3))
model.add(ELU())
model.add(
Dense(100,
kernel_initializer='he_normal',
kernel_regularizer=l2(0.001)))
model.add(Dropout(0.5))
model.add(ELU())
model.add(
Dense(50, kernel_initializer='he_normal',
kernel_regularizer=l2(0.001)))
model.add(Dropout(0.5))
model.add(ELU())
model.add(
Dense(10, kernel_initializer='he_normal',
kernel_regularizer=l2(0.001)))
model.add(Dropout(0.5))
model.add(ELU())
model.add(Dense(1, activation='linear', kernel_initializer='he_normal'))
model.compile(optimizer='adam', loss='mse')
return model
x_train_sequence = pad_sequences(x_train_sequence, maxlen=max_sequence_length)
x_test_sequence = pad_sequences(x_test_sequence, maxlen=max_sequence_length)
print('Shape of training tensor:', x_train_sequence.shape)
print(x_train_sequence)
print('Shape of testing tensor:', x_test_sequence.shape)
print(x_test_sequence)
# ===============
# Model creation
# ===============
model = Sequential()
model.add(Embedding(vocab_size, embedding_dim, input_length = max_sequence_length, mask_zero=True))
model.add(LSTM(hidden_layer_size, recurrent_dropout=recurrent_dropout, return_sequences=False))
model.add(ELU())
model.add(Dropout(dropout))
model.add(Dense(2, activation='softmax'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=["binary_accuracy"])
model.summary()
# Fit the model and evaluate
history=model.fit(x_train_sequence, y_train, batch_size=batch_size, callbacks=[Stopping, Checkpointer],
validation_data=(x_test_sequence, y_test),verbose=verbose, shuffle=True, epochs=num_epochs)
score, acc = model.evaluate(x_test_sequence, y_test, batch_size=batch_size)
predict = model.predict_classes(x_test_sequence, verbose=verbose)
outputdf = pd.DataFrame({"Date":list(Testing_dataframe['Date']), "label":list(predict)})
print('Test score:', score)
print('Test accuracy:', acc)
# ===============
def get_nvidia_model(summary=True):
"""
Get the keras Model corresponding to the NVIDIA architecture described in:
Bojarski, Mariusz, et al. "End to end learning for self-driving cars."
The paper describes the network architecture but doesn't go into details for some aspects.
Input normalization, as well as ELU activations are just my personal implementation choice.
:param summary: show model summary
:return: keras Model of NVIDIA architecture
"""
init = 'glorot_uniform'
if K.backend() == 'theano':
input_frame = Input(shape=(CONFIG['input_channels'], NVIDIA_H,
NVIDIA_W))
else:
input_frame = Input(shape=(NVIDIA_H, NVIDIA_W,
CONFIG['input_channels']))
# standardize input
x = Lambda(lambda z: z / 127.5 - 1.)(input_frame)
x = Convolution2D(24,
5,
5,
border_mode='valid',
subsample=(2, 2),
init=init)(x)
x = ELU()(x)
x = Dropout(0.2)(x)
x = Convolution2D(36,
5,
5,
border_mode='valid',
subsample=(2, 2),
init=init)(x)
x = ELU()(x)
x = Dropout(0.2)(x)
x = Convolution2D(48,
5,
5,
border_mode='valid',
subsample=(2, 2),
init=init)(x)
x = ELU()(x)
x = Dropout(0.2)(x)
x = Convolution2D(64, 3, 3, border_mode='valid', init=init)(x)
x = ELU()(x)
x = Dropout(0.2)(x)
x = Convolution2D(64, 3, 3, border_mode='valid', init=init)(x)
x = ELU()(x)
x = Dropout(0.2)(x)
x = Flatten()(x)
x = Dense(100, init=init)(x)
x = ELU()(x)
x = Dropout(0.5)(x)
x = Dense(50, init=init)(x)
x = ELU()(x)
x = Dropout(0.5)(x)
x = Dense(10, init=init)(x)
x = ELU()(x)
out = Dense(1, init=init)(x)
model = Model(input=input_frame, output=out)
if summary:
model.summary()
return model
clf_tree.fit(X_train, y_train)
pred_tree = clf_tree.predict(X_test)
# predicted values as input to stage 2 neural network
input2 = pd.DataFrame({'softplus':pred_lin[:,0],\
'Lasso' : pred_lasso,\
'Decision_tree' : pred_tree,\
'Actual': y_test[:,0]})
# Stage 2 neural network
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import ELU
regressor = Sequential()
regressor.add(Dense(output_dim = 5, init = 'uniform', input_dim = 3))
ELU(alpha=1.0)
regressor.add(Dense(output_dim = 5, init = 'uniform'))
ELU(alpha=1.0)
regressor.add(Dense(output_dim = 3, init = 'uniform'))
ELU(alpha=1.0)
regressor.add(Dense(output_dim = 1, init = 'uniform'))
regressor.compile(optimizer = 'adam', loss = 'mean_squared_error', metrics = ['accuracy'])
X2, y2 = input2.iloc[:,0:3].values, input2.iloc[:,3].values
regressor.fit(X2, y2, validation_split = 0.33, batch_size = 5, nb_epoch = 100)
prednn = regressor.predict(X2)
for i,j in zip(prednn, y_test):
print(i,j)
def nvidia_model():
inputShape = (N_img_height, N_img_width, N_img_channels)
model = Sequential()
# normalization
# perform custom normalization before lambda layer in network
model.add(Lambda(lambda x: x / 127.5 - 1, input_shape=inputShape))
model.add(
Convolution2D(24, (5, 5),
strides=(2, 2),
padding='valid',
kernel_initializer='he_normal',
name='conv1'))
model.add(ELU())
model.add(
Convolution2D(36, (5, 5),
strides=(2, 2),
padding='valid',
kernel_initializer='he_normal',
name='conv2'))
model.add(ELU())
model.add(
Convolution2D(48, (5, 5),
strides=(2, 2),
padding='valid',
kernel_initializer='he_normal',
name='conv3'))
model.add(ELU())
model.add(Dropout(0.5))
model.add(
Convolution2D(64, (3, 3),
strides=(1, 1),
padding='valid',
kernel_initializer='he_normal',
name='conv4'))
model.add(ELU())
model.add(
Convolution2D(64, (3, 3),
strides=(1, 1),
padding='valid',
kernel_initializer='he_normal',
name='conv5'))
model.add(Flatten(name='flatten'))
model.add(ELU())
model.add(Dense(100, kernel_initializer='he_normal', name='fc1'))
model.add(ELU())
model.add(Dense(50, kernel_initializer='he_normal', name='fc2'))
model.add(ELU())
model.add(Dense(10, kernel_initializer='he_normal', name='fc3'))
model.add(ELU())
# do not put activation at the end because we want to exact output, not a class identifier
model.add(Dense(1, name='output', kernel_initializer='he_normal'))
adam = Adam(lr=1e-4, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
model.compile(optimizer=adam, loss='mean_squared_error')
return model
def nvidia_model(time_len=1):
ch, row, col = 3, 66, 200 # camera format
INIT = 'glorot_uniform' # 'he_normal', glorot_uniform
keep_prob = 0.2
reg_val = 0.01
model = Sequential()
model.add(
Lambda(lambda x: x / 127.5 - 1.,
input_shape=(row, col, ch),
output_shape=(row, col, ch)))
model.add(
Convolution2D(24,
5,
5,
subsample=(2, 2),
border_mode="valid",
init=INIT,
W_regularizer=l2(reg_val)))
# W_regularizer=l2(reg_val)
model.add(ELU())
model.add(Dropout(keep_prob))
model.add(
Convolution2D(36,
5,
5,
subsample=(2, 2),
border_mode="valid",
init=INIT))
model.add(ELU())
model.add(Dropout(keep_prob))
model.add(
Convolution2D(48,
5,
5,
subsample=(2, 2),
border_mode="valid",
init=INIT))
model.add(ELU())
model.add(Dropout(keep_prob))
model.add(
Convolution2D(64,
3,
3,
subsample=(1, 1),
border_mode="valid",
init=INIT))
model.add(ELU())
model.add(Dropout(keep_prob))
model.add(
Convolution2D(64,
3,
3,
subsample=(1, 1),
border_mode="valid",
init=INIT))
model.add(ELU())
model.add(Dropout(keep_prob))
model.add(Flatten())
model.add(Dense(100))
model.add(ELU())
model.add(Dropout(0.2))
model.add(Dense(50))
model.add(ELU())
model.add(Dropout(0.2))
model.add(Dense(10))
model.add(ELU())
model.add(Dense(1))
model.compile(optimizer="adam", loss="mse") # , metrics=['accuracy']
return model
def set_up_model_architecture(self):
""" Builds neural network architecture.
Returns:
-
"""
refs_input = Input(shape=(self.data.trainRefsOneHotX.shape[1], ),
name='refs_input')
refs = Dense(256,
input_dim=self.data.trainRefsOneHotX.shape[1],
activation=None)(refs_input)
refs = Dropout(self.dropout)(refs)
refs = BatchNormalization()(refs)
refs = ELU()(refs)
refs = Dense(64, activation=None)(refs)
refs = Dropout(self.dropout)(refs)
refs = BatchNormalization()(refs)
refs = ELU()(refs)
cpcs_input = Input(shape=(self.data.trainCpcOneHotX.shape[1], ),
name='cpcs_input')
cpcs = Dense(32,
input_dim=self.data.trainCpcOneHotX.shape[1],
activation=None)(cpcs_input)
cpcs = Dropout(self.dropout)(cpcs)
cpcs = BatchNormalization()(cpcs)
cpcs = ELU()(cpcs)
# Use pre-trained Word2Vec embeddings
embedding_layer_input = Input(shape=(self.max_seq_length, ),
name='embed_input')
embedding_layer = Embedding(
self.data.embedding_model.embedding_weights.shape[0],
self.data.embedding_model.embedding_weights.shape[1],
weights=[self.data.embedding_model.embedding_weights],
input_length=self.max_seq_length,
trainable=False)(embedding_layer_input)
deep = LSTM(self.lstm_size,
dropout=self.dropout,
recurrent_dropout=self.dropout,
return_sequences=False,
name='LSTM_1')(embedding_layer)
deep = Dense(300, activation=None)(deep)
deep = Dropout(self.dropout)(deep)
deep = BatchNormalization()(deep)
deep = ELU()(deep)
# model_inputs_to_concat = [cpcs, refs, deep]
model_inputs_to_concat = [refs, deep]
final_layer = Concatenate(
name='concatenated_layer')(model_inputs_to_concat)
output = Dense(64, activation=None)(final_layer)
output = Dropout(self.dropout)(output)
output = BatchNormalization()(output)
output = ELU()(output)
output = Dense(1, activation='sigmoid')(output)
# model = Model(inputs=[cpcs_input, refs_input, embedding_layer_input], outputs=output, name='model')
model = Model(inputs=[refs_input, embedding_layer_input],
outputs=output,
name='model')
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy', precision, recall, f1score])
self.tf_model = model
print('Done building graph.')
print(self.tf_model.summary())
def get_model(config):
"""Returns deep fingerprinting model to run_model.py
Args:
config (dict): Deserialized JSON config file (see config.json)
"""
num_mon_sites = config['num_mon_sites']
num_mon_inst_test = config['num_mon_inst_test']
num_mon_inst_train = config['num_mon_inst_train']
num_unmon_sites_test = config['num_unmon_sites_test']
num_unmon_sites_train = config['num_unmon_sites_train']
num_unmon_sites = num_unmon_sites_test + num_unmon_sites_train
seq_length = config['seq_length']
dir_input = Input(shape=(seq_length, 1,), name='dir_input')
# Block 1
x = Conv1D(32, 8, strides=1, padding='same')(dir_input)
x = BatchNormalization()(x)
x = ELU(alpha=1.0)(x)
x = Conv1D(32, 8, strides=1, padding='same')(x)
x = BatchNormalization()(x)
x = ELU(alpha=1.0)(x)
x = MaxPooling1D(pool_size=8, strides=4)(x)
x = Dropout(0.1)(x)
# Block 2
x = Conv1D(64, 8, strides=1, padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv1D(64, 8, strides=1, padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling1D(pool_size=8, strides=4)(x)
x = Dropout(0.1)(x)
# Block 3
x = Conv1D(128, 8, strides=1, padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv1D(128, 8, strides=1, padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling1D(pool_size=8, strides=4)(x)
x = Dropout(0.1)(x)
# Block 4
x = Conv1D(256, 8, strides=1, padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv1D(256, 8, strides=1, padding='same')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling1D(pool_size=8, strides=4)(x)
x = Dropout(0.1)(x)
x = Flatten()(x)
# FC layers
x = Dense(512)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(0.7)(x)
x = Dense(512)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(0.5)(x)
# Add final softmax layer
output_classes = num_mon_sites if num_unmon_sites == 0 else num_mon_sites + 1
model_output = Dense(units=output_classes, activation='softmax',
name='model_output')(x)
model = Model(inputs=dir_input, outputs=model_output)
model.compile(loss='categorical_crossentropy',
optimizer=Adamax(0.002),
metrics=['accuracy'])
callbacks = []
return model, callbacks
def build(self):
"""
Construct the main structure of the network
"""
print('DNN input shape', self.input_shape)
if K.image_dim_ordering() == 'tf':
batch_sz, bands, frames, channels = self.input_shape
assert channels >= 1
channel_axis = 3
freq_axis = 1
nn_shape = (bands, frames, channels)
else:
raise NotImplementedError('[ERROR] Only for TensorFlow background.')
nb_filters = self.config['feature_maps']
# Input block
feat_input = Input(shape=nn_shape, name='input')
x = BatchNormalization(axis=freq_axis, name='bn_0_freq')(feat_input)
# Conv block 1
x = Convolution2D(nb_filters, 3, 3, border_mode='same', name='conv1')(x)
x = BatchNormalization(axis=channel_axis, mode=0, name='bn1')(x)
x = ELU()(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(x)
x = Dropout(self.config['dropout'], name='dropout1')(x)
# Conv block 2
x = Convolution2D(nb_filters, 3, 3, border_mode='same', name='conv2')(x)
x = BatchNormalization(axis=channel_axis, mode=0, name='bn2')(x)
x = ELU()(x)
x = MaxPooling2D(pool_size=(4, 2), strides=(4, 2), name='pool2')(x)
x = Dropout(self.config['dropout'], name='dropout2')(x)
# Conv block 3
x = Convolution2D(2 * nb_filters, 3, 3, border_mode='same', name='conv3')(x)
x = BatchNormalization(axis=channel_axis, mode=0, name='bn3')(x)
x = ELU()(x)
x = MaxPooling2D(pool_size=(4, 1), strides=(4, 1), name='pool3')(x)
x = Dropout(self.config['dropout'], name='dropout3')(x)
# Conv block 4
x = Convolution2D(2 * nb_filters, 3, 3, border_mode='same', name='conv4')(x)
x = BatchNormalization(axis=channel_axis, mode=0, name='bn4')(x)
x = ELU()(x)
x = MaxPooling2D(pool_size=(2, 1), strides=(2, 1), name='pool4')(x)
x = Dropout(self.config['dropout'], name='dropout4')(x)
x = Reshape((-1, 2 * nb_filters))(x)
# GRU block 1, 2, output
# x = GRU(32, return_sequences=True, name='gru1')(x)
x = GRU(32, return_sequences=False, name='gru2')(x)
x = Dropout(self.config['dropout'])(x)
# Affine transformation
x = Dense(self.nclass, name='output_preactivation')(x)
y_pred = Activation(activation=self.config['out_score'], name='output')(x)
self._compile_model(input=feat_input, output=y_pred, params=self.config)
if type is 4:
num_tests = 4
for i in range(num_tests):
keras.backend.clear_session()
features = np.random.uniform(size=[2, 16])
model = Sequential()
model.add(Dense(name="dense", units=32, input_shape=(16, )))
if type is 0:
model.add(LeakyReLU(alpha=0.5))
layername = "LeakyReLU"
elif type is 1:
model.add(ELU(alpha=0.7))
layername = "ELU"
elif type is 2:
model.add(ThresholdedReLU(theta=0.7))
layername = "ThresholdReLU"
elif type is 3:
model.add(Softmax())
layername = "Softmax"
elif type is 4:
if i is 0:
model.add(
ReLU(max_value=None, negative_slope=0.0, threshold=0.0))
elif i is 1:
model.add(
ReLU(max_value=6.0, negative_slope=0.0, threshold=0.0))
elif i is 2:
def createModel(learning_rate, dropout):
'''
Function creates a model of a the network
'''
img_input = Input(shape=(HEIGHT, WIDTH, DEPTH))
x = Convolution2D(24,
5,
5,
border_mode='valid',
subsample=(2, 2),
W_regularizer=l2(learning_rate))(img_input)
x = ELU()(x)
x = Convolution2D(36,
5,
5,
border_mode='valid',
subsample=(2, 2),
W_regularizer=l2(learning_rate))(x)
x = ELU()(x)
x = Convolution2D(48,
5,
5,
border_mode='valid',
subsample=(2, 2),
W_regularizer=l2(learning_rate))(x)
x = ELU()(x)
x = Convolution2D(64,
3,
3,
border_mode='valid',
subsample=(1, 1),
W_regularizer=l2(learning_rate))(x)
x = ELU()(x)
x = Convolution2D(64,
3,
3,
border_mode='valid',
subsample=(1, 1),
W_regularizer=l2(learning_rate))(x)
x = ELU()(x)
x = Flatten()(x)
speed_input = Input(shape=(8, ), name='speed_input')
xx = Dense(100)(speed_input)
xx = ELU()(xx)
xx = Dense(50)(xx)
xx = ELU()(xx)
xx = Dense(10)(xx)
xx = ELU()(xx)
x = keras.layers.merge([x, xx], mode='concat', concat_axis=-1)
x = Dense(100)(x)
x = ELU()(x)
x = Dropout(dropout)(x)
x = Dense(50)(x)
x = ELU()(x)
x = Dropout(dropout)(x)
x = Dense(10)(x)
x = ELU()(x)
predictions = Dense(3)(x)
model = Model(input=[img_input, speed_input], output=predictions)
model.compile(optimizer='adam', loss='mse')
print(model.summary())
return model
def model(self):
inputs_img = Input(shape=(self.img_height, self.img_width,
self.num_channels))
masks = Input(shape=(self.img_height, self.img_width,
self.num_channels))
neg_masks = BinaryNegation()(masks)
inputs = Multiply()([inputs_img, neg_masks])
# Encoder-branch-1
eb1 = Conv2D(filters=32, kernel_size=7, strides=(1, 1),
padding='same')(inputs)
eb1 = ELU()(eb1)
eb1 = Conv2D(filters=64, kernel_size=7, strides=(2, 2),
padding='same')(eb1)
eb1 = ELU()(eb1)
eb1 = Conv2D(filters=64, kernel_size=7, strides=(1, 1),
padding='same')(eb1)
eb1 = ELU()(eb1)
eb1 = Conv2D(filters=128,
kernel_size=7,
strides=(2, 2),
padding='same')(eb1)
eb1 = ELU()(eb1)
eb1 = Conv2D(filters=128,
kernel_size=7,
strides=(1, 1),
padding='same')(eb1)
eb1 = ELU()(eb1)
eb1 = Conv2D(filters=128,
kernel_size=7,
strides=(1, 1),
padding='same')(eb1)
eb1 = Conv2D(filters=128,
kernel_size=7,
strides=(1, 1),
padding='same',
dilation_rate=(2, 2))(eb1)
eb1 = ELU()(eb1)
eb1 = Conv2D(filters=128,
kernel_size=7,
strides=(1, 1),
padding='same',
dilation_rate=(4, 4))(eb1)
eb1 = ELU()(eb1)
eb1 = Conv2D(filters=128,
kernel_size=7,
strides=(1, 1),
padding='same',
dilation_rate=(8, 8))(eb1)
eb1 = ELU()(eb1)
eb1 = Conv2D(filters=128,
kernel_size=7,
strides=(1, 1),
padding='same',
dilation_rate=(16, 16))(eb1)
eb1 = ELU()(eb1)
eb1 = Conv2D(filters=128,
kernel_size=7,
strides=(1, 1),
padding='same')(eb1)
eb1 = ELU()(eb1)
eb1 = Conv2D(filters=128,
kernel_size=7,
strides=(1, 1),
padding='same')(eb1)
eb1 = ELU()(eb1)
eb1 = UpSampling2D(size=(4, 4))(eb1)
# Encoder-branch-2
eb2 = Conv2D(filters=32, kernel_size=5, strides=(1, 1),
padding='same')(inputs)
eb2 = ELU()(eb2)
eb2 = Conv2D(filters=64, kernel_size=5, strides=(2, 2),
padding='same')(eb2)
eb2 = ELU()(eb2)
eb2 = Conv2D(filters=64, kernel_size=5, strides=(1, 1),
padding='same')(eb2)
eb2 = ELU()(eb2)
eb2 = Conv2D(filters=128,
kernel_size=5,
strides=(2, 2),
padding='same')(eb2)
eb2 = ELU()(eb2)
eb2 = Conv2D(filters=128,
kernel_size=5,
strides=(1, 1),
padding='same')(eb2)
eb2 = ELU()(eb2)
eb2 = Conv2D(filters=128,
kernel_size=5,
strides=(1, 1),
padding='same')(eb2)
eb2 = ELU()(eb2)
eb2 = Conv2D(filters=128,
kernel_size=5,
strides=(1, 1),
padding='same',
dilation_rate=(2, 2))(eb2)
eb2 = ELU()(eb2)
eb2 = Conv2D(filters=128,
kernel_size=5,
strides=(1, 1),
padding='same',
dilation_rate=(4, 4))(eb2)
eb2 = ELU()(eb2)
eb2 = Conv2D(filters=128,
kernel_size=5,
strides=(1, 1),
padding='same',
dilation_rate=(8, 8))(eb2)
eb2 = ELU()(eb2)
eb2 = Conv2D(filters=128,
kernel_size=5,
strides=(1, 1),
padding='same',
dilation_rate=(16, 16))(eb2)
eb2 = ELU()(eb2)
eb2 = Conv2D(filters=128,
kernel_size=5,
strides=(1, 1),
padding='same')(eb2)
eb2 = ELU()(eb2)
eb2 = Conv2D(filters=128,
kernel_size=5,
strides=(1, 1),
padding='same')(eb2)
eb2 = ELU()(eb2)
eb2 = UpSampling2D(size=(2, 2))(eb2)
eb2 = Conv2D(filters=64, kernel_size=5, strides=(1, 1),
padding='same')(eb2)
eb2 = ELU()(eb2)
eb2 = Conv2D(filters=64, kernel_size=5, strides=(1, 1),
padding='same')(eb2)
eb2 = ELU()(eb2)
eb2 = UpSampling2D(size=(2, 2))(eb2)
# Encoder-branch-3
eb3 = Conv2D(filters=32, kernel_size=3, strides=(1, 1),
padding='same')(inputs)
eb3 = ELU()(eb3)
eb3 = Conv2D(filters=64, kernel_size=3, strides=(2, 2),
padding='same')(eb3)
eb3 = ELU()(eb3)
eb3 = Conv2D(filters=64, kernel_size=3, strides=(1, 1),
padding='same')(eb3)
eb3 = ELU()(eb3)
eb3 = Conv2D(filters=128,
kernel_size=3,
strides=(2, 2),
padding='same')(eb3)
eb3 = ELU()(eb3)
eb3 = Conv2D(filters=128,
kernel_size=3,
strides=(1, 1),
padding='same')(eb3)
eb3 = ELU()(eb3)
eb3 = Conv2D(filters=128,
kernel_size=3,
strides=(1, 1),
padding='same')(eb3)
eb3 = ELU()(eb3)
eb3 = Conv2D(filters=128,
kernel_size=3,
strides=(1, 1),
padding='same',
dilation_rate=(2, 2))(eb3)
eb3 = ELU()(eb3)
eb3 = Conv2D(filters=128,
kernel_size=3,
strides=(1, 1),
padding='same',
dilation_rate=(4, 4))(eb3)
eb3 = ELU()(eb3)
eb3 = Conv2D(filters=128,
kernel_size=3,
strides=(1, 1),
padding='same',
dilation_rate=(8, 8))(eb3)
eb3 = ELU()(eb3)
eb3 = Conv2D(filters=128,
kernel_size=3,
strides=(1, 1),
padding='same',
dilation_rate=(16, 16))(eb3)
eb3 = ELU()(eb3)
eb3 = Conv2D(filters=128,
kernel_size=3,
strides=(1, 1),
padding='same')(eb3)
eb3 = ELU()(eb3)
eb3 = Conv2D(filters=128,
kernel_size=3,
strides=(1, 1),
padding='same')(eb3)
eb3 = ELU()(eb3)
eb3 = UpSampling2D(size=(2, 2))(eb3)
eb3 = Conv2D(filters=64, kernel_size=5, strides=(1, 1),
padding='same')(eb3)
eb3 = ELU()(eb3)
eb3 = Conv2D(filters=64, kernel_size=5, strides=(1, 1),
padding='same')(eb3)
eb3 = ELU()(eb3)
eb3 = UpSampling2D(size=(2, 2))(eb3)
eb3 = Conv2D(filters=64, kernel_size=3, strides=(1, 1),
padding='same')(eb3)
eb3 = ELU()(eb3)
eb3 = Conv2D(filters=64, kernel_size=3, strides=(1, 1),
padding='same')(eb3)
eb3 = ELU()(eb3)
decoder = Concatenate(axis=3)([eb1, eb2, eb3])
decoder = Conv2D(filters=16,
kernel_size=3,
strides=(1, 1),
padding='same')(decoder)
decoder = ELU()(decoder)
decoder = Conv2D(filters=3,
kernel_size=3,
strides=(1, 1),
padding='same')(decoder)
# linearly norm to (-1, 1)
decoder = Clip()(decoder)
model = Model(name=self.model_name,
inputs=[inputs_img, masks],
outputs=[decoder])
return model
import numpy as np
import keras
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Model
from keras.layers import Input, Conv2D, ELU, MaxPooling2D, Flatten, Dense, Dropout, normalization
from keras.optimizers import SGD, rmsprop, adam
from keras import backend as K
import os
import scipy.io as sio
#left image
left_image = Input(shape=(32, 32, 3))
#conv1
left_conv1 = Conv2D(16, (3, 3), padding='same', name='conv1_left')(left_image)
left_elu1 = ELU()(left_conv1)
left_pool1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2),
name='pool1_left')(left_elu1)
#conv2
left_conv2 = Conv2D(16, (3, 3), padding='same', name='conv2_left')(left_pool1)
left_elu2 = ELU()(left_conv2)
left_pool2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2),
name='pool2_left')(left_elu2)
#conv3
left_conv3 = Conv2D(32, (3, 3), padding='same', name='conv3_left')(left_pool2)
left_elu3 = ELU()(left_conv3)
#conv4
left_conv4 = Conv2D(32, (3, 3), padding='same', name='conv4_left')(left_elu3)
left_elu4 = ELU()(left_conv4)
#conv5
left_conv5 = Conv2D(64, (3, 3), padding='same', name='conv5_left')(left_elu4)
def _create_MobileNet_with_embedding_5(self, num_of_classes, input_shape):
mobilenet_model_face = mobilenet_v2.MobileNetV2(
input_shape=input_shape,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=None,
pooling=None)
mobilenet_model_face.layers.pop()
'''eyes'''
mobilenet_model_eyes = mobilenet_v2.MobileNetV2(
input_shape=input_shape,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=None,
pooling=None)
mobilenet_model_eyes.layers.pop()
'''nose'''
mobilenet_model_nose = mobilenet_v2.MobileNetV2(
input_shape=input_shape,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=None,
pooling=None)
mobilenet_model_nose.layers.pop()
'''mouth'''
mobilenet_model_mouth = mobilenet_v2.MobileNetV2(
input_shape=input_shape,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=None,
pooling=None)
mobilenet_model_mouth.layers.pop()
for layer in mobilenet_model_face.layers:
layer._name = 'face_' + layer.name
for layer in mobilenet_model_eyes.layers:
layer._name = 'eyes_' + layer.name
for layer in mobilenet_model_nose.layers:
layer._name = 'nose_' + layer.name
for layer in mobilenet_model_mouth.layers:
layer._name = 'mouth_' + layer.name
# mobilenet_model_mouth.summary()
''''''
o_relu_l_face = mobilenet_model_face.get_layer(
'face_out_relu').output # 1280
o_relu_l_eyes = mobilenet_model_eyes.get_layer(
'eyes_out_relu').output # 1280
o_relu_l_nose = mobilenet_model_nose.get_layer(
'nose_out_relu').output # 1280
o_relu_l_mouth = mobilenet_model_mouth.get_layer(
'mouth_out_relu').output # 1280
'''embedding'''
g_x_l_face = GlobalAveragePooling2D()(o_relu_l_face)
x_l_face = Dense(
LearningConfig.embedding_size * 2,
kernel_regularizer=tf.keras.regularizers.l2(0.0001))(g_x_l_face)
x_l_face = BatchNormalization()(x_l_face)
x_l_face = Dropout(rate=0.3)(x_l_face)
x_l_face = ELU()(x_l_face)
x_l_face = tf.keras.layers.Dense(
LearningConfig.embedding_size,
activation=None)(x_l_face) # No activation on final dense layer
embedding_layer_face = tf.keras.layers.Lambda(
lambda x: tf.math.l2_normalize(x, axis=1))(
x_l_face) # L2 normalize embeddings
# eyes
g_x_l_eyes = GlobalAveragePooling2D()(o_relu_l_eyes)
x_l_eyes = Dense(
LearningConfig.embedding_size * 2,
kernel_regularizer=tf.keras.regularizers.l2(0.0001))(g_x_l_eyes)
x_l_eyes = BatchNormalization()(x_l_eyes)
x_l_eyes = Dropout(rate=0.3)(x_l_eyes)
x_l_eyes = ELU()(x_l_eyes)
x_l_eyes = tf.keras.layers.Dense(
LearningConfig.embedding_size,
activation=None)(x_l_eyes) # No activation on final dense layer
embedding_layer_eyes = tf.keras.layers.Lambda(
lambda x: tf.math.l2_normalize(x, axis=1))(
x_l_eyes) # L2 normalize embeddings
# nose
g_x_l_nose = GlobalAveragePooling2D()(o_relu_l_nose)
x_l_nose = Dense(
LearningConfig.embedding_size * 2,
kernel_regularizer=tf.keras.regularizers.l2(0.0001))(g_x_l_nose)
x_l_nose = BatchNormalization()(x_l_nose)
x_l_nose = Dropout(rate=0.3)(x_l_nose)
x_l_nose = ELU()(x_l_nose)
x_l_nose = tf.keras.layers.Dense(
LearningConfig.embedding_size,
activation=None)(x_l_nose) # No activation on final dense layer
embedding_layer_nose = tf.keras.layers.Lambda(
lambda x: tf.math.l2_normalize(x, axis=1))(
x_l_nose) # L2 normalize embeddings
# mouth
g_x_l_mouth = GlobalAveragePooling2D()(o_relu_l_mouth)
x_l_mouth = Dense(
LearningConfig.embedding_size * 2,
kernel_regularizer=tf.keras.regularizers.l2(0.0001))(g_x_l_mouth)
x_l_mouth = BatchNormalization()(x_l_mouth)
x_l_mouth = Dropout(rate=0.3)(x_l_mouth)
x_l_mouth = ELU()(x_l_mouth)
x_l_mouth = tf.keras.layers.Dense(
LearningConfig.embedding_size,
activation=None)(x_l_mouth) # No activation on final dense layer
embedding_layer_mouth = tf.keras.layers.Lambda(
lambda x: tf.math.l2_normalize(x, axis=1))(
x_l_mouth) # L2 normalize embeddings
'''concat'''
# concat_embeddings = tf.keras.layers.Concatenate(axis=1)([embedding_layer_face,
# embedding_layer_eyes,
# embedding_layer_nose,
# embedding_layer_mouth])
concat_embeddings = tf.keras.layers.Concatenate(axis=1)(
[o_relu_l_face, o_relu_l_eyes, o_relu_l_nose, o_relu_l_mouth])
fused_global_avg_pool = GlobalAveragePooling2D()(concat_embeddings)
'''FC layer for out'''
x_l = Dense(
LearningConfig.embedding_size,
kernel_regularizer=tf.keras.regularizers.l2(0.0001),
)(fused_global_avg_pool)
x_l = BatchNormalization()(x_l)
x_l = Dropout(rate=0.2)(x_l)
x_l = ELU()(x_l)
x_l = Dense(
LearningConfig.embedding_size // 2,
kernel_regularizer=tf.keras.regularizers.l2(0.0001),
)(x_l)
x_l = BatchNormalization()(x_l)
x_l = Dropout(rate=0.2)(x_l)
x_l = ELU()(x_l)
x_l = Dense(
LearningConfig.embedding_size // 4,
kernel_regularizer=tf.keras.regularizers.l2(0.0001),
)(x_l)
x_l = BatchNormalization()(x_l)
x_l = Dropout(rate=0.2)(x_l)
x_l = ELU()(x_l)
x_l = Dense(
LearningConfig.embedding_size // 8,
kernel_regularizer=tf.keras.regularizers.l2(0.0001),
)(x_l)
x_l = BatchNormalization()(x_l)
x_l = Dropout(rate=0.2)(x_l)
x_l = ELU()(x_l)
x_l = Dense(
LearningConfig.embedding_size // 16,
kernel_regularizer=tf.keras.regularizers.l2(0.0001),
)(x_l)
x_l = BatchNormalization()(x_l)
x_l = Dropout(rate=0.2)(x_l)
x_l = ELU()(x_l)
'''out'''
out_categorical = Dense(num_of_classes,
activation='softmax',
name='out')(x_l)
inp = [
mobilenet_model_face.input, mobilenet_model_eyes.input,
mobilenet_model_nose.input, mobilenet_model_mouth.input
]
revised_model = Model(inp, [
out_categorical, embedding_layer_face, embedding_layer_eyes,
embedding_layer_nose, embedding_layer_mouth
])
revised_model.summary()
'''save json'''
model_json = revised_model.to_json()
with open("./model_archs/mn_v2_cat_emb.json", "w") as json_file:
json_file.write(model_json)
return revised_model
def build_model(image_size,
n_classes):
''' NOTE(GP) original description. Unused argument descriptions have been removed
Build a Keras model with SSD architecture, see references.
The model consists of convolutional feature layers and a number of convolutional
predictor layers that take their input from different feature layers.
The model is fully convolutional.
The implementation found here is a smaller version of the original architecture
used in the paper (where the base network consists of a modified VGG-16 extended
by a few convolutional feature layers), but of course it could easily be changed to
an arbitrarily large SSD architecture by following the general design pattern used here.
This implementation has 7 convolutional layers and 4 convolutional predictor
layers that take their input from layers 4, 5, 6, and 7, respectively.
In case you're wondering why this function has so many arguments: All arguments except
the first two (`image_size` and `n_classes`) are only needed so that the anchor box
layers can produce the correct anchor boxes. In case you're training the network, the
parameters passed here must be the same as the ones used to set up `SSDBoxEncoder`.
In case you're loading trained weights, the parameters passed here must be the same
as the ones used to produce the trained weights.
Some of these arguments are explained in more detail in the documentation of the
`SSDBoxEncoder` class.
Note: Requires Keras v2.0 or later. Training currently works only with the
TensorFlow backend (v1.0 or later).
Arguments:
image_size (tuple): The input image size in the format `(height, width, channels)`.
n_classes (int): The number of categories for classification including
the background class (i.e. the number of positive classes +1 for
the background calss).
Returns:
model: The Keras SSD model.
References:
https://arxiv.org/abs/1512.02325v5
'''
# Input image format
img_height, img_width, img_channels = image_size[0], image_size[1], image_size[2]
# Design the actual network
x = Input(shape=(img_height, img_width, img_channels))
normed = Lambda(lambda z: z/127.5 - 1., # Convert input feature range to [-1,1]
output_shape=(img_height, img_width, img_channels),
name='lambda1')(x)
conv1 = Conv2D(16, (5, 5), name='conv1', strides=(1, 1), padding="same")(normed)
conv1 = BatchNormalization(axis=3, momentum=0.99, name='bn1')(conv1)
conv1 = ELU(name='elu1')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2), name='pool1')(conv1)
# 200 * 200
conv2 = Conv2D(32, (3, 3), name='conv2', strides=(1, 1), padding="same")(pool1)
conv2 = BatchNormalization(axis=3, momentum=0.99, name='bn2')(conv2)
conv2 = ELU(name='elu2')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2), name='pool2')(conv2)
# 100 * 100
conv3 = Conv2D(64, (3, 3), name='conv3', strides=(1, 1), padding="same")(pool2)
conv3 = BatchNormalization(axis=3, momentum=0.99, name='bn3')(conv3)
conv3 = ELU(name='elu3')(conv3)
# LAYER REMOVED FOR TIME BEING
# conv3b = Conv2D(32, (3, 3), name='conv3b', strides=(1, 1), padding="same")(conv3)
# conv3b = BatchNormalization(axis=3, momentum=0.99, name='bn3b')(conv3b)
# conv3b = ELU(name='elu3b')(conv3b)
pool3 = MaxPooling2D(pool_size=(2, 2), name='pool3')(conv3)
# LAYER REMOVED FOR TIME BEING
# conv4 = Conv2D(64, (3, 3), name='conv4', strides=(1, 1), padding="same")(pool3)
# conv4 = BatchNormalization(axis=3, momentum=0.99, name='bn4')(conv4)
# conv4 = ELU(name='elu4')(conv4)
# pool4 = MaxPooling2D(pool_size=(2, 2), name='pool4')(conv4)
# 50 * 50
conv5 = Conv2D(128, (3, 3), name='conv5', strides=(1, 1), padding="same")(pool3)
conv5 = BatchNormalization(axis=3, momentum=0.99, name='bn5')(conv5)
conv5 = ELU(name='elu5')(conv5)
pool5 = MaxPooling2D(pool_size=(2, 2), name='pool5')(conv5)
# 25 * 25
conv6 = Conv2D(128, (3, 3), name='conv6', strides=(1, 1), padding="valid")(pool5)
conv6 = BatchNormalization(axis=3, momentum=0.99, name='bn6')(conv6)
conv6 = ELU(name='elu6')(conv6)
pool6 = MaxPooling2D(pool_size=(2, 2), name='pool6')(conv6)
# 12*12
conv6b = Conv2D(128, (3, 3), name='conv6b', strides=(1, 1), padding="same")(pool6)
conv6b = BatchNormalization(axis=3, momentum=0.99, name='bn6b')(conv6b)
conv6b = ELU(name='elu6b')(conv6b)
pool6b = MaxPooling2D(pool_size=(2, 2), name='pool6b')(conv6b)
# 6*6
# NOTE (GP) These layers upsample the image back to 51*51
deconv1 = Conv2DTranspose(128, (3,3),name='deconv1', strides=(1, 1), padding='same')(pool6b)
deconv1 = BatchNormalization(axis=3, momentum=0.99, name='bndc1')(deconv1)
deconv1 = ELU(name='eludc1')(deconv1)
uppool1 = UpSampling2D(3)(deconv1)
deconv2 = Conv2DTranspose(64, (3,3),name='deconv2', strides=(1, 1), padding='valid')(uppool1)
deconv2 = BatchNormalization(axis=3, momentum=0.99, name='bndc2')(deconv2)
deconv2 = ELU(name='eludc2')(deconv2)
uppool2 = UpSampling2D(3)(deconv2)
# 51 * 51
# NOTE(GP) This final layer produces the softmax predictions of each class in a 51 * 51 area
out = Conv2DTranspose(21, (3,3),name='out', strides=(1, 1), activation='softmax', padding='same')(uppool2)
predictions = out
model = Model(inputs=x, outputs=predictions)
return model
def build_model(image_size,
n_classes,
l2_regularization=0.0,
min_scale=0.1,
max_scale=0.9,
scales=None,
aspect_ratios_global=[0.5, 1.0, 2.0],
aspect_ratios_per_layer=None,
two_boxes_for_ar1=True,
steps=None,
offsets=None,
limit_boxes=False,
variances=[1.0, 1.0, 1.0, 1.0],
coords='centroids',
normalize_coords=False,
subtract_mean=None,
divide_by_stddev=None,
swap_channels=False,
return_predictor_sizes=False):
'''
Build a Keras model with SSD architecture, see references.
The model consists of convolutional feature layers and a number of convolutional
predictor layers that take their input from different feature layers.
The model is fully convolutional.
The implementation found here is a smaller version of the original architecture
used in the paper (where the base network consists of a modified VGG-16 extended
by a few convolutional feature layers), but of course it could easily be changed to
an arbitrarily large SSD architecture by following the general design pattern used here.
This implementation has 7 convolutional layers and 4 convolutional predictor
layers that take their input from layers 4, 5, 6, and 7, respectively.
Most of the arguments that this function takes are only needed for the anchor
box layers. In case you're training the network, the parameters passed here must
be the same as the ones used to set up `SSDBoxEncoder`. In case you're loading
trained weights, the parameters passed here must be the same as the ones used
to produce the trained weights.
Some of these arguments are explained in more detail in the documentation of the
`SSDBoxEncoder` class.
Note: Requires Keras v2.0 or later. Training currently works only with the
TensorFlow backend (v1.0 or later).
Arguments:
image_size (tuple): The input image size in the format `(height, width, channels)`.
n_classes (int): The number of positive classes, e.g. 20 for Pascal VOC, 80 for MS COCO.
l2_regularization (float, optional): The L2-regularization rate. Applies to all convolutional layers.
min_scale (float, optional): The smallest scaling factor for the size of the anchor boxes as a fraction
of the shorter side of the input images.
max_scale (float, optional): The largest scaling factor for the size of the anchor boxes as a fraction
of the shorter side of the input images. All scaling factors between the smallest and the
largest will be linearly interpolated. Note that the second to last of the linearly interpolated
scaling factors will actually be the scaling factor for the last predictor layer, while the last
scaling factor is used for the second box for aspect ratio 1 in the last predictor layer
if `two_boxes_for_ar1` is `True`.
scales (list, optional): A list of floats containing scaling factors per convolutional predictor layer.
This list must be one element longer than the number of predictor layers. The first `k` elements are the
scaling factors for the `k` predictor layers, while the last element is used for the second box
for aspect ratio 1 in the last predictor layer if `two_boxes_for_ar1` is `True`. This additional
last scaling factor must be passed either way, even if it is not being used.
Defaults to `None`. If a list is passed, this argument overrides `min_scale` and
`max_scale`. All scaling factors must be greater than zero.
aspect_ratios_global (list, optional): The list of aspect ratios for which anchor boxes are to be
generated. This list is valid for all predictor layers. The original implementation uses more aspect ratios
for some predictor layers and fewer for others. If you want to do that, too, then use the next argument instead.
Defaults to `[0.5, 1.0, 2.0]`.
aspect_ratios_per_layer (list, optional): A list containing one aspect ratio list for each predictor layer.
This allows you to set the aspect ratios for each predictor layer individually. If a list is passed,
it overrides `aspect_ratios_global`.
two_boxes_for_ar1 (bool, optional): Only relevant for aspect ratio lists that contain 1. Will be ignored otherwise.
If `True`, two anchor boxes will be generated for aspect ratio 1. The first will be generated
using the scaling factor for the respective layer, the second one will be generated using
geometric mean of said scaling factor and next bigger scaling factor. Defaults to `True`, following the original
implementation.
steps (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be
either ints/floats or tuples of two ints/floats. These numbers represent for each predictor layer how many
pixels apart the anchor box center points should be vertically and horizontally along the spatial grid over
the image. If the list contains ints/floats, then that value will be used for both spatial dimensions.
If the list contains tuples of two ints/floats, then they represent `(step_height, step_width)`.
If no steps are provided, then they will be computed such that the anchor box center points will form an
equidistant grid within the image dimensions.
offsets (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be
either floats or tuples of two floats. These numbers represent for each predictor layer how many
pixels from the top and left boarders of the image the top-most and left-most anchor box center points should be
as a fraction of `steps`. The last bit is important: The offsets are not absolute pixel values, but fractions
of the step size specified in the `steps` argument. If the list contains floats, then that value will
be used for both spatial dimensions. If the list contains tuples of two floats, then they represent
`(vertical_offset, horizontal_offset)`. If no offsets are provided, then they will default to 0.5 of the step size,
which is also the recommended setting.
limit_boxes (bool, optional): If `True`, limits box coordinates to stay within image boundaries.
This would normally be set to `True`, but here it defaults to `False`, following the original
implementation.
variances (list, optional): A list of 4 floats >0 with scaling factors (actually it's not factors but divisors
to be precise) for the encoded predicted box coordinates. A variance value of 1.0 would apply
no scaling at all to the predictions, while values in (0,1) upscale the encoded predictions and values greater
than 1.0 downscale the encoded predictions. If you want to reproduce the configuration of the original SSD,
set this to `[0.1, 0.1, 0.2, 0.2]`, provided the coordinate format is 'centroids'.
coords (str, optional): The box coordinate format to be used. Can be either 'centroids' for the format
`(cx, cy, w, h)` (box center coordinates, width, and height) or 'minmax' for the format
`(xmin, xmax, ymin, ymax)`.
normalize_coords (bool, optional): Set to `True` if the model is supposed to use relative instead of absolute coordinates,
i.e. if the model predicts box coordinates within [0,1] instead of absolute coordinates.
subtract_mean (array-like, optional): `None` or an array-like object of integers or floating point values
of any shape that is broadcast-compatible with the image shape. The elements of this array will be
subtracted from the image pixel intensity values. For example, pass a list of three integers
to perform per-channel mean normalization for color images.
divide_by_stddev (array-like, optional): `None` or an array-like object of non-zero integers or
floating point values of any shape that is broadcast-compatible with the image shape. The image pixel
intensity values will be divided by the elements of this array. For example, pass a list
of three integers to perform per-channel standard deviation normalization for color images.
swap_channels (bool, optional): If `True`, the color channel order of the input images will be reversed,
i.e. if the input color channel order is RGB, the color channels will be swapped to BGR.
return_predictor_sizes (bool, optional): If `True`, this function not only returns the model, but also
a list containing the spatial dimensions of the predictor layers. This isn't strictly necessary since
you can always get their sizes easily via the Keras API, but it's convenient and less error-prone
to get them this way. They are only relevant for training anyway (SSDBoxEncoder needs to know the
spatial dimensions of the predictor layers), for inference you don't need them.
Returns:
model: The Keras SSD model.
predictor_sizes (optional): A Numpy array containing the `(height, width)` portion
of the output tensor shape for each convolutional predictor layer. During
training, the generator function needs this in order to transform
the ground truth labels into tensors of identical structure as the
output tensors of the model, which is in turn needed for the cost
function.
References:
https://arxiv.org/abs/1512.02325v5
'''
n_predictor_layers = 4 # 在网络中用来做预测的卷积层个数
n_classes += 1 # 加上背景类别个数
l2_reg = l2_regularization # 缩写，方便一点
img_height, img_width, img_channels = image_size[0], image_size[1], image_size[2]
############################################################################
# 异常检测
############################################################################
if aspect_ratios_global is None and aspect_ratios_per_layer is None:
raise ValueError("`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified.")
if aspect_ratios_per_layer:
if len(aspect_ratios_per_layer) != n_predictor_layers:
raise ValueError("It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}.".format(n_predictor_layers, len(aspect_ratios_per_layer)))
if (min_scale is None or max_scale is None) and scales is None:
raise ValueError("Either `min_scale` and `max_scale` or `scales` need to be specified.")
if scales:
if len(scales) != n_predictor_layers+1:
raise ValueError("It must be either scales is None or len(scales) == {}, but len(scales) == {}.".format(n_predictor_layers+1, len(scales)))
else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale`
scales = np.linspace(min_scale, max_scale, n_predictor_layers+1)
if len(variances) != 4: # We need one variance value for each of the four box coordinates
raise ValueError("4 variance values must be pased, but {} values were received.".format(len(variances)))
variances = np.array(variances)
if np.any(variances <= 0):
raise ValueError("All variances must be >0, but the variances given are {}".format(variances))
if (not (steps is None)) and (len(steps) != n_predictor_layers):
raise ValueError("You must provide at least one step value per predictor layer.")
if (not (offsets is None)) and (len(offsets) != n_predictor_layers):
raise ValueError("You must provide at least one offset value per predictor layer.")
############################################################################
# 计算预设框参数
############################################################################
# Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers.
if aspect_ratios_per_layer:
aspect_ratios = aspect_ratios_per_layer
else:
aspect_ratios = [aspect_ratios_global] * n_predictor_layers
# Compute the number of boxes to be predicted per cell for each predictor layer.
# We need this so that we know how many channels the predictor layers need to have.
if aspect_ratios_per_layer:
n_boxes = []
for ar in aspect_ratios_per_layer:
if (1 in ar) & two_boxes_for_ar1:
n_boxes.append(len(ar) + 1) # +1 for the second box for aspect ratio 1
else:
n_boxes.append(len(ar))
else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer
if (1 in aspect_ratios_global) & two_boxes_for_ar1:
n_boxes = len(aspect_ratios_global) + 1
else:
n_boxes = len(aspect_ratios_global)
n_boxes = [n_boxes] * n_predictor_layers
if steps is None:
steps = [None] * n_predictor_layers
if offsets is None:
offsets = [None] * n_predictor_layers
############################################################################
# 搭建网络
############################################################################
x = Input(shape=(img_height, img_width, img_channels))
# The following identity layer is only needed so that the subsequent lambda layers can be optional.
x1 = Lambda(lambda z: z, output_shape=(img_height, img_width, img_channels), name='identity_layer')(x)
if not (subtract_mean is None):
x1 = Lambda(lambda z: z - np.array(subtract_mean), output_shape=(img_height, img_width, img_channels), name='input_mean_normalization')(x1)
if not (divide_by_stddev is None):
x1 = Lambda(lambda z: z / np.array(divide_by_stddev), output_shape=(img_height, img_width, img_channels), name='input_stddev_normalization')(x1)
if swap_channels and (img_channels == 3):
x1 = Lambda(lambda z: z[...,::-1], output_shape=(img_height, img_width, img_channels), name='input_channel_swap')(x1)
conv1 = Conv2D(32, (5, 5), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1')(x1)
conv1 = BatchNormalization(axis=3, momentum=0.99, name='bn1')(conv1) # Tensorflow uses filter format [filter_height, filter_width, in_channels, out_channels], hence axis = 3
conv1 = ELU(name='elu1')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2), name='pool1')(conv1)
conv2 = Conv2D(48, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv2')(pool1)
conv2 = BatchNormalization(axis=3, momentum=0.99, name='bn2')(conv2)
conv2 = ELU(name='elu2')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2), name='pool2')(conv2)
conv3 = Conv2D(64, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv3')(pool2)
conv3 = BatchNormalization(axis=3, momentum=0.99, name='bn3')(conv3)
conv3 = ELU(name='elu3')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2), name='pool3')(conv3)
conv4 = Conv2D(64, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv4')(pool3)
conv4 = BatchNormalization(axis=3, momentum=0.99, name='bn4')(conv4)
conv4 = ELU(name='elu4')(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2), name='pool4')(conv4)
conv5 = Conv2D(48, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv5')(pool4)
conv5 = BatchNormalization(axis=3, momentum=0.99, name='bn5')(conv5)
conv5 = ELU(name='elu5')(conv5)
pool5 = MaxPooling2D(pool_size=(2, 2), name='pool5')(conv5)
conv6 = Conv2D(48, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv6')(pool5)
conv6 = BatchNormalization(axis=3, momentum=0.99, name='bn6')(conv6)
conv6 = ELU(name='elu6')(conv6)
pool6 = MaxPooling2D(pool_size=(2, 2), name='pool6')(conv6)
conv7 = Conv2D(32, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv7')(pool6)
conv7 = BatchNormalization(axis=3, momentum=0.99, name='bn7')(conv7)
conv7 = ELU(name='elu7')(conv7)
# The next part is to add the convolutional predictor layers on top of the base network
# that we defined above. Note that I use the term "base network" differently than the paper does.
# To me, the base network is everything that is not convolutional predictor layers or anchor
# box layers. In this case we'll have four predictor layers, but of course you could
# easily rewrite this into an arbitrarily deep base network and add an arbitrary number of
# predictor layers on top of the base network by simply following the pattern shown here.
# 预测层
# Build the convolutional predictor layers on top of conv layers 4, 5, 6, and 7.
# We build two predictor layers on top of each of these layers: One for class prediction (classification), one for box coordinate prediction (localization)
# We precidt `n_classes` confidence values for each box, hence the `classes` predictors have depth `n_boxes * n_classes`
# We predict 4 box coordinates for each box, hence the `boxes` predictors have depth `n_boxes * 4`
# Output shape of `classes`: `(batch, height, width, n_boxes * n_classes)`
classes4 = Conv2D(n_boxes[0] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes4')(conv4)
classes5 = Conv2D(n_boxes[1] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes5')(conv5)
classes6 = Conv2D(n_boxes[2] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes6')(conv6)
classes7 = Conv2D(n_boxes[3] * n_classes, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='classes7')(conv7)
# Output shape of `boxes`: `(batch, height, width, n_boxes * 4)`
boxes4 = Conv2D(n_boxes[0] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes4')(conv4)
boxes5 = Conv2D(n_boxes[1] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes5')(conv5)
boxes6 = Conv2D(n_boxes[2] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes6')(conv6)
boxes7 = Conv2D(n_boxes[3] * 4, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='boxes7')(conv7)
# Generate the anchor boxes
# Output shape of `anchors`: `(batch, height, width, n_boxes, 8)`
anchors4 = AnchorBoxes(img_height, img_width, this_scale=scales[0], next_scale=scales[1], aspect_ratios=aspect_ratios[0],
two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[0], this_offsets=offsets[0],
limit_boxes=limit_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors4')(boxes4)
anchors5 = AnchorBoxes(img_height, img_width, this_scale=scales[1], next_scale=scales[2], aspect_ratios=aspect_ratios[1],
two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[1], this_offsets=offsets[1],
limit_boxes=limit_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors5')(boxes5)
anchors6 = AnchorBoxes(img_height, img_width, this_scale=scales[2], next_scale=scales[3], aspect_ratios=aspect_ratios[2],
two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[2], this_offsets=offsets[2],
limit_boxes=limit_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors6')(boxes6)
anchors7 = AnchorBoxes(img_height, img_width, this_scale=scales[3], next_scale=scales[4], aspect_ratios=aspect_ratios[3],
two_boxes_for_ar1=two_boxes_for_ar1, this_steps=steps[3], this_offsets=offsets[3],
limit_boxes=limit_boxes, variances=variances, coords=coords, normalize_coords=normalize_coords, name='anchors7')(boxes7)
# Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)`
# We want the classes isolated in the last axis to perform softmax on them
classes4_reshaped = Reshape((-1, n_classes), name='classes4_reshape')(classes4)
classes5_reshaped = Reshape((-1, n_classes), name='classes5_reshape')(classes5)
classes6_reshaped = Reshape((-1, n_classes), name='classes6_reshape')(classes6)
classes7_reshaped = Reshape((-1, n_classes), name='classes7_reshape')(classes7)
# Reshape the box coordinate predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)`
# We want the four box coordinates isolated in the last axis to compute the smooth L1 loss
boxes4_reshaped = Reshape((-1, 4), name='boxes4_reshape')(boxes4)
boxes5_reshaped = Reshape((-1, 4), name='boxes5_reshape')(boxes5)
boxes6_reshaped = Reshape((-1, 4), name='boxes6_reshape')(boxes6)
boxes7_reshaped = Reshape((-1, 4), name='boxes7_reshape')(boxes7)
# Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)`
anchors4_reshaped = Reshape((-1, 8), name='anchors4_reshape')(anchors4)
anchors5_reshaped = Reshape((-1, 8), name='anchors5_reshape')(anchors5)
anchors6_reshaped = Reshape((-1, 8), name='anchors6_reshape')(anchors6)
anchors7_reshaped = Reshape((-1, 8), name='anchors7_reshape')(anchors7)
# Concatenate the predictions from the different layers and the assosciated anchor box tensors
# Axis 0 (batch) and axis 2 (n_classes or 4, respectively) are identical for all layer predictions,
# so we want to concatenate along axis 1
# Output shape of `classes_concat`: (batch, n_boxes_total, n_classes)
classes_concat = Concatenate(axis=1, name='classes_concat')([classes4_reshaped,
classes5_reshaped,
classes6_reshaped,
classes7_reshaped])
# Output shape of `boxes_concat`: (batch, n_boxes_total, 4)
boxes_concat = Concatenate(axis=1, name='boxes_concat')([boxes4_reshaped,
boxes5_reshaped,
boxes6_reshaped,
boxes7_reshaped])
# Output shape of `anchors_concat`: (batch, n_boxes_total, 8)
anchors_concat = Concatenate(axis=1, name='anchors_concat')([anchors4_reshaped,
anchors5_reshaped,
anchors6_reshaped,
anchors7_reshaped])
# The box coordinate predictions will go into the loss function just the way they are,
# but for the class predictions, we'll apply a softmax activation layer first
classes_softmax = Activation('softmax', name='classes_softmax')(classes_concat)
# Concatenate the class and box coordinate predictions and the anchors to one large predictions tensor
# Output shape of `predictions`: (batch, n_boxes_total, n_classes + 4 + 8)
predictions = Concatenate(axis=2, name='predictions')([classes_softmax, boxes_concat, anchors_concat])
# 模型
model = Model(inputs=x, outputs=predictions)
if return_predictor_sizes:
# Get the spatial dimensions (height, width) of the convolutional predictor layers, we need them to generate the default boxes
# The spatial dimensions are the same for the `classes` and `boxes` predictors
predictor_sizes = np.array([classes4._keras_shape[1:3],
classes5._keras_shape[1:3],
classes6._keras_shape[1:3],
classes7._keras_shape[1:3]])
return model, predictor_sizes
else:
return model
def build_ssd7(image_size,
n_classes,
mode='training',
l2_regularization=0.0,
min_scale=0.1,
max_scale=0.9,
scales=None,
aspect_ratios_global=[0.5, 1.0, 2.0],
aspect_ratios_per_layer=None,
two_boxes_for_ar1=True,
steps=None,
offsets=None,
clip_boxes=False,
variances=[1.0, 1.0, 1.0, 1.0],
coords='centroids',
normalize_coords=False,
subtract_mean=None,
divide_by_stddev=None,
swap_channels=False,
confidence_thresh=0.01,
iou_threshold=0.45,
top_k=200,
nms_max_output_size=400,
return_predictor_sizes=False):
"""
Build a SSD model with Keras.
The model consists of convolutional feature layers and
a number of convolutional predictor layers that take their input
from different feature layers. The model is fully convolutional.
This implementation has 7 convolutional layers and 4 convolutional predictor
layers that take their input from layers 4, 5, 6, and 7, respectively.
Arguments:
image_size (tuple): The input image size in the format `(height, width, channels)`.
n_classes (int): The number of positive classes, e.g. 20 for Pascal VOC, 80 for MS COCO.
mode (str, optional): One of 'training', 'inference' and 'inference_fast'. In 'training' mode,
the model outputs the raw prediction tensor, while in 'inference' and 'inference_fast' modes,
the raw predictions are decoded into absolute coordinates and filtered via confidence thresholding,
non-maximum suppression, and top-k filtering. The difference between latter two modes is that
'inference' follows the exact procedure of the original Caffe implementation, while
'inference_fast' uses a faster prediction decoding procedure.
l2_regularization (float, optional): The L2-regularization rate. Applies to all convolutional layers.
min_scale (float, optional): The smallest scaling factor for the size of the anchor boxes as a fraction
of the shorter side of the input images.
max_scale (float, optional): The largest scaling factor for the size of the anchor boxes as a fraction
of the shorter side of the input images. All scaling factors between the smallest and the
largest will be linearly interpolated. Note that the second to last of the linearly interpolated
scaling factors will actually be the scaling factor for the last predictor layer, while the last
scaling factor is used for the second box for aspect ratio 1 in the last predictor layer
if `two_boxes_for_ar1` is `True`.
scales (list, optional): A list of floats containing scaling factors per convolutional predictor layer.
This list must be one element longer than the number of predictor layers. The first `k` elements are the
scaling factors for the `k` predictor layers, while the last element is used for the second box
for aspect ratio 1 in the last predictor layer if `two_boxes_for_ar1` is `True`. This additional
last scaling factor must be passed either way, even if it is not being used. If a list is passed,
this argument overrides `min_scale` and `max_scale`. All scaling factors must be greater than zero.
aspect_ratios_global (list, optional): The list of aspect ratios for which anchor boxes are to be
generated. This list is valid for all predictor layers. The original implementation uses more aspect ratios
for some predictor layers and fewer for others. If you want to do that, too, then use the next argument instead.
aspect_ratios_per_layer (list, optional): A list containing one aspect ratio list for each predictor layer.
This allows you to set the aspect ratios for each predictor layer individually. If a list is passed,
it overrides `aspect_ratios_global`.
two_boxes_for_ar1 (bool, optional): Only relevant for aspect ratio lists that contain 1. Will be ignored otherwise.
If `True`, two anchor boxes will be generated for aspect ratio 1. The first will be generated
using the scaling factor for the respective layer, the second one will be generated using
geometric mean of said scaling factor and next bigger scaling factor.
steps (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be
either ints/floats or tuples of two ints/floats. These numbers represent for each predictor layer how many
pixels apart the anchor box center points should be vertically and horizontally along the spatial grid over
the image. If the list contains ints/floats, then that value will be used for both spatial dimensions.
If the list contains tuples of two ints/floats, then they represent `(step_height, step_width)`.
If no steps are provided, then they will be computed such that the anchor box center points will form an
equidistant grid within the image dimensions.
offsets (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be
either floats or tuples of two floats. These numbers represent for each predictor layer how many
pixels from the top and left boarders of the image the top-most and left-most anchor box center points should be
as a fraction of `steps`. The last bit is important: The offsets are not absolute pixel values, but fractions
of the step size specified in the `steps` argument. If the list contains floats, then that value will
be used for both spatial dimensions. If the list contains tuples of two floats, then they represent
`(vertical_offset, horizontal_offset)`. If no offsets are provided, then they will default to 0.5 of the step size,
which is also the recommended setting.
clip_boxes (bool, optional): If `True`, clips the anchor box coordinates to stay within image boundaries.
variances (list, optional): A list of 4 floats >0. The anchor box offset for each coordinate will be divided by
its respective variance value.
coords (str, optional): The box coordinate format to be used internally by the model (i.e. this is not the input format
of the ground truth labels). Can be either 'centroids' for the format `(cx, cy, w, h)` (box center coordinates, width,
and height), 'minmax' for the format `(xmin, xmax, ymin, ymax)`, or 'corners' for the format `(xmin, ymin, xmax, ymax)`.
normalize_coords (bool, optional): Set to `True` if the model is supposed to use relative instead of absolute coordinates,
i.e. if the model predicts box coordinates within [0,1] instead of absolute coordinates.
subtract_mean (array-like, optional): `None` or an array-like object of integers or floating point values
of any shape that is broadcast-compatible with the image shape. The elements of this array will be
subtracted from the image pixel intensity values. For example, pass a list of three integers
to perform per-channel mean normalization for color images.
divide_by_stddev (array-like, optional): `None` or an array-like object of non-zero integers or
floating point values of any shape that is broadcast-compatible with the image shape. The image pixel
intensity values will be divided by the elements of this array. For example, pass a list
of three integers to perform per-channel standard deviation normalization for color images.
swap_channels (list, optional): Either `False` or a list of integers representing the desired order in which the input
image channels should be swapped.
confidence_thresh (float, optional): A float in [0,1), the minimum classification confidence in a specific
positive class in order to be considered for the non-maximum suppression stage for the respective class.
A lower value will result in a larger part of the selection process being done by the non-maximum suppression
stage, while a larger value will result in a larger part of the selection process happening in the confidence
thresholding stage.
iou_threshold (float, optional): A float in [0,1]. All boxes that have a Jaccard similarity of greater than `iou_threshold`
with a locally maximal box will be removed from the set of predictions for a given class, where 'maximal' refers
to the box's confidence score.
top_k (int, optional): The number of highest scoring predictions to be kept for each batch item after the
non-maximum suppression stage.
nms_max_output_size (int, optional): The maximal number of predictions that will be left over after the NMS stage.
return_predictor_sizes (bool, optional): If `True`, this function not only returns the model, but also
a list containing the spatial dimensions of the predictor layers. This isn't strictly necessary since
you can always get their sizes easily via the Keras API, but it's convenient and less error-prone
to get them this way. They are only relevant for training anyway (SSDBoxEncoder needs to know the
spatial dimensions of the predictor layers), for inference you don't need them.
Returns:
model: The Keras SSD model.
predictor_sizes (optional): A Numpy array containing the `(height, width)` portion
of the output tensor shape for each convolutional predictor layer. During
training, the generator function needs this in order to transform
the ground truth labels into tensors of identical structure as the
output tensors of the model, which is in turn needed for the cost
function.
References:
https://arxiv.org/abs/1512.02325v5
"""
n_predictor_layers = 4 # The number of predictor conv layers
n_classes += 1 # Account for the background class
img_height, img_width, img_channels = image_size
#################
# Some exceptions
#################
if aspect_ratios_global is None and aspect_ratios_per_layer is None:
raise ValueError(
'"aspect_ratio_global" and "aspect_ratios_per_layer" cannot both be None.'
'At least one needs to be specified.')
if aspect_ratios_per_layer and len(
aspect_ratios_per_layer) != n_predictor_layers:
raise ValueError(
'It must be either "aspect_ratio_per_layer" is None or '
f'len(aspect_ratio_per_layer == {n_predictor_layers}, '
f'but len(aspect_ratio_per_layer) == {len(aspect_ratios_per_layer)}.'
)
if (min_scale is None or max_scale is None) and scales is None:
raise ValueError(
'Either "min_scale" and "max_scale" or "scales" need to be specified.'
)
if scales:
if len(scales) != n_predictor_layers + 1:
raise ValueError(
f'It must be either "scales" is None or len(scales) =={n_predictor_layers + 1} '
f'but len(scales) == {len(scales)}')
# If no explicit list of scaling factors is passed,
# compute the list of scaling from min_scale and max_scale
else:
scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1)
if len(variances) != 4:
raise ValueError(
f'4 variance values must be passed, but {len(variances)} values were received.'
)
variances = np.array(variances)
if np.any(variances <= 0):
raise ValueError(
f'All variances must be > 0, but the variances given are {variances}'
)
if steps is not None and len(steps) != n_predictor_layers:
raise ValueError(
'You must provide at least one step value per predictor layer.')
if offsets is not None and len(offsets) != n_predictor_layers:
raise ValueError(
'You must provide at least one offset value per predictor layer.')
###################################
# Compute the anchor box parameters
###################################
# Set the aspect ratio for each predictor layer.
# There are only need for the anchor box layers.
if aspect_ratios_per_layer:
aspect_ratios = aspect_ratios_per_layer
else:
aspect_ratios = [aspect_ratios_global] * n_predictor_layers
# Compute the number of boxes to be predicted per cell for each predictor layer.
# We need this so that we know how many channels the predictor layer needs to have.
if aspect_ratios_per_layer:
n_boxes = []
for ar in aspect_ratios_per_layer:
if (1 in ar) & two_boxes_for_ar1:
n_boxes.append(
len(ar) +
1) # +1 for the second box for the aspect ratio 1
else:
n_boxes.append(len(ar))
# If only a global aspect ratio list was passed,
# then the number of boxes is the same for each predictor layer
else:
if (1 in aspect_ratios_global) & two_boxes_for_ar1:
n_box = len(aspect_ratios_global) + 1
else:
n_box = len(aspect_ratios_global)
n_boxes = [n_box] * n_predictor_layers
if steps is None:
steps = [None] * n_predictor_layers
if offsets is None:
offsets = [None] * n_predictor_layers
##############################################
# Define functions for the lambda layers below
##############################################
def identity_layer(tensor):
return tensor
def input_mean_normalization(tensor):
return tensor - np.array(subtract_mean)
def input_stddev_normalization(tensor):
return tensor / np.array(divide_by_stddev)
def input_channel_swap(tensor):
if len(swap_channels) == 3:
return K.stack(tensor[..., swap_channels[0]],
tensor[..., swap_channels[1]],
tensor[..., swap_channels[2]],
axis=-1)
elif len(swap_channels) == 4:
return K.stack(tensor[..., swap_channels[0]],
tensor[..., swap_channels[1]],
tensor[..., swap_channels[2]],
tensor[..., swap_channels[3]],
axis=-1)
###################
# Build the network
###################
x = Input(shape=(img_height, img_width, img_channels))
# The following identity layer is only needed so that the subsequent lambda layer can be optional.
x1 = Lambda(identity_layer,
output_shape=(img_height, img_width, img_channels),
name='identity_layer')(x)
if subtract_mean is not None:
x1 = Lambda(input_mean_normalization,
output_shape=(img_height, img_width, img_channels),
name='input_mean_normalization')(x1)
if divide_by_stddev is not None:
x1 = Lambda(input_stddev_normalization,
output_shape=(img_height, img_width, img_channels),
name='input_stddev_normalization')(x1)
if swap_channels:
x1 = Lambda(input_channel_swap,
output_shape=(img_height, img_width, img_channels),
name='input_channel_swap')(x1)
# Layer 1
conv1 = Conv2D(filters=32,
kernel_size=(5, 5),
strides=(1, 1),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_regularization),
name='conv1')(x1)
# Tensorflow uses filter format [filter_height, filter_width, in_channels, out_channels], hence axis = 3
conv1 = BatchNormalization(axis=3, momentum=0.99, name='bn1')(conv1)
conv1 = ELU(name='elu1')(conv1)
conv1 = MaxPooling2D(pool_size=(2, 2), name='pool1')(conv1)
# Layer 2
conv2 = Conv2D(filters=48,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_regularization),
name='conv2')(conv1)
conv2 = BatchNormalization(axis=3, momentum=0.99, name='bn2')(conv2)
conv2 = ELU(name='elu2')(conv2)
conv2 = MaxPooling2D(pool_size=(2, 2), name='pool2')(conv2)
# Layer 3
conv3 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_regularization),
name='conv3')(conv2)
conv3 = BatchNormalization(axis=3, momentum=0.99, name='bn3')(conv3)
conv3 = ELU(name='elu3')(conv3)
conv3 = MaxPooling2D(pool_size=(2, 2), name='pool3')(conv3)
# Layer 4
conv4 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_regularization),
name='conv4')(conv3)
conv4 = BatchNormalization(axis=3, momentum=0.99, name='bn4')(conv4)
conv4 = ELU(name='elu4')(conv4)
conv4 = MaxPooling2D(pool_size=(2, 2), name='pool4')(conv4)
# Layer 5
conv5 = Conv2D(filters=48,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_regularization),
name='conv5')(conv4)
conv5 = BatchNormalization(axis=3, momentum=0.99, name='bn5')(conv5)
conv5 = ELU(name='elu5')(conv5)
conv5 = MaxPooling2D(pool_size=(2, 2), name='pool5')(conv5)
# Layer 6
conv6 = Conv2D(filters=48,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_regularization),
name='conv6')(conv5)
conv6 = BatchNormalization(axis=3, momentum=0.99, name='bn6')(conv6)
conv6 = ELU(name='elu6')(conv6)
conv6 = MaxPooling2D(pool_size=(2, 2), name='pool6')(conv6)
# Layer 7
conv7 = Conv2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_regularization),
name='conv7')(conv6)
conv7 = BatchNormalization(axis=3, momentum=0.99, name='bn7')(conv7)
conv7 = ELU(name='elu7')(conv7)
# The next part is to add the convolutional predictor layers.
# Build the convolutional predictor layers on the top of conv layers 4, 5, 6 and 7.
# We build two predictor layers on top of each of these layers:
# 1. One for class prediction (classification).
# 2. One for box coordinate prediction (localization).
# We predict n_classes confidence values for each box,
# hence the classes predictors have depth n_boxes * n_classes.
# We predict 4 coordinates for each box,
# hence the boxes predictors have depth n_boxes * 4.
# Output shape of class predictors: (batch, height, width, n_boxes * n_classes)
classes4 = Conv2D(filters=n_boxes[0] * n_classes,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_regularization),
name='classes4')(conv4)
classes5 = Conv2D(filters=n_boxes[1] * n_classes,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_regularization),
name='classes5')(conv5)
classes6 = Conv2D(filters=n_boxes[2] * n_classes,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_regularization),
name='classes6')(conv6)
classes7 = Conv2D(filters=n_boxes[3] * n_classes,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_regularization),
name='classes7')(conv7)
# Output shape of boxes predictor: (batch, height, width, n_boxes * 4)
boxes4 = Conv2D(filters=n_boxes[0] * 4,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_regularization),
name='boxes4')(conv4)
boxes5 = Conv2D(filters=n_boxes[1] * 4,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_regularization),
name='boxes5')(conv5)
boxes6 = Conv2D(filters=n_boxes[2] * 4,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_regularization),
name='boxes6')(conv6)
boxes7 = Conv2D(filters=n_boxes[3] * 4,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_regularization),
name='boxes7')(conv7)
# Generate the anchor boxes
# Output shape of anchors: (batch, height, width, n_boxes, 8)
anchors4 = AnchorBoxes(img_height,
img_width,
this_scale=scales[0],
next_scale=scales[1],
aspect_ratios=aspect_ratios[0],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[0],
this_offsets=offsets[0],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='anchors4')(boxes4)
anchors5 = AnchorBoxes(img_height,
img_width,
this_scale=scales[1],
next_scale=scales[2],
aspect_ratios=aspect_ratios[1],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[1],
this_offsets=offsets[1],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='anchors5')(boxes5)
anchors6 = AnchorBoxes(img_height,
img_width,
this_scale=scales[2],
next_scale=scales[3],
aspect_ratios=aspect_ratios[2],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[2],
this_offsets=offsets[2],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='anchors6')(boxes6)
anchors7 = AnchorBoxes(img_height,
img_width,
this_scale=scales[3],
next_scale=scales[4],
aspect_ratios=aspect_ratios[3],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[3],
this_offsets=offsets[3],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='anchors7')(boxes7)
conv_class_box_anchor = [
(classes4, boxes4, anchors4),
(classes5, boxes5, anchors5),
(classes6, boxes6, anchors6),
(classes7, boxes7, anchors7),
]
# Reshape the class predictions, yielding 3D tensors of shape
# (batch, height * width * n_boxes, n_classes)
# We want the class isolated in the last axis to perform softmax on them
classes4_reshaped = Reshape((-1, n_classes),
name='classes4_reshaped')(classes4)
classes5_reshaped = Reshape((-1, n_classes),
name='classes5_reshaped')(classes5)
classes6_reshaped = Reshape((-1, n_classes),
name='classes6_reshaped')(classes6)
classes7_reshaped = Reshape((-1, n_classes),
name='classes7_reshaped')(classes7)
# Reshape the box coordinate prediction, yielding 3D tensors of shape
# (batch, height * width * n_boxes, 4)
# We want the four box coordinates isolated in the last axis to compute the smooth L1 loss
boxes4_reshaped = Reshape((-1, 4), name='boxes4_reshaped')(boxes4)
boxes5_reshaped = Reshape((-1, 4), name='boxes5_reshaped')(boxes5)
boxes6_reshaped = Reshape((-1, 4), name='boxes6_reshaped')(boxes6)
boxes7_reshaped = Reshape((-1, 4), name='boxes7_reshaped')(boxes7)
# Reshape the anchor box tensors, yielding 3D tensor of shape
# (batch, height * width * n_boxes, 8)
anchors4_reshaped = Reshape((-1, 8), name='anchors4_reshaped')(anchors4)
anchors5_reshaped = Reshape((-1, 8), name='anchors5_reshaped')(anchors5)
anchors6_reshaped = Reshape((-1, 8), name='anchors6_reshaped')(anchors6)
anchors7_reshaped = Reshape((-1, 8), name='anchors7_reshaped')(anchors7)
# Concatenate the predictions from the different layers and
# the associated anchor box tensors.
# Output shape classes_concat: (batch, n_boxes_total, n_classes)
classes_concat = Concatenate(axis=1, name='classes_concat')([
classes4_reshaped, classes5_reshaped, classes6_reshaped,
classes7_reshaped
])
# Output shape of boxes_concat: (batch, n_boxes_total, 4)
boxes_concat = Concatenate(axis=1, name='boxes_concat')(
[boxes4_reshaped, boxes5_reshaped, boxes6_reshaped, boxes7_reshaped])
# Output shape of anchors_concat: (batch, n_boxes_total, 8)
anchors_concat = Concatenate(axis=1, name='anchors_concat')([
anchors4_reshaped, anchors5_reshaped, anchors6_reshaped,
anchors7_reshaped
])
# The box coordinate predictions will go into the loss functions just the way they are,
# but for the class predictions, we will apply a softmax activation layer first.
classes_softmax = Activation('softmax',
name='classes_softmax')(classes_concat)
# Concatenate the class and box coordinate predictions and the anchors to
# one large predictions tensor.
# Output shape of predictions: (batch, n_boxes_total, n_classes + 4 + 8)
predictions = Concatenate(axis=2, name='predictions')(
[classes_softmax, boxes_concat, anchors_concat])
if mode == 'training':
model = Model(inputs=x, outputs=predictions)
elif mode == 'inference':
decoded_predictions = DecodeDetections(
confidence_thresh=confidence_thresh,
iou_threshold=iou_threshold,
top_k=top_k,
nms_max_output_size=nms_max_output_size,
coords=coords,
normalize_coords=normalize_coords,
img_height=img_height,
img_width=img_width,
name='decoded_predictions')(predictions)
model = Model(inputs=x, outputs=decoded_predictions)
else:
raise ValueError(
"`mode` must be one of 'training', 'inference' or 'inference_fast', "
"but received '{}'.".format(mode))
if return_predictor_sizes:
# The spatial dimensions are the same for the `classes` and `boxes` predictor layers.
predictor_sizes = np.array([
classes4._keras_shape[1:3], classes5._keras_shape[1:3],
classes6._keras_shape[1:3], classes7._keras_shape[1:3]
])
return model, predictor_sizes
else:
return model
import keras
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Model
from keras.layers import Input, Conv2D, ELU, MaxPooling2D, Flatten, Dense, Dropout, normalization
from keras.optimizers import SGD, rmsprop, adam
from keras import backend as K
import os
import scipy.io as sio
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
#left image
left_image = Input(shape=(32, 32, 3))
#conv1
left_conv1 = Conv2D(32, (3, 3), padding='same', name='conv1_left')(left_image)
left_elu1 = ELU()(left_conv1)
left_pool1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2),
name='pool1_left')(left_elu1)
#conv2
left_conv2 = Conv2D(32, (3, 3), padding='same', name='conv2_left')(left_pool1)
left_elu2 = ELU()(left_conv2)
left_pool2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2),
name='pool2_left')(left_elu2)
#conv3
left_conv3 = Conv2D(64, (3, 3), padding='same', name='conv3_left')(left_pool2)
left_elu3 = ELU()(left_conv3)
#conv4
left_conv4 = Conv2D(64, (3, 3), padding='same', name='conv4_left')(left_elu3)
left_elu4 = ELU()(left_conv4)
#conv5
left_conv5 = Conv2D(128, (3, 3), padding='same', name='conv5_left')(left_elu4)
w = 630 // 5
r = np.random.randint(inputs.shape[1] - w + 1)
return inputs[:, r:r + w, :]
for j in range(n_models):
msets[j] = " " # mset
m_in = Input(shape=x[0][0].shape)
# m_off = Lambda(offset_slice)(m_in)
# m_noise = GaussianNoise(np.std(x[0][0] / 100))(m_off) # how much noice to have????
m_t = Conv1D(30, 64, padding='causal')(m_in)
m_t = BatchNormalization()(m_t)
m_t = ELU()(m_t)
m_t = AveragePooling1D(2)(m_t)
m_t = Dropout(0.2)(m_t)
m_t = Conv1D(15, 32, padding='causal')(m_t)
m_t = BatchNormalization()(m_t)
m_t = ELU()(m_t)
m_t = AveragePooling1D(2)(m_t)
m_t = Dropout(0.3)(m_t)
m_t = Conv1D(10, 16, padding='causal')(m_t)
m_t = BatchNormalization()(m_t)
m_t = ELU()(m_t)
m_t = AveragePooling1D(2)(m_t)
m_t = Dropout(0.4)(m_t)
def test_delete_channels_advanced_activations(channel_index, data_format):
layer_test_helper_flatten_2d(LeakyReLU(), channel_index, data_format)
layer_test_helper_flatten_2d(ELU(), channel_index, data_format)
layer_test_helper_flatten_2d(ThresholdedReLU(), channel_index, data_format)
def build_model3(n_classes, img_width=224, img_height=224, channels=3):
if K.image_data_format() == 'channels_first':
input_shape = (channels, img_width, img_height)
else:
input_shape = (img_width, img_height, channels)
# model = Sequential()
#ADD CRAZYNET HERE
l2_reg = 0.0
x = Input(shape=input_shape)
# layer_sizes = [48, 64, 64, 80, 80, 96, 96, 112, 112, 128, 128, 128, 128, 112, 112, 96, 96, 80, 80, 64, 64, 48, 48]
layer_sizes = [48, 64, 64, 48, 48]
conv1 = Conv2D(32, (5,5), padding="same",
kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1')(x)
conv1 = BatchNormalization(axis=3, momentum=0.99, name='bn1')(conv1)
conv1 = ELU(name='elu')(conv1)
drop1 = Dropout(0.05)(conv1)
pool1 = MaxPooling2D(name='pool1')(drop1)
i = 2
last_pool = pool1
for size in layer_sizes:
conv = Conv2D(size, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv{0}'.format(i))(last_pool)
conv = BatchNormalization(axis=3, momentum=0.99, name='bn{0}'.format(i))(conv)
conv = ELU(name='elu{0}'.format(i))(conv)
last_pool = MaxPooling2D(pool_size=(2, 2), name='pool{0}'.format(i))(conv)
# if(i%3 == 0):
# conv = Dropout(0.25)(conv)
# if i%1==0:
# if i == 5:
#
# last_pool = MaxPooling2D(pool_size=(2, 2), name='pool{0}'.format(i))(conv)
i+=1
conv_last = Conv2D(32, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv_last')(last_pool)
conv_last = BatchNormalization(axis=3, momentum=0.99, name='bn_last')(conv_last)
conv_last = ELU(name='elu_last')(conv_last)
flat = Flatten()(conv_last)
dense = Dense(128, activation='relu')(flat)
drop = Dropout(0.15)(dense)
dense = Dense(256, activation='relu')(drop)
drop = Dropout(0.15)(dense)
dense = Dense(512, activation='relu')(drop)
drop = Dropout(0.15)(dense)
dense = Dense(256, activation='relu')(drop)
drop = Dropout(0.25)(dense)
dense = Dense(128, activation='relu')(drop)
# drop = Dropout(0.25)(dense)
pool = GlobalAveragePooling2D( )(dense)
output = Dense(n_classes, activation='softmax')(pool)
# classes_softmax = Activation('softmax', name='classes_softmax')(classes_concat)
model = Model(inputs=x, outputs=output)
return model
ndata = 0 imgsize = 64 # frame size nrows = 64 ncols = 64 # speed, accel, distance, angle real_in = Input(shape=(2, ), name='real_input') # video frame in, grayscale frame_in = Input(shape=(3, nrows, ncols)) # convolution for image input conv1 = Convolution2D(8, 5, 5, border_mode='same') conv_l1 = conv1(frame_in) Econv_l1 = ELU()(conv_l1) pool_l1 = MaxPooling2D(pool_size=(2, 2))(Econv_l1) conv2 = Convolution2D(8, 5, 5, border_mode='same') conv_l2 = conv2(pool_l1) Econv_l2 = ELU()(conv_l2) pool_l2 = MaxPooling2D(pool_size=(2, 2))(Econv_l2) flat = Flatten()(pool_l2) M = merge([flat, real_in], mode='concat', concat_axis=1) D1 = Dense(64)(M) ED1 = ELU()(D1) D2 = Dense(32)(ED1) ED2 = ELU()(D2)
def build_model(n_classes, img_width=224, img_height=224, channels=3):
if K.image_data_format() == 'channels_first':
input_shape = (channels, img_width, img_height)
else:
input_shape = (img_width, img_height, channels)
model = Sequential()
#ADD CRAZYNET HERE
l2_reg = 0.0
x = Input(shape=input_shape)
layer_sizes = [48, 64, 64, 96, 96, 48, 48]
conv1 = Conv2D(32, (5,5), padding="same",
kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv1')(x)
conv1 = BatchNormalization(axis=3, momentum=0.99, name='bn1')(conv1)
conv1 = ELU(name='elu')(conv1)
pool1 = MaxPooling2D(name='pool1')(conv1)
i = 2
last_pool = pool1
# for size in layer_sizes:
# conv = Conv2D(48, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv{0}'.format(i))(last_pool)
# conv = BatchNormalization(axis=3, momentum=0.99, name='bn{0}'.format(i))(conv)
# conv = ELU(name='elu{0}'.format(i))(conv)
# last_pool = MaxPooling2D(pool_size=(2, 2), name='pool{0}'.format(i))(conv)
conv2 = Conv2D(layer_sizes[i-2], (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv{0}'.format(i))(pool1)
conv2 = BatchNormalization(axis=3, momentum=0.99, name='bn{0}'.format(i))(conv2)
conv2 = ELU(name='elu{0}'.format(i))(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2), name='pool{0}'.format(i))(conv2)
i+=1
conv3 = Conv2D(layer_sizes[i-2], (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv{0}'.format(i))(pool2)
conv3 = BatchNormalization(axis=3, momentum=0.99, name='bn{0}'.format(i))(conv3)
conv3 = ELU(name='elu{0}'.format(i))(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2), name='pool{0}'.format(i))(conv3)
i+=1
conv4 = Conv2D(layer_sizes[i-2], (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv{0}'.format(i))(pool3)
conv4 = BatchNormalization(axis=3, momentum=0.99, name='bn{0}'.format(i))(conv4)
conv4 = ELU(name='elu{0}'.format(i))(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2), name='pool{0}'.format(i))(conv4)
i+=1
conv5 = Conv2D(layer_sizes[i-2], (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv{0}'.format(i))(pool4)
conv5 = BatchNormalization(axis=3, momentum=0.99, name='bn{0}'.format(i))(conv5)
conv5 = ELU(name='elu{0}'.format(i))(conv5)
pool5 = MaxPooling2D(pool_size=(2, 2), name='pool{0}'.format(i))(conv5)
i+=1
conv6 = Conv2D(layer_sizes[i-2], (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv{0}'.format(i))(pool5)
conv6 = BatchNormalization(axis=3, momentum=0.99, name='bn{0}'.format(i))(conv6)
conv6 = ELU(name='elu{0}'.format(i))(conv6)
pool6 = MaxPooling2D(pool_size=(2, 2), name='pool{0}'.format(i))(conv6)
i+=1
conv7 = Conv2D(layer_sizes[i-2], (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv{0}'.format(i))(pool6)
conv7 = BatchNormalization(axis=3, momentum=0.99, name='bn{0}'.format(i))(conv7)
conv7 = ELU(name='elu{0}'.format(i))(conv7)
pool7 = MaxPooling2D(pool_size=(2, 2), name='pool{0}'.format(i))(conv7)
i+=1
conv8 = Conv2D(layer_sizes[i-2], (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv{0}'.format(i))(pool7)
conv8 = BatchNormalization(axis=3, momentum=0.99, name='bn{0}'.format(i))(conv8)
conv8 = ELU(name='elu{0}'.format(i))(conv8)
# pool8 = MaxPooling2D(pool_size=(2, 2), name='pool{0}'.format(i))(conv8)
i+=1
conv_last = Conv2D(32, (3, 3), strides=(1, 1), padding="same", kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg), name='conv_last')(conv8)
conv_last = BatchNormalization(axis=3, momentum=0.99, name='bn_last')(conv_last)
conv_last = ELU(name='elu_last')(conv_last)
flat = Flatten()(conv_last)
dense = Dense(256, activation='relu')(flat)
drop = Dropout(0.1)(dense)
output = Dense(n_classes, activation='softmax')(drop)
# classes_softmax = Activation('softmax', name='classes_softmax')(classes_concat)
model = Model(inputs=x, outputs=output)
return model
def build_model(image_size,
n_classes,
mode='training',
l2_regularization=0.0,
min_scale=0.1,
max_scale=0.9,
scales=None,
aspect_ratios_global=[0.5, 1.0, 2.0],
aspect_ratios_per_layer=None,
two_boxes_for_ar1=True,
steps=None,
offsets=None,
clip_boxes=False,
variances=[1.0, 1.0, 1.0, 1.0],
coords='centroids',
normalize_coords=False,
subtract_mean=None,
divide_by_stddev=None,
swap_channels=False,
confidence_thresh=0.01,
iou_threshold=0.45,
top_k=200,
nms_max_output_size=400,
return_predictor_sizes=False):
'''
Build a Keras model with SSD architecture, see references.
The model consists of convolutional feature layers and a number of convolutional
predictor layers that take their input from different feature layers.
The model is fully convolutional.
The implementation found here is a smaller version of the original architecture
used in the paper (where the base network consists of a modified VGG-16 extended
by a few convolutional feature layers), but of course it could easily be changed to
an arbitrarily large SSD architecture by following the general design pattern used here.
This implementation has 7 convolutional layers and 4 convolutional predictor
layers that take their input from layers 4, 5, 6, and 7, respectively.
Most of the arguments that this function takes are only needed for the anchor
box layers. In case you're training the network, the parameters passed here must
be the same as the ones used to set up `SSDBoxEncoder`. In case you're loading
trained weights, the parameters passed here must be the same as the ones used
to produce the trained weights.
Some of these arguments are explained in more detail in the documentation of the
`SSDBoxEncoder` class.
Note: Requires Keras v2.0 or later. Training currently works only with the
TensorFlow backend (v1.0 or later).
Arguments:
image_size (tuple): The input image size in the format `(height, width, channels)`.
n_classes (int): The number of positive classes, e.g. 20 for Pascal VOC, 80 for MS COCO.
mode (str, optional): One of 'training', 'inference' and 'inference_fast'. In 'training' mode,
the model outputs the raw prediction tensor, while in 'inference' and 'inference_fast' modes,
the raw predictions are decoded into absolute coordinates and filtered via confidence thresholding,
non-maximum suppression, and top-k filtering. The difference between latter two modes is that
'inference' follows the exact procedure of the original Caffe implementation, while
'inference_fast' uses a faster prediction decoding procedure.
l2_regularization (float, optional): The L2-regularization rate. Applies to all convolutional layers.
min_scale (float, optional): The smallest scaling factor for the size of the anchor boxes as a fraction
of the shorter side of the input images.
max_scale (float, optional): The largest scaling factor for the size of the anchor boxes as a fraction
of the shorter side of the input images. All scaling factors between the smallest and the
largest will be linearly interpolated. Note that the second to last of the linearly interpolated
scaling factors will actually be the scaling factor for the last predictor layer, while the last
scaling factor is used for the second box for aspect ratio 1 in the last predictor layer
if `two_boxes_for_ar1` is `True`.
scales (list, optional): A list of floats containing scaling factors per convolutional predictor layer.
This list must be one element longer than the number of predictor layers. The first `k` elements are the
scaling factors for the `k` predictor layers, while the last element is used for the second box
for aspect ratio 1 in the last predictor layer if `two_boxes_for_ar1` is `True`. This additional
last scaling factor must be passed either way, even if it is not being used. If a list is passed,
this argument overrides `min_scale` and `max_scale`. All scaling factors must be greater than zero.
aspect_ratios_global (list, optional): The list of aspect ratios for which anchor boxes are to be
generated. This list is valid for all predictor layers. The original implementation uses more aspect ratios
for some predictor layers and fewer for others. If you want to do that, too, then use the next argument instead.
aspect_ratios_per_layer (list, optional): A list containing one aspect ratio list for each predictor layer.
This allows you to set the aspect ratios for each predictor layer individually. If a list is passed,
it overrides `aspect_ratios_global`.
two_boxes_for_ar1 (bool, optional): Only relevant for aspect ratio lists that contain 1. Will be ignored otherwise.
If `True`, two anchor boxes will be generated for aspect ratio 1. The first will be generated
using the scaling factor for the respective layer, the second one will be generated using
geometric mean of said scaling factor and next bigger scaling factor.
steps (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be
either ints/floats or tuples of two ints/floats. These numbers represent for each predictor layer how many
pixels apart the anchor box center points should be vertically and horizontally along the spatial grid over
the image. If the list contains ints/floats, then that value will be used for both spatial dimensions.
If the list contains tuples of two ints/floats, then they represent `(step_height, step_width)`.
If no steps are provided, then they will be computed such that the anchor box center points will form an
equidistant grid within the image dimensions.
offsets (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be
either floats or tuples of two floats. These numbers represent for each predictor layer how many
pixels from the top and left boarders of the image the top-most and left-most anchor box center points should be
as a fraction of `steps`. The last bit is important: The offsets are not absolute pixel values, but fractions
of the step size specified in the `steps` argument. If the list contains floats, then that value will
be used for both spatial dimensions. If the list contains tuples of two floats, then they represent
`(vertical_offset, horizontal_offset)`. If no offsets are provided, then they will default to 0.5 of the step size,
which is also the recommended setting.
clip_boxes (bool, optional): If `True`, clips the anchor box coordinates to stay within image boundaries.
variances (list, optional): A list of 4 floats >0. The anchor box offset for each coordinate will be divided by
its respective variance value.
coords (str, optional): The box coordinate format to be used internally by the model (i.e. this is not the input format
of the ground truth labels). Can be either 'centroids' for the format `(cx, cy, w, h)` (box center coordinates, width,
and height), 'minmax' for the format `(xmin, xmax, ymin, ymax)`, or 'corners' for the format `(xmin, ymin, xmax, ymax)`.
normalize_coords (bool, optional): Set to `True` if the model is supposed to use relative instead of absolute coordinates,
i.e. if the model predicts box coordinates within [0,1] instead of absolute coordinates.
subtract_mean (array-like, optional): `None` or an array-like object of integers or floating point values
of any shape that is broadcast-compatible with the image shape. The elements of this array will be
subtracted from the image pixel intensity values. For example, pass a list of three integers
to perform per-channel mean normalization for color images.
divide_by_stddev (array-like, optional): `None` or an array-like object of non-zero integers or
floating point values of any shape that is broadcast-compatible with the image shape. The image pixel
intensity values will be divided by the elements of this array. For example, pass a list
of three integers to perform per-channel standard deviation normalization for color images.
swap_channels (list, optional): Either `False` or a list of integers representing the desired order in which the input
image channels should be swapped.
confidence_thresh (float, optional): A float in [0,1), the minimum classification confidence in a specific
positive class in order to be considered for the non-maximum suppression stage for the respective class.
A lower value will result in a larger part of the selection process being done by the non-maximum suppression
stage, while a larger value will result in a larger part of the selection process happening in the confidence
thresholding stage.
iou_threshold (float, optional): A float in [0,1]. All boxes that have a Jaccard similarity of greater than `iou_threshold`
with a locally maximal box will be removed from the set of predictions for a given class, where 'maximal' refers
to the box's confidence score.
top_k (int, optional): The number of highest scoring predictions to be kept for each batch item after the
non-maximum suppression stage.
nms_max_output_size (int, optional): The maximal number of predictions that will be left over after the NMS stage.
return_predictor_sizes (bool, optional): If `True`, this function not only returns the model, but also
a list containing the spatial dimensions of the predictor layers. This isn't strictly necessary since
you can always get their sizes easily via the Keras API, but it's convenient and less error-prone
to get them this way. They are only relevant for training anyway (SSDBoxEncoder needs to know the
spatial dimensions of the predictor layers), for inference you don't need them.
Returns:
model: The Keras SSD model.
predictor_sizes (optional): A Numpy array containing the `(height, width)` portion
of the output tensor shape for each convolutional predictor layer. During
training, the generator function needs this in order to transform
the ground truth labels into tensors of identical structure as the
output tensors of the model, which is in turn needed for the cost
function.
References:
https://arxiv.org/abs/1512.02325v5
'''
n_predictor_layers = 4 # The number of predictor conv layers in the network
n_classes += 1 # Account for the background class.
l2_reg = l2_regularization # Make the internal name shorter.
img_height, img_width, img_channels = image_size[0], image_size[
1], image_size[2]
############################################################################
# Get a few exceptions out of the way.
############################################################################
if aspect_ratios_global is None and aspect_ratios_per_layer is None:
raise ValueError(
"`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified."
)
if aspect_ratios_per_layer:
if len(aspect_ratios_per_layer) != n_predictor_layers:
raise ValueError(
"It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}."
.format(n_predictor_layers, len(aspect_ratios_per_layer)))
if (min_scale is None or max_scale is None) and scales is None:
raise ValueError(
"Either `min_scale` and `max_scale` or `scales` need to be specified."
)
if scales:
if len(scales) != n_predictor_layers + 1:
raise ValueError(
"It must be either scales is None or len(scales) == {}, but len(scales) == {}."
.format(n_predictor_layers + 1, len(scales)))
else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale`
scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1)
if len(
variances
) != 4: # We need one variance value for each of the four box coordinates
raise ValueError(
"4 variance values must be pased, but {} values were received.".
format(len(variances)))
variances = np.array(variances)
if np.any(variances <= 0):
raise ValueError(
"All variances must be >0, but the variances given are {}".format(
variances))
if (not (steps is None)) and (len(steps) != n_predictor_layers):
raise ValueError(
"You must provide at least one step value per predictor layer.")
if (not (offsets is None)) and (len(offsets) != n_predictor_layers):
raise ValueError(
"You must provide at least one offset value per predictor layer.")
############################################################################
# Compute the anchor box parameters.
############################################################################
# Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers.
if aspect_ratios_per_layer:
aspect_ratios = aspect_ratios_per_layer
else:
aspect_ratios = [aspect_ratios_global] * n_predictor_layers
# Compute the number of boxes to be predicted per cell for each predictor layer.
# We need this so that we know how many channels the predictor layers need to have.
if aspect_ratios_per_layer:
n_boxes = []
for ar in aspect_ratios_per_layer:
if (1 in ar) & two_boxes_for_ar1:
n_boxes.append(len(ar) +
1) # +1 for the second box for aspect ratio 1
else:
n_boxes.append(len(ar))
else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer
if (1 in aspect_ratios_global) & two_boxes_for_ar1:
n_boxes = len(aspect_ratios_global) + 1
else:
n_boxes = len(aspect_ratios_global)
n_boxes = [n_boxes] * n_predictor_layers
if steps is None:
steps = [None] * n_predictor_layers
if offsets is None:
offsets = [None] * n_predictor_layers
############################################################################
# Define functions for the Lambda layers below.
############################################################################
def identity_layer(tensor):
return tensor
def input_mean_normalization(tensor):
return tensor - np.array(subtract_mean)
def input_stddev_normalization(tensor):
return tensor / np.array(divide_by_stddev)
def input_channel_swap(tensor):
if len(swap_channels) == 3:
return K.stack([
tensor[..., swap_channels[0]], tensor[..., swap_channels[1]],
tensor[..., swap_channels[2]]
],
axis=-1)
elif len(swap_channels) == 4:
return K.stack([
tensor[..., swap_channels[0]], tensor[..., swap_channels[1]],
tensor[..., swap_channels[2]], tensor[..., swap_channels[3]]
],
axis=-1)
############################################################################
# Build the network.
############################################################################
# x = Input(shape=(img_height, img_width, img_channels))
input_shape = (img_width, img_height, img_channels)
inp = Input(shape=input_shape)
l2_reg = 0.0
# The following identity layer is only needed so that the subsequent lambda layers can be optional.
x1 = Lambda(identity_layer,
output_shape=(img_height, img_width, img_channels),
name='identity_layer')(inp)
if not (subtract_mean is None):
x1 = Lambda(input_mean_normalization,
output_shape=(img_height, img_width, img_channels),
name='input_mean_normalization')(x1)
if not (divide_by_stddev is None):
x1 = Lambda(input_stddev_normalization,
output_shape=(img_height, img_width, img_channels),
name='input_stddev_normalization')(x1)
if swap_channels:
x1 = Lambda(input_channel_swap,
output_shape=(img_height, img_width, img_channels),
name='input_channel_swap')(x1)
#SSD/RESNET Hybrid
x = ZeroPadding2D((3, 3))(x1)
x = Conv2D(64, (7, 7),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv1')(x)
x = BatchNormalization(axis=3, momentum=0.99, name='bn1')(x)
x = ELU(name='elu1')(x)
x = Dropout(0.1)(x)
x = MaxPooling2D(pool_size=(2, 2), name='pool1')(x)
#Additional conv_block
x = conv_block(x, 3, [64, 64, 128], stage=1, block='a', strides=(1, 1))
#RESNET COMPONENTS
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
x1 = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')
x2 = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
x3 = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')
x4 = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')
# The next part is to add the convolutional predictor layers on top of the base network
# that we defined above. Note that I use the term "base network" differently than the paper does.
# To me, the base network is everything that is not convolutional predictor layers or anchor
# box layers. In this case we'll have four predictor layers, but of course you could
# easily rewrite this into an arbitrarily deep base network and add an arbitrary number of
# predictor layers on top of the base network by simply following the pattern shown here.
# Build the convolutional predictor layers on top of conv layers 4, 5, 6, and 7.
# We build two predictor layers on top of each of these layers: One for class prediction (classification), one for box coordinate prediction (localization)
# We precidt `n_classes` confidence values for each box, hence the `classes` predictors have depth `n_boxes * n_classes`
# We predict 4 box coordinates for each box, hence the `boxes` predictors have depth `n_boxes * 4`
# Output shape of `classes`: `(batch, height, width, n_boxes * n_classes)`
classes4 = Conv2D(n_boxes[0] * n_classes, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='classes4')(x1)
classes5 = Conv2D(n_boxes[1] * n_classes, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='classes5')(x2)
classes6 = Conv2D(n_boxes[2] * n_classes, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='classes6')(x3)
classes7 = Conv2D(n_boxes[3] * n_classes, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='classes7')(x4)
# Output shape of `boxes`: `(batch, height, width, n_boxes * 4)`
boxes4 = Conv2D(n_boxes[0] * 4, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='boxes4')(x1)
boxes5 = Conv2D(n_boxes[1] * 4, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='boxes5')(x2)
boxes6 = Conv2D(n_boxes[2] * 4, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='boxes6')(x3)
boxes7 = Conv2D(n_boxes[3] * 4, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='boxes7')(x4)
# Generate the anchor boxes
# Output shape of `anchors`: `(batch, height, width, n_boxes, 8)`
anchors4 = AnchorBoxes(img_height,
img_width,
this_scale=scales[0],
next_scale=scales[1],
aspect_ratios=aspect_ratios[0],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[0],
this_offsets=offsets[0],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='anchors4')(boxes4)
anchors5 = AnchorBoxes(img_height,
img_width,
this_scale=scales[1],
next_scale=scales[2],
aspect_ratios=aspect_ratios[1],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[1],
this_offsets=offsets[1],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='anchors5')(boxes5)
anchors6 = AnchorBoxes(img_height,
img_width,
this_scale=scales[2],
next_scale=scales[3],
aspect_ratios=aspect_ratios[2],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[2],
this_offsets=offsets[2],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='anchors6')(boxes6)
anchors7 = AnchorBoxes(img_height,
img_width,
this_scale=scales[3],
next_scale=scales[4],
aspect_ratios=aspect_ratios[3],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[3],
this_offsets=offsets[3],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='anchors7')(boxes7)
# Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)`
# We want the classes isolated in the last axis to perform softmax on them
classes4_reshaped = Reshape((-1, n_classes),
name='classes4_reshape')(classes4)
classes5_reshaped = Reshape((-1, n_classes),
name='classes5_reshape')(classes5)
classes6_reshaped = Reshape((-1, n_classes),
name='classes6_reshape')(classes6)
classes7_reshaped = Reshape((-1, n_classes),
name='classes7_reshape')(classes7)
# Reshape the box coordinate predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)`
# We want the four box coordinates isolated in the last axis to compute the smooth L1 loss
boxes4_reshaped = Reshape((-1, 4), name='boxes4_reshape')(boxes4)
boxes5_reshaped = Reshape((-1, 4), name='boxes5_reshape')(boxes5)
boxes6_reshaped = Reshape((-1, 4), name='boxes6_reshape')(boxes6)
boxes7_reshaped = Reshape((-1, 4), name='boxes7_reshape')(boxes7)
# Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)`
anchors4_reshaped = Reshape((-1, 8), name='anchors4_reshape')(anchors4)
anchors5_reshaped = Reshape((-1, 8), name='anchors5_reshape')(anchors5)
anchors6_reshaped = Reshape((-1, 8), name='anchors6_reshape')(anchors6)
anchors7_reshaped = Reshape((-1, 8), name='anchors7_reshape')(anchors7)
# Concatenate the predictions from the different layers and the assosciated anchor box tensors
# Axis 0 (batch) and axis 2 (n_classes or 4, respectively) are identical for all layer predictions,
# so we want to concatenate along axis 1
# Output shape of `classes_concat`: (batch, n_boxes_total, n_classes)
classes_concat = Concatenate(axis=1, name='classes_concat')([
classes4_reshaped, classes5_reshaped, classes6_reshaped,
classes7_reshaped
])
# Output shape of `boxes_concat`: (batch, n_boxes_total, 4)
boxes_concat = Concatenate(axis=1, name='boxes_concat')(
[boxes4_reshaped, boxes5_reshaped, boxes6_reshaped, boxes7_reshaped])
# Output shape of `anchors_concat`: (batch, n_boxes_total, 8)
anchors_concat = Concatenate(axis=1, name='anchors_concat')([
anchors4_reshaped, anchors5_reshaped, anchors6_reshaped,
anchors7_reshaped
])
# The box coordinate predictions will go into the loss function just the way they are,
# but for the class predictions, we'll apply a softmax activation layer first
classes_softmax = Activation('softmax',
name='classes_softmax')(classes_concat)
# Concatenate the class and box coordinate predictions and the anchors to one large predictions tensor
# Output shape of `predictions`: (batch, n_boxes_total, n_classes + 4 + 8)
predictions = Concatenate(axis=2, name='predictions')(
[classes_softmax, boxes_concat, anchors_concat])
if mode == 'training':
model = Model(inputs=inp, outputs=predictions)
elif mode == 'inference':
decoded_predictions = DecodeDetections(
confidence_thresh=confidence_thresh,
iou_threshold=iou_threshold,
top_k=top_k,
nms_max_output_size=nms_max_output_size,
coords=coords,
normalize_coords=normalize_coords,
img_height=img_height,
img_width=img_width,
name='decoded_predictions')(predictions)
model = Model(inputs=x, outputs=decoded_predictions)
elif mode == 'inference_fast':
decoded_predictions = DecodeDetectionsFast(
confidence_thresh=confidence_thresh,
iou_threshold=iou_threshold,
top_k=top_k,
nms_max_output_size=nms_max_output_size,
coords=coords,
normalize_coords=normalize_coords,
img_height=img_height,
img_width=img_width,
name='decoded_predictions')(predictions)
model = Model(inputs=inp, outputs=decoded_predictions)
else:
raise ValueError(
"`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'."
.format(mode))
if return_predictor_sizes:
# The spatial dimensions are the same for the `classes` and `boxes` predictor layers.
predictor_sizes = np.array([
classes4._keras_shape[1:3], classes5._keras_shape[1:3],
classes6._keras_shape[1:3], classes7._keras_shape[1:3]
])
return model, predictor_sizes
else:
return model
def get_model_5(params):
input_tensor=None
include_top=True
# Determine proper input shape
if K.image_dim_ordering() == 'th':
input_shape = (1, 96, 1366)
else:
input_shape = (96, 1366, 1)
if input_tensor is None:
melgram_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
melgram_input = Input(tensor=input_tensor, shape=input_shape)
else:
melgram_input = input_tensor
# Determine input axis
if K.image_dim_ordering() == 'th':
channel_axis = 1
freq_axis = 2
time_axis = 3
else:
channel_axis = 3
freq_axis = 1
time_axis = 2
# Input block
x = ZeroPadding2D(padding=(0, 37))(melgram_input)
x = BatchNormalization(axis=freq_axis, name='bn_0_freq')(x)
x = Permute((1, 3, 2))(x)
# Conv block 1
x = Convolution2D(64, 3, 3, border_mode='same', name='conv1')(x)
x = BatchNormalization(axis=channel_axis, mode=0, name='bn1')(x)
x = ELU()(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(x)
x = Dropout(0.1, name='dropout1')(x)
# Conv block 2
x = Convolution2D(128, 3, 3, border_mode='same', name='conv2')(x)
x = BatchNormalization(axis=channel_axis, mode=0, name='bn2')(x)
x = ELU()(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(3, 3), name='pool2')(x)
x = Dropout(0.1, name='dropout2')(x)
# Conv block 3
x = Convolution2D(128, 3, 3, border_mode='same', name='conv3')(x)
x = BatchNormalization(axis=channel_axis, mode=0, name='bn3')(x)
x = ELU()(x)
x = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool3')(x)
x = Dropout(0.1, name='dropout3')(x)
# Conv block 4
x = Convolution2D(128, 3, 3, border_mode='same', name='conv4')(x)
x = BatchNormalization(axis=channel_axis, mode=0, name='bn4')(x)
x = ELU()(x)
x = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool4')(x)
x = Dropout(0.1, name='dropout4')(x)
# reshaping
if K.image_dim_ordering() == 'th':
x = Permute((3, 2, 1))(x)
x = Reshape((15, 128))(x)
# GRU block 1, 2, output
x = GRU(32, return_sequences=True, name='gru1')(x)
x = GRU(32, return_sequences=False, name='gru2')(x)
x = Dropout(0.3)(x)
if include_top:
x = Dense(params["n_out"], activation=params['final_activation'], name='output')(x)
if params['final_activation'] == 'linear':
reg = Lambda(lambda x :K.l2_normalize(x, axis=1))
x = reg(x)
# Create model
model = Model(melgram_input, x)
return model
def do_model(train_i, test_i):
'''train a model, evaluate it, return model, history and results of evaluations'''
input_dim = 494 #num of features
elu_alpha = 1.0
seq = Sequential()
seq.add(
Dense(494,
input_shape=(input_dim, ),
kernel_regularizer=regularizers.l2(0.01)))
seq.add(Dropout(0.2))
seq.add(BatchNormalization())
seq.add(ELU(alpha=elu_alpha))
seq.add(Dense(490))
seq.add(Dropout(0.1))
seq.add(BatchNormalization())
seq.add(ELU(alpha=elu_alpha))
seq.add(Dense(200))
seq.add(Dropout(0.2))
seq.add(BatchNormalization())
seq.add(ELU(alpha=elu_alpha))
seq.add(Dense(25))
seq.add(Dropout(0.1))
seq.add(ELU(alpha=elu_alpha))
seq.add(Dense(1, activation='sigmoid'))
adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, amsgrad=False)
seq.compile(loss='binary_crossentropy',
optimizer=adam,
metrics=['accuracy'])
history = seq.fit(np.array(X_new)[train_i],
np.array(y_new)[train_i],
epochs=EPOCHS,
batch_size=BATCH_SIZE,
verbose=0,
validation_data=(np.array(X_new)[test_i],
np.array(y_new)[test_i]))
#get accuracy and loss on validation data. From history get the loss and acc on test and train data.
evaluation = seq.evaluate(np.array(X_val), np.array(y_val), verbose=0)
#From classification report get data.
y_pred = [np.round(i) for i in seq.predict(np.array(X_val))]
report = classification_report(y_val, y_pred)
report_dict = {'0': [], '1': [], 'macro': [], 'weighted': []}
for line in report.split('\n'):
items = line.split()
if len(items) > 0 and items[0] in report_dict.keys():
for i in items[1:]:
if i != 'avg':
report_dict[items[0]] += [float(i)]
#Get TP etc for average confusion matrix.
cm = confusion_matrix(y_true=y_val, y_pred=y_pred, labels=[1, 0])
TP = cm[0][0]
FN = cm[0][1]
FP = cm[1][0]
TN = cm[1][1]
#Get MCC
mcc = matthews_corrcoef(y_val, y_pred)
return evaluation[1], evaluation[
0], TP, FN, FP, TN, history, seq, report_dict, mcc
def _create_MobileNet_with_embedding_3_v2(self, num_of_classes,
input_shape):
mobilenet_model_face = mobilenet_v2.MobileNetV2(
input_shape=input_shape,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=None,
pooling=None)
mobilenet_model_face.layers.pop()
'''eyes'''
mobilenet_model_eyes = mobilenet_v2.MobileNetV2(
input_shape=input_shape,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=None,
pooling=None)
mobilenet_model_eyes.layers.pop()
'''nose'''
mobilenet_model_nose = mobilenet_v2.MobileNetV2(
input_shape=input_shape,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=None,
pooling=None)
mobilenet_model_nose.layers.pop()
'''mouth'''
mobilenet_model_mouth = mobilenet_v2.MobileNetV2(
input_shape=input_shape,
alpha=1.0,
include_top=True,
weights=None,
input_tensor=None,
pooling=None)
mobilenet_model_mouth.layers.pop()
for layer in mobilenet_model_face.layers:
layer._name = 'face_' + layer.name
for layer in mobilenet_model_eyes.layers:
layer._name = 'eyes_' + layer.name
for layer in mobilenet_model_nose.layers:
layer._name = 'nose_' + layer.name
for layer in mobilenet_model_mouth.layers:
layer._name = 'mouth_' + layer.name
# mobilenet_model_mouth.summary()
''''''
g_x_l_face = mobilenet_model_face.get_layer(
'face_global_average_pooling2d').output # 1280
g_x_l_eyes = mobilenet_model_eyes.get_layer(
'eyes_global_average_pooling2d_1').output # 1280
g_x_l_nose = mobilenet_model_nose.get_layer(
'nose_global_average_pooling2d_2').output # 1280
g_x_l_mouth = mobilenet_model_mouth.get_layer(
'mouth_global_average_pooling2d_3').output # 1280
embedding_layer_face = tf.keras.layers.Dense(
LearningConfig.embedding_size, activation='relu')(g_x_l_face)
embedding_layer_eyes = tf.keras.layers.Dense(
LearningConfig.embedding_size, activation='relu')(g_x_l_eyes)
embedding_layer_nose = tf.keras.layers.Dense(
LearningConfig.embedding_size, activation='relu')(g_x_l_nose)
embedding_layer_mouth = tf.keras.layers.Dense(
LearningConfig.embedding_size, activation='relu')(g_x_l_mouth)
'''concat'''
concat_globs = tf.keras.layers.Concatenate(axis=1)([
embedding_layer_face, embedding_layer_eyes, embedding_layer_nose,
embedding_layer_mouth
])
'''FC layer for out'''
x_l = Dense(LearningConfig.embedding_size)(concat_globs)
x_l = BatchNormalization()(x_l)
x_l = Dropout(rate=0.5)(x_l)
x_l = ELU()(x_l)
x_l = Dense(LearningConfig.embedding_size // 2)(x_l)
x_l = BatchNormalization()(x_l)
x_l = Dropout(rate=0.5)(x_l)
x_l = ELU()(x_l)
x_l = Dense(LearningConfig.embedding_size // 4)(x_l)
x_l = BatchNormalization()(x_l)
x_l = Dropout(rate=0.5)(x_l)
x_l = ELU()(x_l)
x_l = Dense(LearningConfig.embedding_size // 8)(x_l)
x_l = BatchNormalization()(x_l)
x_l = Dropout(rate=0.5)(x_l)
x_l = ELU()(x_l)
x_l = Dense(LearningConfig.embedding_size // 16)(x_l)
x_l = BatchNormalization()(x_l)
x_l = Dropout(rate=0.5)(x_l)
x_l = ELU()(x_l)
out_categorical = Dense(num_of_classes,
activation='softmax',
name='out')(x_l)
inp = [
mobilenet_model_face.input, mobilenet_model_eyes.input,
mobilenet_model_nose.input, mobilenet_model_mouth.input
]
revised_model = Model(inp, [
out_categorical, embedding_layer_face, embedding_layer_eyes,
embedding_layer_nose, embedding_layer_mouth
])
revised_model.summary()
'''save json'''
model_json = revised_model.to_json()
with open("./model_archs/mn_v2_cat_emb_3_v2.json", "w") as json_file:
json_file.write(model_json)
return revised_model
main_input = Input(shape=(maxlen, ), dtype='float64', name='main_input')
emb = Embedding(256,
embed_size,
input_length=maxlen,
embeddings_regularizer=l2(1e-4))(main_input)
#emb = Dropout(0.2)(emb)
x = Flatten()(emb)
num_layers = 4
for i in range(num_layers):
W_reg = l2(1e-4)
if i == 0:
W_reg = l1_l2(1e-4)
#W_reg = l2(1e-4)
x = Dense(50, activation='linear', kernel_regularizer=W_reg)(x)
x = BatchNormalization()(x)
x = ELU()(x)
# if i == num_layers-1:
# x = DeCovRegularization(0.1)(x)
x = Dropout(0.5)(x)
loss_out = Dense(1, activation='sigmoid', name='loss_out')(x)
# Replicates `model` on 8 GPUs.
# This assumes that your machine has 8 available GPUs.
#parallel_model = multi_gpu_model(model, gpus=8)
model = Model(inputs=[main_input], outputs=[loss_out])
optimizer = Adam(lr=0.001, decay=0.5)
model.compile(optimizer,
loss='binary_crossentropy',
metrics=['binary_accuracy'])
def model(input_shape,input_shape2,n_y):
#This input stores real channel matrix normalized to values between -1 and 1, this is given to the DNN as the input.
X_input = Input(input_shape)
#This input stores each rank-1 approximation of channel matrix seperately.
X_input2 = Input(input_shape2)
X2 = Flatten()(X_input2)
#This input stores real channel matrix, it is used for estimating rate.
X_input3 = Input(input_shape)
X3 = Flatten()(X_input3)
X = ZeroPadding2D((7, 7))(X_input)
#Currently 2 convolutional layers are active.
X = Conv2D(32, (3, 3), strides = (1, 1), name = 'conv0')(X)
X = ELU(alpha=1.0)(X)
X = Conv2D(64, (3, 3), strides = (1, 1), name = 'conv1')(X)
X = ELU(alpha=1.0)(X)
# X = Conv2D(64, (3, 3), strides = (1, 1), name = 'conv2')(X)
# X = ELU(alpha=1.0)(X)
# X = Conv2D(128, (3, 3), strides = (1, 1), name = 'conv3')(X)
# X = ELU(alpha=1.0)(X)
# X = Conv2D(128, (3, 3), strides = (1, 1), name = 'conv4')(X)
# X = ELU(alpha=1.0)(X)
# X = Conv2D(128, (3, 3), strides = (1, 1), name = 'conv5')(X)
# X = ELU(alpha=1.0)(X)
# MAXPOOL
X = MaxPooling2D((2, 2), name='max_pool')(X)
X = Dropout(0.2)(X)
X = Flatten()(X)
#Unconstrained estimated RF precoder (T_RF), baseband precoder (T_BB), RF combiner (R_RF), baseband combiner (R_BB)
T_RF = Dense(n_y-2*Num_Stream)(X)
T_RF = ELU(alpha=1.0)(T_RF)
T_RF = Dropout(0.1)(T_RF)
T_RF = Dense((n_y-2*Num_Stream)/4, activation='sigmoid')(T_RF)
T_BB = Dense(n_y-2*Num_Stream)(X)
T_BB = ELU(alpha=1.0)(T_BB)
T_BB = Dropout(0.1)(T_BB)
T_BB = Dense((n_y-2*Num_Stream)/2, activation='linear')(T_BB)
R_RF = Dense(n_y-2*Num_Stream)(X)
R_RF = ELU(alpha=1.0)(R_RF)
R_RF = Dropout(0.1)(R_RF)
R_RF = Dense((n_y-2*Num_Stream)/4, activation='sigmoid')(R_RF)
R_BB = Dense(n_y-2*Num_Stream)(X)
R_BB = ELU(alpha=1.0)(R_BB)
R_BB = Dropout(0.1)(R_BB)
R_BB = Dense((n_y-2*Num_Stream)/2, activation='linear')(R_BB)
X_temp = Dense(2*Num_Stream, activation='linear')(X)
X_in=concatenate([X_temp, T_RF, T_BB, R_RF, R_BB],axis=1)
# myFunc consists of quantization and normalization layers, generate concatenated constrained T_RF, T_BB, R_RF, R_BB.
X_out=Lambda(myFunc)(X_in)
output_temp_hybrid=concatenate([X_out, X2, X3],axis=1)
model = Model(inputs = [X_input, X_input2,X_input3], outputs = output_temp_hybrid, name='model')
return model
