def autoencoder_hyperopt(_n_features, _hparams=default_autoencoder_hyperopt):
"""
Creazione struttura autoencoder Keras parametrica.
:param _n_features: int
numero input rete neurale. Configurazione tensore principale
:param _hparams: dict
parametri per la configurazione della rete
:return: Model
autoencoder creato. Keras model
"""
input_layer = Input(shape=(_n_features, ))
for layer in range(1, int(_hparams['num_layers']) + 1):
hidden = Dense(
units=int(_hparams['num_unit_' + str(layer)]),
activation=_hparams['actv_func'],
activity_regularizer=regularizers.l1(
_hparams['l1_reg']))(input_layer if layer == 1 else hidden)
if _hparams['drop_enabled']:
hidden = Dropout(rate=_hparams['drop_factor'])(hidden)
for layer in reversed(range(1, int(_hparams['num_layers']) + 1)):
hidden = Dense(units=int(_hparams['num_unit_' + str(layer)]),
activation=_hparams['actv_func'],
activity_regularizer=regularizers.l1(
_hparams['l1_reg']))(hidden)
if _hparams['drop_enabled']:
hidden = Dropout(rate=_hparams['drop_factor'])(hidden)
output_layer = Dense(_n_features,
activation=_hparams['actv_func_out'])(hidden)
autoencoder = Model(input_layer, output_layer)
if _hparams['optimizer'] == 'adadelta':
opt_net = Adadelta(_hparams['learn_rate_opt'], rho=0.95)
elif _hparams['optimizer'] == 'adam':
opt_net = Adam(_hparams['learn_rate_opt'],
beta_1=0.9,
beta_2=0.999,
amsgrad=False)
else:
opt_net = Adam(_hparams['learn_rate_opt'],
beta_1=0.9,
beta_2=0.999,
amsgrad=False)
autoencoder.compile(optimizer=opt_net, loss=_hparams['loss_func'])
return autoencoder
def block(self,
inputs,
filters,
kernel_size,
strides,
dilation_rate=(1, 1)):
inputs = tf.keras.layers.Conv2D(
filters=filters,
dilation_rate=dilation_rate,
kernel_size=kernel_size,
strides=strides,
kernel_regularizer=l1(0.1),
kernel_initializer=self.utils.msra_initializer(
kernel_size, filters),
padding='SAME',
)(inputs)
inputs = tf.layers.BatchNormalization(
fused=True,
renorm_clipping={
'rmax': 3,
'rmin': 0.3333,
'dmax': 5
},
epsilon=1.001e-5,
)(inputs, training=self.utils.training)
inputs = tf.keras.layers.LeakyReLU(0.01)(inputs)
return inputs
def __init__(self,
hidden_state_size,
is_on=True,
vocab_size=11,
utterance_len=6,
entropy_reg=0.001):
"""
lambda is an entropy regularization term
"""
self.is_on = is_on
self.utterance_len = utterance_len
self.vocab = list(range(vocab_size))
self.vocab_size = vocab_size
if self.is_on:
inputs = Input(batch_shape=(1, 1, 1), name='utter_input')
lstm1 = LSTM(
100,
stateful=True,
name='utter_lstm',
activity_regularizer=regularizers.l1(entropy_reg))(inputs)
dense = Dense(vocab_size, activation='softmax',
name='utter_dense')(lstm1)
model = Model(inputs=inputs, outputs=[dense])
model.compile(
optimizer='adam',
loss='categorical_crossentropy'
# TODO might be cool to use the one below (requires different shape in training)
# loss='sparse_categorical_crossentropy',
# metrics=['categorical_accuracy']
# sample_weight_mode="temporal"
)
self.model = model
def build(input_size, channels, latent_dim):
layer_units = [512, 256]
input_shape = (input_size, channels)
drop_rate = 0.8
inputs = Input(shape=input_shape)
x = inputs
x = Dropout(0.4, input_shape=(None, 978, 1))(x)
for f in layer_units:
x = Dense(f)(x)
x = LeakyReLU(alpha=0.2)(x)
x = Dropout(drop_rate, input_shape=(None, input_size, layer_units[1]))(x)
shape = K.int_shape(x)
x = Flatten()(x)
latent = Dense(latent_dim, kernel_regularizer=regularizers.l2(1e-5),
activity_regularizer=regularizers.l1(1e-5))(x)
#, kernel_regularizer=regularizers.l2(1e-5),
# activity_regularizer=regularizers.l1(1e-5)
encoder = Model(inputs, latent, name="encoder")
latent_inputs = Input(shape=(latent_dim,))
x = Dense(shape[1] * shape[2])(latent_inputs)
x = Reshape((shape[1], shape[2]))(x)
for f in layer_units[::-1]:
x = Dense(f)(x)
x = LeakyReLU(alpha=0.2)(x)
x = Dropout(drop_rate, input_shape=(None, input_size, layer_units[0]))(x)
x = Dense(1)(x)
outputs = Activation("tanh")(x)
decoder = Model(latent_inputs, outputs, name="decoder")
autoencoder = Model(inputs, decoder(encoder(inputs)), name="autoencoder")
return autoencoder
def __init__(self, hidden_state_size, entropy_reg=0.05):
# single feedforward layer with sigmoid function
self.model = Sequential([
Dense(
1,
input_shape=(hidden_state_size, ),
# kernel_initializer='random_uniform', # TODO or maybe random_normal
kernel_initializer=
'random_normal', # TODO or maybe random_normal
activity_regularizer=regularizers.l1(entropy_reg)),
Activation('sigmoid')
])
# Accuracy is not the right measure for your model's performance. What you are trying to do here is more of a
# regression task than a classification task. The same can be seen from your loss function, you are using
# 'mean_squared_error' rather than something like 'categorical_crossentropy'.
optimizer = optimizers.Adam()
# lr=learning_rate 0.001 by default which is fine
self.model.compile(
optimizer=optimizer,
loss=
'binary_crossentropy' # TODO these are random, needs to be checked
# metrics=['accuracy']
)
def cnn_layer(self, index, inputs, filters, kernel_size, strides):
"""卷积-BN-激活函数-池化结构生成器"""
# for i in range(len(kernel_size)):
with tf.keras.backend.name_scope('unit-{}'.format(index + 1)):
x = tf.keras.layers.Conv2D(
filters=filters,
kernel_size=kernel_size,
strides=strides[0],
kernel_regularizer=l1(0.01),
kernel_initializer=self.msra_initializer(kernel_size, filters),
padding='same',
name='cnn-{}'.format(index + 1),
)(inputs)
x = tf.layers.BatchNormalization(fused=True,
renorm_clipping={
'rmax': 3,
'rmin': 0.3333,
'dmax': 5
} if index == 0 else None,
epsilon=1.001e-5,
name='bn{}'.format(index + 1))(
x, training=self.training)
x = tf.keras.layers.LeakyReLU(0.01)(x)
x = tf.keras.layers.MaxPooling2D(pool_size=(2, 2),
strides=strides[1])(x)
return x
def build_and_compile_nn(num_of_features, layer_size=100, dropout_rate=0.2, verbose=0):
# Build the model
model = keras.models.Sequential([
keras.layers.Dense(layer_size, input_shape=[num_of_features], activation='relu',
kernel_regularizer=regularizers.l1(0.001)),
keras.layers.Dropout(dropout_rate),
keras.layers.Dense(int(layer_size / 2), activation='relu'),
keras.layers.Dropout(dropout_rate),
keras.layers.Dense(1),
])
# keras.layers.Dense(3, activation='relu'),
# keras.layers.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis = 1))
# z = L.Input((nh,), name="Patient")
# x = L.Dense(100, activation="relu", name="d1")(z)
# x = L.Dense(100, activation="relu", name="d2")(x)
# p1 = L.Dense(3, activation="linear", name="p1")(x)
# p2 = L.Dense(3, activation="relu", name="p2")(x)
# preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1),
# name="preds")([p1, p2])
if verbose > 0:
print(model.summary())
# Construct loss function
# loss_fn = calc_tensor_score
# loss_fn = mloss(0.8)
loss_fn = 'mae'
# compile loss function into model
model.compile(optimizer=keras.optimizers.Adam(lr=nn_config.learning_rate, beta_1=0.9, beta_2=0.999, epsilon=None,
decay=nn_config.learning_rate / 10, amsgrad=False),
loss=loss_fn,
metrics=['mae'])
return model
def cnn_layer(self, index, inputs, filters, kernel_size, strides):
"""卷积-BN-激活函数-池化结构块"""
with tf.keras.backend.name_scope('unit-{}'.format(index + 1)):
x = tf.keras.layers.Conv2D(
filters=filters,
kernel_size=kernel_size,
strides=strides[0],
kernel_regularizer=l1(0.01),
kernel_initializer=self.msra_initializer(kernel_size, filters),
padding='same',
name='cnn-{}'.format(index + 1),
)(inputs)
x = tf.layers.batch_normalization(x,
fused=True,
renorm_clipping={
'rmax': 3,
'rmin': 0.3333,
'dmax': 5
} if index == 0 else None,
reuse=False,
momentum=0.9,
name='bn{}'.format(index + 1),
training=self.is_training)
x = tf.keras.layers.LeakyReLU(0.01)(x)
x = tf.keras.layers.MaxPooling2D(
pool_size=(2, 2),
strides=strides[1],
padding='same',
)(x)
return x
def autoencoder_dense(data, smell, layers=1, encoding_dimension=32, epochs=10, with_bottleneck=True, is_final=False, threshold=400000):
encoding_dim = encoding_dimension
input_layer = Input(shape=(data.max_input_length,))
no_of_layers = layers
prev_layer = input_layer
for i in range(no_of_layers):
encoder = Dense(int(encoding_dim / pow(2, i)), activation="relu",
activity_regularizer=regularizers.l1(10e-3))(prev_layer)
prev_layer = encoder
# bottleneck
if with_bottleneck:
prev_layer = Dense(int(encoding_dim / pow(2, no_of_layers)), activation="relu")(prev_layer)
for j in range(no_of_layers - 1, -1, -1):
decoder = Dense(int(encoding_dim / pow(2, j)), activation='relu')(prev_layer)
prev_layer = decoder
prev_layer = Dense(data.max_input_length, activation='relu')(prev_layer)
autoencoder = Model(inputs=input_layer, outputs=prev_layer)
autoencoder.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
autoencoder.summary()
batch_sizes = [32, 64, 128]
# batch_sizes = [32, 64, 128, 256, 512]
b_size = int(len(data.train_data) / batch_sizes[len(batch_sizes) - 1])
if b_size > len(batch_sizes) - 1:
b_size = len(batch_sizes) - 1
val_split = 0.2
if is_final:
val_split = 0
history = autoencoder.fit(data.train_data,
data.train_data,
epochs=epochs,
# batch_size=batch_size,
batch_size=batch_sizes[b_size],
verbose=1,
validation_split=val_split,
shuffle=True).history
# plt.plot(history['loss'])
# plt.plot(history['val_loss'])
# plt.title('model loss')
# plt.ylabel('loss')
# plt.xlabel('epoch')
# plt.legend(['train', 'test'], loc='upper right')
# plt.show()
predictions = autoencoder.predict(data.eval_data)
mse = np.mean(np.power(data.eval_data - predictions, 2), axis=1)
error_df = pd.DataFrame({'Reconstruction_error': mse,
'True_class': data.eval_labels})
# print(error_df.describe())
if is_final:
return find_metrics(error_df, threshold)
else:
return find_optimal(error_df)
def _build_model(self, input_size):
input_data = Input(shape=(input_size,))
x = input_data
x = Dense(2048, activation='relu')(x)
x = Dropout(0.75)(x)
x = Dense(128, activation='relu')(x)
x = Dense(NB_CAT, activation='sigmoid', kernel_regularizer=l1(1e-5))(x)
model = Model(inputs=input_data, outputs=x)
return model
def build_model_xception_avg(lock_base_model: bool):
base_model = Xception(input_shape=INPUT_SHAPE, include_top=False, pooling=None, weights=None)
if lock_base_model:
for layer in base_model.layers:
layer.trainable = False
x = GlobalAveragePooling2D(name='avg_pool_final')(base_model.layers[-1].output)
res = Dense(NB_CLASSES, activation='sigmoid', name='classes', kernel_initializer='zero',
kernel_regularizer=l1(1e-5))(x)
model = Model(inputs=base_model.inputs, outputs=res)
return model
def build_model_nasnet_mobile(lock_base_model: True):
img_input = Input(shape=INPUT_SHAPE)
base_model = NASNetMobile(input_tensor=img_input, include_top=False, pooling='avg')
if lock_base_model:
for layer in base_model.layers:
layer.trainable = False
res = Dense(NB_CLASSES, activation='sigmoid', name='classes', kernel_initializer='zero',
kernel_regularizer=l1(1e-5))(base_model.layers[-1].output)
model = Model(inputs=img_input, outputs=res)
return model
def modify_model(model: Model, class_index: int,
importance_type: ImportanceType) -> Model:
gamma_initializer: str = "zeros"
if importance_type & ImportanceType.GAMMA:
gamma_initializer = "ones"
gamma_regularizer = None
if importance_type & ImportanceType.L1 and not importance_type & ImportanceType.L2:
gamma_regularizer = l1()
if not importance_type & ImportanceType.L1 and importance_type & ImportanceType.L2:
gamma_regularizer = l2()
if importance_type & ImportanceType.L1 and importance_type & ImportanceType.L2:
gamma_regularizer = l1_l2()
max_layer: int = len(model.layers)
last_output: Input = None
network_input: Input = None
for i, layer in enumerate(model.layers):
if i == 0:
last_output = layer.output
network_input = layer.input
if 0 < i < max_layer:
new_layer: Union[BatchNormalization,
BatchNormalization] = BatchNormalization(
center=(importance_type
& ImportanceType.CENTERING),
gamma_initializer=gamma_initializer,
gamma_regularizer=gamma_regularizer)
last_output = new_layer(last_output)
if i == max_layer - 1:
new_end_layer: Dense = Dense(2,
activation="softmax",
name="binary_output_layer")
last_output = new_end_layer(last_output)
old_weights = layer.get_weights()
old_weights[0] = np.transpose(old_weights[0], (1, 0))
new_weights: List[np.array] = [
np.append(old_weights[0][class_index:class_index + 1],
np.subtract(
np.sum(old_weights[0], axis=0, keepdims=True),
old_weights[0][class_index:class_index + 1]),
axis=0),
np.append(old_weights[1][class_index:class_index + 1],
np.subtract(
np.sum(old_weights[1], axis=0, keepdims=True),
old_weights[1][class_index:class_index + 1]),
axis=0)
]
new_weights[0] = np.transpose(new_weights[0], (1, 0))
new_end_layer.set_weights(new_weights)
elif i > 0:
last_output = layer(last_output)
return Model(inputs=network_input, outputs=last_output)
def build(self):
input_seq = Input(shape=self.input_shape)
x = Conv3D(filters=32,
kernel_size=(self.filter_dims, 1, 1),
padding='same',
use_bias=False,
kernel_regularizer=l1(1e-3))(input_seq)
x = BatchNormalization(-1)(x)
x = AveragePooling3D(pool_size=(self.filter_dims, 1, 1),
strides=(int(self.filter_dims / 3), 1, 1),
padding='valid')(x)
x = ConvLSTM2D(filters=64,
kernel_size=(3, 3),
return_sequences=False,
padding='same')(x)
x = Conv2D(filters=128,
kernel_size=(3, 3),
activation='relu',
kernel_regularizer=l2(1e-3))(x)
x = BatchNormalization(-1)(x)
x = Dropout(0.5)(x)
x = Conv2D(filters=256,
kernel_size=(3, 3),
activation='relu',
kernel_regularizer=l2(1e-3))(x)
x = BatchNormalization(-1)(x)
x = Dropout(0.5)(x)
x = Conv2D(filters=512,
kernel_size=(3, 3),
activation='relu',
kernel_regularizer=l2(1e-3))(x)
x = BatchNormalization(-1)(x)
x = Dropout(0.5)(x)
x = Conv2D(filters=1024,
kernel_size=(5, 5),
activation='relu',
kernel_regularizer=l2(1e-3))(x)
x = BatchNormalization(-1)(x)
x = Dropout(0.5)(x)
x = Flatten()(x)
logits = Dense(self.output_classes,
activation='softmax',
kernel_regularizer=l2(1e-3))(x)
model = Model(inputs=input_seq, outputs=logits)
model.summary()
return model
def build_model_resnet50(lock_base_model: bool):
base_model = ResNet50(input_shape=INPUT_SHAPE, include_top=False, pooling=None)
if lock_base_model:
for layer in base_model.layers:
layer.trainable = False
x = AveragePooling2D((5, 5), name='avg_pool5', strides=1)(base_model.layers[-2].output)
x = GlobalMaxPooling2D()(x)
res = Dense(NB_CLASSES, activation='sigmoid', name='classes', kernel_initializer='zero',
kernel_regularizer=l1(1e-5))(x)
model = Model(inputs=base_model.inputs, outputs=res)
return model
def test_kernel_reg(case):
x, y, model_fn, backbone = case
l1_reg = regularizers.l1(0.1)
model = model_fn(backbone)
reg_model = set_regularization(model, kernel_regularizer=l1_reg)
_test_regularizer(model, reg_model, x, y)
l2_reg = regularizers.l2(0.1)
model = model_fn(backbone, encoder_weights=None)
reg_model = set_regularization(model, kernel_regularizer=l2_reg)
_test_regularizer(model, reg_model, x, y)
def build(input_size, latent_dim, regul_stren=0):
if regul_stren == 0:
noise_dropout = 0.1
l1_weight = 0
dropout = 0.5
elif regul_stren == 1:
noise_dropout = 0.2
l1_weight = 1e-8
dropout = 0.8
elif regul_stren == 2:
noise_dropout = 0.5
l1_weight = 1e-7
dropout = 0.8
else:
noise_dropout = 0.5
l1_weight = 1e-4
dropout = 0.9
layer_units = [512, 256]
input_shape = (input_size, 1)
inputs = Input(shape=input_shape)
x = inputs
xd = Dropout(noise_dropout, input_shape=(None, 978, 1))(x)
x = xd
for f in layer_units:
x = Dense(f)(x)
x = LeakyReLU(alpha=0.2)(x)
shape = K.int_shape(x)
x = Flatten()(x)
latent = Dense(latent_dim,
use_bias=False,
activity_regularizer=regularizers.l1(l1_weight))(x)
encoder = Model(inputs, latent, name="encoder")
latent_inputs = Input(shape=(latent_dim, ))
xd_input = Input(shape=input_shape)
x = Dense(shape[1] * shape[2])(latent_inputs)
x = Reshape((shape[1], shape[2]))(x)
for f in layer_units[::-1]:
x = Dense(f)(x)
x = LeakyReLU(alpha=0.2)(x)
x = Dropout(dropout, input_shape=(None, input_size, layer_units[0]))(x)
z = tf.keras.layers.Concatenate(axis=-1)([x, xd_input])
x = Dense(1)(z)
outputs = Activation("tanh")(x)
decoder = Model([xd_input, latent_inputs], outputs, name="decoder")
autoencoder = Model(inputs,
decoder([xd, encoder(inputs)]),
name="autoencoder")
return autoencoder
def create_network():
# Weights initializer (to initialize weights from a normal distribution)
initializer = keras.initializers.glorot_normal(seed=None)
# Building the model
model = keras.Sequential([
# 2 conv layers with 100 filters and kernel dimension 10
keras.layers.Conv1D(100, 10, activation='relu', input_shape=(139, 3)),
keras.layers.Conv1D(100, 10, activation='relu'),
# Max pooling, dividing the input map, generated by the conv layers, by 3
keras.layers.MaxPooling1D(3),
# Same concept applied again, but with global average pooling for dimensionality reduction
keras.layers.Conv1D(160, 10, activation='relu'),
keras.layers.Conv1D(160, 10, activation='relu'),
keras.layers.GlobalAveragePooling1D(),
# Dropout layer, giving each weight a 50% probability to become 0, avoiding over-fitting
keras.layers.Dropout(0.5),
# 2 hidden layers (with 128 and 64 nodes respectively) followed by the output layer
keras.layers.Dense(128,
activation=tf.nn.relu,
kernel_initializer=initializer,
kernel_regularizer=regularizers.l1(0.01),
activity_regularizer=regularizers.l2(0.01)),
keras.layers.Dense(64,
activation=tf.nn.relu,
kernel_initializer=initializer,
kernel_regularizer=regularizers.l1(0.01),
activity_regularizer=regularizers.l2(0.01)),
keras.layers.Dense(20, activation=tf.nn.softmax)
])
# Model compiling with adam optimizer (which gives an adaptive learning rate)
opt = optimizers.Adam(lr=0.001)
model.compile(optimizer=opt,
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
return model
def CNN_Video(nb_channels=3, dropoutRate = 0.5, act='relu', k_size=3, d_layer = 512,
k_regularizer = regularizers.l1(0.001), img_size=32,time_slot = 100,num_color_chan=1):
"""
Deep convolutional 3D neural network with softmax classifier
:param nb_channels: number of class
:param dropoutRate: drop-out rate of last layer
:param act: activation function
:param k_size: convolutional kernel size
:param k_regularizer: kernel regularizer
:param d_layer: number of hidden unit in the last layer
:param img_size: image size
:param time_slot: number of frames/images in a video, length of the video
:param num_color_chan = number of color channel in the image/frame, no RGB values used real values of electrodes are used
:param input_dimension: size of the input
Expecting 100x32x32x1 video data as input
Conv3D<32> - Conv3D<32> - Conv3D<32> - Conv3D<32> - MaxPool3D<2,2,2> -
Conv3D<64> - Conv3D<64> - MaxPool3D<2,2,2> -
Dense<512> - Dense<3>
"""
strides = None
# In each convolutional layer, 10 consecutive images are convolved
kernel = (10, k_size, k_size)
print('PARAMETERS OF MODELS: ', act, ' ', k_size, ' ', d_layer)
model = Sequential()
# add layers
model.add(Conv3D(32, kernel_size=kernel, input_shape=(time_slot,img_size,img_size,num_color_chan), activation=act))
model.add(Conv3D(32, kernel_size=kernel, padding='same', kernel_initializer='glorot_uniform', activation=act ))
model.add(Conv3D(32, kernel_size=kernel, padding='same', kernel_initializer='glorot_uniform', activation=act ))
model.add(Conv3D(32, kernel_size=kernel, padding='same', kernel_initializer='glorot_uniform', activation=act ))
model.add(MaxPooling3D(pool_size=kernel, strides=strides, data_format='channels_last'))
# new layer
model.add(Conv3D(64, kernel_size=kernel, padding='same', kernel_initializer='glorot_uniform', activation=act))
model.add(Conv3D(64, kernel_size=kernel, padding='same', kernel_initializer='glorot_uniform', activation=act))
model.add(MaxPooling3D(pool_size=(2,2,2),strides=strides, data_format='channels_last'))
# flatten and check
model.add(Flatten())
model.add(Dense(d_layer))
model.add(Dropout(rate=dropoutRate))
model.add(Dense(nb_channels, activation='softmax'))
return model
def build_nn(hidden_layer_sizes, input_dim, l1_reg, activation_func='relu'): input_layer = Input(shape=(input_dim,)) x = input_layer x = Dropout(0.3, input_shape=(input_dim,))(x) # Fully connected hidden layers for i, size in enumerate(hidden_layer_sizes): x = Dense(size, activation = activation_func, kernel_regularizer=regularizers.l1(l1_reg))(x) x = Dropout(0.1)(x) x = BatchNormalization()(x) # Wrap base_nn into a model base_nn = Model(inputs = input_layer, outputs = x, name = "base_nn") return(base_nn, x)
def __init__(
self,
input_dim,
first_layer_size=100,
latent_dim=3,
encoder_layer_num=4,
drop_out=0.05,
gaussian_noise=0.1
):
super(Autoencoder, self).__init__()
self.latent_dim = latent_dim
self.first_layer_size = first_layer_size
self.encoder_layer_num = encoder_layer_num
self.input_dim = input_dim
self.layer_sizes = [
int(i) for i in np.linspace(first_layer_size, latent_dim, encoder_layer_num)
]
encoder_layers = []
for size in self.layer_sizes[:-1]:
encoder_layers.append(
Dense(size, activation='relu', activity_regularizer=regularizers.l1(1e-7))
)
encoder_layers.append(Dropout(drop_out))
encoder_layers.append(GaussianNoise(gaussian_noise))
encoder_layers.append(
Dense(latent_dim, activation='relu', activity_regularizer=regularizers.l1(1e-7))
)
decoder_layers = []
for size in self.layer_sizes[::-1][1:]:
decoder_layers.append(Dense(size, activation='relu'))
decoder_layers.append(Dense(input_dim, activation='sigmoid'))
self.encoder = tf.keras.Sequential(encoder_layers)
self.decoder = tf.keras.Sequential(decoder_layers)
def build_model_resnet50_avg(lock_base_model: bool):
"""
Build the Resnet50 based level 1 model
:param lock_base_model:
:return:
"""
base_model = ResNet50(input_shape=INPUT_SHAPE, include_top=False, pooling=None)
if lock_base_model:
for layer in base_model.layers:
layer.trainable = False
x = GlobalAveragePooling2D(name='avg_pool_final')(base_model.layers[-2].output)
res = Dense(NB_CLASSES, activation='sigmoid', name='classes', kernel_initializer='zero',
kernel_regularizer=l1(1e-5))(x)
model = Model(inputs=base_model.inputs, outputs=res)
return model
def KerasModelSmallNet50(self, imgInput):
"""
Construct small net. The image size is 50*50, which is suitable for map.
"""
x = Conv2D(16, (3, 3), activation='tanh')(imgInput)
x = Conv2D(32, (3, 3), activation='relu')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Conv2D(32, (3, 3), activation='relu')(x)
x = Conv2D(32, (3, 3), activation='relu')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Conv2D(32, (3, 3), activation='relu')(x)
x = Conv2D(32, (3, 3), activation='relu')(x)
x = Flatten()(x)
x = Dense(self.featureDim, kernel_regularizer=regularizers.l2(0.0002),
activity_regularizer=regularizers.l1(0.0002), name='fc_feature')(x)
x = PReLU()(x)
return x
def block(self, inputs, filters, kernel_size, strides, dilation_rate=(1, 1)):
inputs = tf.keras.layers.Conv2D(
filters=filters,
dilation_rate=dilation_rate,
kernel_size=kernel_size,
strides=strides,
kernel_regularizer=l1(0.1),
kernel_initializer=self.utils.msra_initializer(kernel_size, filters),
padding='SAME',
)(inputs)
inputs = tf.layers.batch_normalization(
inputs,
reuse=False,
momentum=0.9,
training=self.utils.is_training
)
inputs = self.utils.hard_swish(inputs)
return inputs
def KerasModelResNet(self, imgInput):
"""
Construct resNet. The image size is 150*150, which is suitable for image.
"""
bn_axis = 3
x = ZeroPadding2D((3, 3))(imgInput)
x = Convolution2D(8, 7, strides=(2, 2), name='conv1')(x)
x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = Activation('relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv_block(x, 3, [8, 8, 16], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [8, 8, 16], stage=2, block='b')
x = identity_block(x, 3, [8, 8, 16], stage=2, block='c')
x = conv_block(x, 3, [16, 16, 32], stage=3, block='a')
x = identity_block(x, 3, [16, 16, 32], stage=3, block='b')
x = identity_block(x, 3, [16, 16, 32], stage=3, block='c')
x = identity_block(x, 3, [16, 16, 32], stage=3, block='d')
x = conv_block(x, 3, [32, 32, 64], stage=4, block='a')
x = identity_block(x, 3, [32, 32, 64], stage=4, block='b')
x = identity_block(x, 3, [32, 32, 64], stage=4, block='c')
x = identity_block(x, 3, [32, 32, 64], stage=4, block='d')
x = identity_block(x, 3, [32, 32, 64], stage=4, block='e')
x = identity_block(x, 3, [32, 32, 64], stage=4, block='f')
x = conv_block(x, 3, [64, 64, 128], stage=5, block='a')
x = identity_block(x, 3, [64, 64, 128], stage=5, block='b')
x = identity_block(x, 3, [64, 64, 128], stage=5, block='c')
x = conv_block(x, 3, [64, 64, 256], stage=6, block='a')
x = identity_block(x, 3, [64, 64, 256], stage=6, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=6, block='c')
x = GlobalAveragePooling2D()(x)
# x = Flatten()(x)
x = Dense(self.featureDim,
kernel_regularizer=regularizers.l2(0.0002),
activity_regularizer=regularizers.l1(0.0002),
name='fc_feature')(x)
x = PReLU()(x)
return x
def build_model_inception_v3_dropout(lock_base_model: True):
base_model = InceptionV3(input_shape=INPUT_SHAPE,
include_top=False,
pooling=None)
if lock_base_model:
for layer in base_model.layers:
layer.trainable = False
# base_model.summary()
x = GlobalAveragePooling2D(name='avg_pool_final')(
base_model.layers[-1].output)
x = Dropout(0.25)(x)
res = Dense(NB_CLASSES,
activation='sigmoid',
name='classes',
kernel_initializer='zero',
kernel_regularizer=l1(1e-5))(x)
model = Model(inputs=base_model.inputs, outputs=res)
# model.summary()
return model
def __init__(self):
self.batch_size = 15
self.learning_rate = 0.0001
self.epochs = 6000
self.time_stamps = 1524
self.num_steps = 64
self.dropout = 0.5
self.regularizer_type = 'l2'
if self.regularizer_type == 'l2':
self.regularizer = l2(1)
elif self.regularizer_type == 'l1':
self.regularizer = l1(1)
elif self.regularizer == 'l1_l2':
self.regularizer = l1_l2(1)
self.model_type = ''
self.data_type = 'all'
def test_bn_reg(case):
x, y, model_fn, backbone = case
l1_reg = regularizers.l1(1)
model = model_fn(backbone)
reg_model = set_regularization(model, gamma_regularizer=l1_reg)
_test_regularizer(model, reg_model, x, y)
model = model_fn(backbone)
reg_model = set_regularization(model, beta_regularizer=l1_reg)
_test_regularizer(model, reg_model, x, y)
l2_reg = regularizers.l2(1)
model = model_fn(backbone)
reg_model = set_regularization(model, gamma_regularizer=l2_reg)
_test_regularizer(model, reg_model, x, y)
model = model_fn(backbone)
reg_model = set_regularization(model, beta_regularizer=l2_reg)
_test_regularizer(model, reg_model, x, y)
def __init__(self,
is_on,
hidden_state_size=100,
item_num=3,
entropy_reg=0.05):
self.is_on = is_on
self.item_num = item_num
if self.is_on:
self.models = []
for _ in range(self.item_num):
model = Sequential([
Dense(100, input_shape=(hidden_state_size, )),
Dense(6,
activity_regularizer=regularizers.l1(entropy_reg)),
Activation('softmax')
])
model.compile(
optimizer='adam',
loss=
'categorical_crossentropy' # TODO these are random, needs to be checked
# metrics=['accuracy']
)
self.models.append(model)
def build_model(self):
self.model = tf.keras.Sequential()
self.model.add(
layers.Dense(128,
input_shape=(64, ),
activation='tanh',
kernel_regularizer=regularizers.l2(L2_RATIO),
activity_regularizer=regularizers.l1(L2_RATIO)))
self.model.add(
layers.Dense(128,
activation='tanh',
kernel_regularizer=regularizers.l2(L2_RATIO),
activity_regularizer=regularizers.l1(L2_RATIO)))
self.model.add(
layers.Dense(128,
activation='tanh',
kernel_regularizer=regularizers.l2(L2_RATIO),
activity_regularizer=regularizers.l1(L2_RATIO)))
self.model.add(
layers.Dense(128,
activation='tanh',
kernel_regularizer=regularizers.l2(L2_RATIO),
activity_regularizer=regularizers.l1(L2_RATIO)))
self.model.add(
layers.Dense(128,
activation='tanh',
kernel_regularizer=regularizers.l2(L2_RATIO),
activity_regularizer=regularizers.l1(L2_RATIO)))
self.model.add(
layers.Dense(3,
activation='softmax',
kernel_regularizer=regularizers.l2(L2_RATIO),
activity_regularizer=regularizers.l1(L2_RATIO)))
self.model.compile(
optimizer=tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE),
loss='categorical_crossentropy',
metrics=['accuracy'])
return