def encoder_model(model):
new_input = model.input
new_output = model.layers[-2].output
img_encoder = Model(new_input, new_output)
return img_encoder
def get_model2(input_shape):
in_x = Input(input_shape)
X = Conv2D(32,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
activation='relu')(in_x)
X = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X)
X = Conv2D(32,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
activation='relu')(X)
X = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X)
X = Conv2D(32,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
activation='relu')(X)
X = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X)
X = Conv2D(32,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
activation='relu')(X)
X = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X)
####################################################################
X1 = Conv2D(32,
kernel_size=(5, 5),
strides=(1, 1),
padding='same',
activation='relu')(in_x)
X1 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X1)
X1 = Conv2D(32,
kernel_size=(5, 5),
strides=(1, 1),
padding='same',
activation='relu')(X1)
X1 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X1)
X1 = Conv2D(32,
kernel_size=(5, 5),
strides=(1, 1),
padding='same',
activation='relu')(X1)
X1 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X1)
X1 = Conv2D(32,
kernel_size=(5, 5),
strides=(1, 1),
padding='same',
activation='relu')(X1)
X1 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X1)
# ####################################################################
X2 = Conv2D(32,
kernel_size=(2, 2),
strides=(1, 1),
padding='same',
activation='relu')(in_x)
X2 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X2)
X2 = Conv2D(32,
kernel_size=(2, 2),
strides=(1, 1),
padding='same',
activation='relu')(X2)
X2 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X2)
X2 = Conv2D(32,
kernel_size=(2, 2),
strides=(1, 1),
padding='same',
activation='relu')(X2)
X2 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X2)
X2 = Conv2D(32,
kernel_size=(2, 2),
strides=(1, 1),
padding='same',
activation='relu')(X2)
X2 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X2)
# ####################################################################
X3 = Conv2D(32,
kernel_size=(4, 4),
strides=(1, 1),
padding='same',
activation='relu')(in_x)
X3 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X3)
X3 = Conv2D(32,
kernel_size=(4, 4),
strides=(1, 1),
padding='same',
activation='relu')(X3)
X3 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X3)
X3 = Conv2D(32,
kernel_size=(4, 4),
strides=(1, 1),
padding='same',
activation='relu')(X3)
X3 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X3)
X3 = Conv2D(32,
kernel_size=(4, 4),
strides=(1, 1),
padding='same',
activation='relu')(X3)
X3 = AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(X3)
X = concatenate([X, X1, X3], axis=3)
print(X)
X = Flatten()(X)
X = Dense(96, activation='relu')(X)
X = Dropout(0.4)(X)
X = Dense(64, activation='relu')(X)
X = Dropout(0.4)(X)
X = Dense(32, activation='relu')(X)
X = Dropout(0.4)(X)
X = Dense(17, activation='softmax')(X)
model = Model(inputs=in_x, outputs=X)
return model
l = 100
t = [(i - 1) / (l - 1) for i in range(1, l + 1)]
bx = normalize([2 * math.pi * i - math.sin(2 * math.pi * i) for i in t])
by = normalize([1 - math.cos(2 * math.pi * i) for i in t])
train = [[bx[i], by[i]] for i in range(len(bx))]
# COMPILATION
k = list()
for n in [9, 10, 11]:
for i in range(5):
inp = Input(shape=(1, ))
n_units = n
out1 = first_layer(units=n_units)(inp)
out2 = second_layer(units=n_units - 2)(out1)
model = Model(inputs=inp, outputs=out2)
opt = tf.keras.optimizers.Adamax(
learning_rate=0.0001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-07,
name='Adamax',
)
model.compile(loss='mae', optimizer=opt)
# TRAINING
history = model.fit(t, train, batch_size=1, epochs=4000, verbose=0)
k.append(history.history['loss'][-1])
# SAVING THE MODEL
# In[5]:
from tensorflow.keras.optimizers import RMSprop
# Flatten the output layer to 1 dimension
x = layers.Flatten()(last_output)
# Add a fully connected layer with 1,024 hidden units and ReLU activation
x = layers.Dense(1024, activation='relu')(x)
# Add a dropout rate of 0.2
x = layers.Dropout(.2)(x)
# Add a final sigmoid layer for classification
x = layers.Dense(1, activation='sigmoid')(x)
model = Model(pre_trained_model.input, x)
model.compile(optimizer=RMSprop(lr=0.0001),
loss='binary_crossentropy',
metrics=['accuracy'])
model.summary()
# Expected output will be large. Last few lines should be:
# mixed7 (Concatenate) (None, 7, 7, 768) 0 activation_248[0][0]
# activation_251[0][0]
# activation_256[0][0]
# activation_257[0][0]
# __________________________________________________________________________________________________
# flatten_4 (Flatten) (None, 37632) 0 mixed7[0][0]
def U_Net_3D(image_shape,
activation='elu',
feature_maps=[32, 64, 128, 256],
depth=3,
drop_values=[0.1, 0.1, 0.1, 0.1],
spatial_dropout=False,
batch_norm=False,
k_init='he_normal',
z_down=2,
loss_type="bce",
optimizer="sgd",
lr=0.001,
n_classes=1):
"""Create 3D U-Net.
Parameters
----------
image_shape : 3D tuple
Dimensions of the input image.
activation : str, optional
Keras available activation type.
feature_maps : array of ints, optional
Feature maps to use on each level. Must have the same length as the
``depth+1``.
depth : int, optional
Depth of the network.
drop_values : float, optional
Dropout value to be fixed.
spatial_dropout : bool, optional
Use spatial dropout instead of the `normal` dropout.
batch_norm : bool, optional
Make batch normalization.
k_init : string, optional
Kernel initialization for convolutional layers.
z_down : int, optional
Downsampling used in z dimension. Set it to 1 if the dataset is not
isotropic.
loss_type : str, optional
Loss type to use, three type available: ``bce`` (Binary Cross Entropy)
, ``w_bce`` (Weighted BCE, based on weigth maps) and ``w_bce_dice``
(Weighted loss: ``weight1*BCE + weight2*Dice``).
optimizer : str, optional
Optimizer used to minimize the loss function. Posible options: ``sgd``
or ``adam``.
lr : float, optional
Learning rate value.
n_classes: int, optional
Number of classes.
Returns
-------
model : Keras model
Model containing the U-Net.
Calling this function with its default parameters returns the following
network:
.. image:: img/unet_3d.png
:width: 100%
:align: center
Image created with `PlotNeuralNet <https://github.com/HarisIqbal88/PlotNeuralNet>`_.
"""
if len(feature_maps) != depth + 1:
raise ValueError("feature_maps dimension must be equal depth+1")
if len(drop_values) != depth + 1:
raise ValueError("'drop_values' dimension must be equal depth+1")
dinamic_dim = (None, ) * (len(image_shape) - 1) + (image_shape[-1], )
inputs = Input(dinamic_dim)
x = inputs
#x = Input(image_shape)
#inputs = x
if loss_type == "w_bce":
weights = Input(image_shape)
# List used to access layers easily to make the skip connections of the U-Net
l = []
# ENCODER
for i in range(depth):
x = Conv3D(feature_maps[i], (3, 3, 3),
activation=None,
kernel_initializer=k_init,
padding='same')(x)
x = BatchNormalization()(x) if batch_norm else x
x = Activation(activation)(x)
if spatial_dropout and drop_values[i] > 0:
x = SpatialDropout3D(drop_values[i])(x)
elif drop_values[i] > 0 and not spatial_dropout:
x = Dropout(drop_values[i])(x)
x = Conv3D(feature_maps[i], (3, 3, 3),
activation=None,
kernel_initializer=k_init,
padding='same')(x)
x = BatchNormalization()(x) if batch_norm else x
x = Activation(activation)(x)
l.append(x)
x = MaxPooling3D((2, 2, z_down))(x)
# BOTTLENECK
x = Conv3D(feature_maps[depth], (3, 3, 3),
activation=None,
kernel_initializer=k_init,
padding='same')(x)
x = BatchNormalization()(x) if batch_norm else x
x = Activation(activation)(x)
if spatial_dropout and drop_values[depth] > 0:
x = SpatialDropout3D(drop_values[depth])(x)
elif drop_values[depth] > 0 and not spatial_dropout:
x = Dropout(drop_values[depth])(x)
x = Conv3D(feature_maps[depth], (3, 3, 3),
activation=None,
kernel_initializer=k_init,
padding='same')(x)
x = BatchNormalization()(x) if batch_norm else x
x = Activation(activation)(x)
# DECODER
for i in range(depth - 1, -1, -1):
x = Conv3DTranspose(feature_maps[i], (2, 2, 2),
strides=(2, 2, z_down),
padding='same')(x)
x = concatenate([x, l[i]])
x = Conv3D(feature_maps[i], (3, 3, 3),
activation=None,
kernel_initializer=k_init,
padding='same')(x)
x = BatchNormalization()(x) if batch_norm else x
x = Activation(activation)(x)
if spatial_dropout and drop_values[i] > 0:
x = SpatialDropout3D(drop_values[i])(x)
elif drop_values[i] > 0 and not spatial_dropout:
x = Dropout(drop_values[i])(x)
x = Conv3D(feature_maps[i], (3, 3, 3),
activation=None,
kernel_initializer=k_init,
padding='same')(x)
x = BatchNormalization()(x) if batch_norm else x
x = Activation(activation)(x)
outputs = Conv3D(n_classes, (1, 1, 1), activation='sigmoid')(x)
# Loss type
if loss_type == "w_bce":
model = Model(inputs=[inputs, weights], outputs=[outputs])
else:
model = Model(inputs=[inputs], outputs=[outputs])
# Select the optimizer
if optimizer == "sgd":
opt = tf.keras.optimizers.SGD(lr=lr,
momentum=0.99,
decay=0.0,
nesterov=False)
elif optimizer == "adam":
opt = tf.keras.optimizers.Adam(lr=lr,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
else:
raise ValueError("Error: optimizer value must be 'sgd' or 'adam'")
# Compile the model
if loss_type == "bce":
if n_classes > 1:
model.compile(optimizer=opt,
loss='categorical_crossentropy',
metrics=[jaccard_index_softmax])
else:
model.compile(optimizer=opt,
loss='binary_crossentropy',
metrics=[jaccard_index])
elif loss_type == "w_bce":
model.compile(optimizer=opt,
loss=binary_crossentropy_weighted(weights),
metrics=[jaccard_index])
elif loss_type == "w_bce_dice":
model.compile(optimizer=opt,
loss=weighted_bce_dice_loss(w_dice=0.66, w_bce=0.33),
metrics=[jaccard_index])
else:
raise ValueError("'loss_type' must be 'bce', 'w_bce' or 'w_bce_dice'")
return model
x = bottleneck_block(64, x)
# Double the size of filters and reduce feature maps by 75% (strides=2, 2) to fit the next Residual Group
x = conv_block(128, x)
# Second Residual Block Group of 128 filters
for _ in range(3):
x = bottleneck_block(128, x)
# Double the size of filters and reduce feature maps by 75% (strides=2, 2) to fit the next Residual Group
x = conv_block(256, x)
# Third Residual Block Group of 256 filters
for _ in range(5):
x = bottleneck_block(256, x)
# Double the size of filters and reduce feature maps by 75% (strides=2, 2) to fit the next Residual Group
x = conv_block(512, x)
# Fourth Residual Block Group of 512 filters
for _ in range(2):
x = bottleneck_block(512, x)
# Now Pool at the end of all the convolutional residual blocks
x = layers.GlobalAveragePooling2D()(x)
# Final Dense Outputting Layer for 1000 outputs
outputs = layers.Dense(1000, activation='softmax')(x)
model = Model(inputs, outputs)
x = layers.BatchNormalization(axis=chanDim)(x)
x = layers.Conv2D(filters=250,
kernel_size=3,
strides=1,
activation="relu",
padding="same")(x)
x = layers.MaxPool2D(2)(x)
x = layers.BatchNormalization(axis=chanDim)(x)
# flatten the network and then construct our latent vector
volumeSize = K.int_shape(x)
x = layers.Flatten()(x)
latent = layers.Dense(latentDim)(x)
# build the encoder model
encoder = Model(input, latent, name="encoder")
#decoder
latentInputs = layers.Input(shape=(latentDim))
x = layers.Dense(np.prod(volumeSize[1:]))(latentInputs)
x = layers.Reshape((volumeSize[1], volumeSize[2], volumeSize[3]))(x)
# apply a CONV_TRANSPOSE => RELU => BN operation
x = layers.Conv2DTranspose(filters=250,
kernel_size=3,
strides=1,
activation="relu",
padding="same")(x)
x = layers.UpSampling2D(2)(x)
x = layers.BatchNormalization(axis=chanDim)(x)
def evaluate_nnca(l1_train,
l1_test,
l2_train,
l2_test,
dimensions,
evaluation_function=reciprocal_rank,
neurons=[100],
activation_function="relu",
max_epochs=100,
dropout=0.2,
optimizer="adam",
loss="MSE"):
callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5)
scores = []
for idz, dimension in enumerate(dimensions):
if len(neurons) == 1:
x = tf.keras.layers.Input(shape=(dimension, ))
d1 = tf.keras.layers.Dropout(dropout)(x)
emb = tf.keras.layers.Dense(neurons[0],
activation=activation_function)(d1)
d2 = tf.keras.layers.Dropout(dropout)(emb)
y = tf.keras.layers.Dense(dimension,
activation=activation_function)(d2)
model = Model(x, y)
elif len(neurons) == 3:
x = tf.keras.layers.Input(shape=(dimension, ))
d1 = tf.keras.layers.Dropout(dropout)(x)
h1 = tf.keras.layers.Dense(neurons[0],
activation=activation_function)(d1)
emb = tf.keras.layers.Dense(neurons[1],
activation=activation_function)(h1)
h2 = tf.keras.layers.Dense(neurons[2],
activation=activation_function)(emb)
d2 = tf.keras.layers.Dropout(dropout)(h2)
y = tf.keras.layers.Dense(dimension, activation=None)(d2)
model = Model(x, y)
if loss == "cosine_sim":
loss = tf.keras.losses.CosineSimilarity()
l1_to_l2_clf = Model(x, y)
l2_to_l1_clf = Model(x, y)
l1_to_l2_clf.compile(optimizer=optimizer,
loss=loss,
metrics=tf.keras.losses.CosineSimilarity())
l2_to_l1_clf.compile(optimizer=optimizer,
loss=loss,
metrics=tf.keras.losses.CosineSimilarity())
hist1 = l1_to_l2_clf.fit(l1_train[:, :dimension],
l2_train[:, :dimension],
epochs=max_epochs,
validation_data=(l1_test[:, :dimension],
l2_test[:, :dimension]),
callbacks=[callback])
hist2 = l2_to_l1_clf.fit(l2_train[:, :dimension],
l1_train[:, :dimension],
epochs=max_epochs,
validation_data=(l2_test[:, :dimension],
l1_test[:, :dimension]),
callbacks=[callback])
fake_fr = l1_to_l2_clf.predict(l1_test[:, :dimension])
fake_en = l2_to_l1_clf.predict(l2_test[:, :dimension])
merged_trans_vecs = np.concatenate((fake_en, l2_test[:, :dimension]),
axis=1)
real_vecs = np.concatenate((l1_test[:, :dimension], fake_fr), axis=1)
score = evaluation_function(merged_trans_vecs, real_vecs)
scores.append(score)
return scores, hist1, hist2
def evaluate_nncc(l1_train,
l1_test,
l2_train,
l2_test,
dimensions,
evaluation_function=reciprocal_rank,
neurons=[100],
activation_function="relu",
max_epochs=100,
dropout=0.2,
optimizer="adam",
loss="MSE"):
callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5)
l1_train, l1_test, l2_train, l2_test = np.asarray(l1_train), np.asarray(
l1_test), np.asarray(l2_train), np.asarray(l2_test)
scores = []
for idz, dimension in enumerate(dimensions):
en = l1_train[:, :dimension] - np.mean(l1_train[:, :dimension], axis=0)
fr = l2_train[:, :dimension] - np.mean(l2_train[:, :dimension], axis=0)
zero_matrix = np.zeros((en.shape[0], dimension))
X1 = np.concatenate((en, zero_matrix), axis=1)
X2 = np.concatenate((zero_matrix, fr), axis=1)
X = np.concatenate((X1, X2), axis=0)
Y1 = np.concatenate((en, fr), axis=1)
Y2 = np.concatenate((en, fr), axis=1)
Y = np.concatenate((Y1, Y2), axis=0)
if len(neurons) == 1:
x = tf.keras.layers.Input(shape=(dimension * 2, ))
d1 = tf.keras.layers.Dropout(dropout)(x)
emb = tf.keras.layers.Dense(neurons[0],
activation=activation_function)(d1)
d2 = tf.keras.layers.Dropout(dropout)(emb)
y = tf.keras.layers.Dense(dimension * 2,
activation=activation_function)(d2)
model = Model(x, y)
elif len(neurons) == 3:
x = tf.keras.layers.Input(shape=(dimension * 2, ))
d1 = tf.keras.layers.Dropout(dropout)(x)
h1 = tf.keras.layers.Dense(neurons[0],
activation=activation_function)(d1)
emb = tf.keras.layers.Dense(neurons[1],
activation=activation_function)(h1)
h2 = tf.keras.layers.Dense(neurons[2],
activation=activation_function)(emb)
d2 = tf.keras.layers.Dropout(dropout)(h2)
y = tf.keras.layers.Dense(dimension * 2, activation=None)(d2)
model = Model(x, y)
if loss == "cosine_sim":
loss = tf.keras.losses.CosineSimilarity()
model.compile(
optimizer=optimizer,
loss=loss, #"MSE",#tf.keras.losses.CosineSimilarity(),
metrics=[tf.keras.losses.CosineSimilarity()])
history = model.fit(X,
Y,
epochs=max_epochs,
validation_split=0.1,
callbacks=[callback])
en = l1_test[:, :dimension] - np.mean(l1_train[:, :dimension], axis=0)
fr = l2_test[:, :dimension] - np.mean(l1_train[:, :dimension], axis=0)
zero_matrix = np.zeros((l1_test.shape[0], dimension))
X1 = np.concatenate((en, zero_matrix), axis=1)
X2 = np.concatenate((zero_matrix, fr), axis=1)
X = np.concatenate((X1, X2), axis=0)
encoder = Model(x, emb)
english_encodings_nncc = encoder.predict(X1)
french_encodings_nncc = encoder.predict(X2)
score = evaluation_function(english_encodings_nncc,
french_encodings_nncc)
scores.append(score)
return scores, history
def MultiOutputCNN(
input_shape,
output_shape: Union[List, Dict],
cnns_per_maxpool=4,
maxpool_layers=4, # increasing `maxpool_layers` prefers fewer `cnns_per_maxpool` (ideal total CNNs = 15 / 16)
cnn_units=32,
cnn_kernel=3,
cnn_strides=1,
dense_layers=1, # `dense_layers=1` is preferred over `2` or `3`
dense_units=256,
activation='relu', # 'relu' | 'crelu' | 'leaky_relu' | 'relu6' | 'softmax' | 'tanh' | 'hard_sigmoid' | 'sigmoid'
dropout=0.25,
regularization=False, # `regularization=True` prefers `global_maxpool=False` and fewer dense_units - but worse results
global_maxpool=False, # `global_maxpool=True` prefers double the number of `dense_units` and +1 `cnns_per_maxpool`
name='',
) -> Model:
function_name = cast(types.FrameType,
inspect.currentframe()).f_code.co_name
model_name = f"{function_name}-{name}" if name else function_name
# model_name = seq([ function_name, name ]).filter(lambda x: x).make_string("-") # remove dependency on pyfunctional - not in Kaggle repo without internet
inputs = Input(shape=input_shape)
x = inputs
for cnn1 in range(1, maxpool_layers + 1):
for cnn2 in range(1, cnns_per_maxpool + 1):
x = Conv2D(cnn_units * cnn1,
kernel_size=cnn_kernel,
strides=cnn_strides,
padding='same',
activation=activation)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = BatchNormalization()(x)
x = Dropout(dropout)(x)
if global_maxpool:
x = GlobalMaxPooling2D()(x)
x = Flatten()(x)
for nn1 in range(0, dense_layers):
if regularization:
x = Dense(dense_units,
activation=activation,
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(x)
else:
x = Dense(dense_units, activation=activation)(x)
x = BatchNormalization()(x)
x = Dropout(dropout)(x)
x = Flatten(name='output')(x)
if isinstance(output_shape, dict):
outputs = [
Dense(output_shape, activation='softmax', name=key)(x)
for key, output_shape in output_shape.items()
]
else:
outputs = [
Dense(output_shape, activation='softmax',
name=f'output_{index}')(x)
for index, output_shape in enumerate(output_shape)
]
model = Model(inputs, outputs, name=model_name)
# plot_model(model, to_file=os.path.join(os.path.dirname(__file__), f"{name}.png"))
return model
(X_train, y_train), (X_test, y_test) = datasets.cifar10.load_data()
Y_train = to_categorical(y_train, num_classes)
Y_test = to_categorical(y_test, num_classes)
base_model = VGG16(weights='imagenet',
include_top=False,
input_shape=(32, 32, 3))
# Extract the last layer from third block of vgg16 model
last = base_model.get_layer('block3_pool').output
# Add classification layers on top of it
x = Flatten()(last)
x = Dense(256, activation='relu')(x)
x = Dropout(0.5)(x)
pred = Dense(10, activation='sigmoid')(x)
model = Model(base_model.input, pred)
# set the base model's layers to non-trainable
# uncomment next two lines if you don't want to
# train the base model
# for layer in base_model.layers:
# layer.trainable = False
# compile the model with a SGD/momentum optimizer
# and a very slow learning rate.
model.compile(loss='binary_crossentropy',
optimizer=tf.optimizers.SGD(lr=args.learning_rate, momentum=0.9),
metrics=['accuracy'])
# prepare data augmentation configuration
train_datagen = ImageDataGenerator(rescale=1. / 255,
def U_Net(
input_size=(256, 256, 1), #Correspond à la taille des images utilisées
initial=64 #Le nombre de features maps utilisé au départ
):
"""
Crée un réseau de type U-net pour la segmentation
Parameters
----------
- input_size : la taille des images à utiliser et le nombre de channels couleur (1 pour les DICOM)
- initial : nombre de feature map à utiliser au déaprt (allourdit le réseau)
Returns
-------
- model : le réseau prêt à l'entrainement
Notes
-----
Ne pas hésiter à modifier les hyperparamètres :
- le learning rate
- l'optimizer
"""
inputs = Input(input_size)
conv1 = Conv2D(initial,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(inputs)
conv1 = Conv2D(initial,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(initial * 2,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool1)
conv2 = Conv2D(initial * 2,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(initial * 4,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool2)
conv3 = Conv2D(initial * 4,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = Conv2D(initial * 8,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool3)
conv4 = Conv2D(initial * 8,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv4)
drop4 = Dropout(0.5)(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2))(drop4)
conv5 = Conv2D(initial * 16,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool4)
conv5 = Conv2D(initial * 16,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv5)
drop5 = Dropout(0.5)(conv5)
up6 = Conv2D(initial * 8,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(drop5))
merge6 = concatenate([drop4, up6], axis=3)
conv6 = Conv2D(initial * 8,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge6)
conv6 = Conv2D(initial * 8,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv6)
up7 = Conv2D(initial * 4,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(conv6))
merge7 = concatenate([conv3, up7], axis=3)
conv7 = Conv2D(initial * 4,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge7)
conv7 = Conv2D(initial * 4,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv7)
up8 = Conv2D(initial * 2,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(conv7))
merge8 = concatenate([conv2, up8], axis=3)
conv8 = Conv2D(initial * 2,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge8)
conv8 = Conv2D(initial * 2,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv8)
up9 = Conv2D(initial,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(conv8))
merge9 = concatenate([conv1, up9], axis=3)
conv9 = Conv2D(initial,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge9)
conv9 = Conv2D(initial,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv9)
conv9 = Conv2D(2,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv9)
conv10 = Conv2D(1, 1, activation='sigmoid')(conv9)
model = Model(inputs=inputs, outputs=conv10)
model.compile(optimizer=Adam(lr=1e-4),
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def YOLOv3Net(cfgfile, model_size, num_classes):
blocks = parse_cfg(cfgfile)
outputs = {}
output_filters = []
filters = []
out_pred = []
scale = 0
inputs = input_image = Input(shape=model_size)
inputs = inputs / 255.0
for i, block in enumerate(blocks[1:]):
if block["type"] == "convolutional":
activation = block["activation"]
filters = int(block["filters"])
kernel_size = int(block["size"])
strides = int(block["stride"])
if strides > 1:
#top and left padding with zeros
inputs = ZeroPadding2D(((1, 0), (1, 0)))(inputs)
inputs = Conv2D(filters,
kernel_size,
strides=strides,
padding='valid' if strides > 1 else 'same',
name='conv_' + str(i),
use_bias=False if
("batch_normalize" in block) else True)(inputs)
if "batch_normalize" in block:
inputs = BatchNormalization(name="bnorm_" + str(i))(inputs)
inputs = LeakyReLU(alpha=0.1, name='leaky_' + str(i))(inputs)
elif block["type"] == "upsample":
stride = int(block["stride"])
inputs = UpSampling2D(stride)(inputs)
elif block["type"] == "route":
block["layers"] = block["layers"].split(',')
start = int(block["layers"][0])
if len(block["layers"]) > 1:
end = int(block["layers"][1]) - i
filters = output_filters[i + start] + output_filters[end]
inputs = tf.concat([outputs[i + start], outputs[i + end]],
axis=-1)
else:
filters = output_filters[i + start]
inputs = outputs[i + start]
elif block["type"] == "shortcut":
from_ = int(block["from"])
inputs = outputs[i - 1] + outputs[i + from_]
elif block["type"] == "yolo":
mask = block["mask"].split(",")
mask = [int(x) for x in mask]
anchors = block["anchors"].split(",")
anchors = [int(a) for a in anchors]
anchors = [(anchors[i], anchors[i + 1])
for i in range(0, len(anchors), 2)]
anchors = [anchors[i] for i in mask]
n_anchors = len(anchors)
out_shape = inputs.get_shape().as_list()
inputs = tf.reshape(
inputs,
[-1, n_anchors * out_shape[1] * out_shape[2], 5 + num_classes])
box_centers = inputs[:, :, 0:2]
box_shapes = inputs[:, :, 2:4]
confidence = inputs[:, :, 4:5]
classes = inputs[:, :, 5:num_classes + 5]
box_centers = tf.sigmoid(box_centers)
confidence = tf.sigmoid(confidence)
classes = tf.sigmoid(classes)
anchors = tf.tile(anchors, [out_shape[1] * out_shape[2], 1])
box_shapes = tf.exp(box_shapes) * tf.cast(anchors,
dtype=tf.float32)
x = tf.range(out_shape[1], dtype=tf.float32)
y = tf.range(out_shape[2], dtype=tf.float32)
cx, cy = tf.meshgrid(x, y)
cx = tf.reshape(cx, (-1, 1))
cy = tf.reshape(cy, (-1, 1))
cxy = tf.concat([cx, cy], axis=-1)
cxy = tf.tile(cxy, [1, n_anchors])
cxy = tf.reshape(cxy, [1, -1, 2])
strides = (input_image.shape[1] // out_shape[1],
input_image.shape[2] // out_shape[2])
box_centers = (box_centers + cxy) * strides
prediction = tf.concat(
[box_centers, box_shapes, confidence, classes], axis=-1)
if scale:
out_pred = tf.concat([out_pred, prediction], axis=1)
else:
out_pred = prediction
scale = 1
outputs[i] = inputs
output_filters.append(filters)
model = Model(input_image, out_pred)
model.summary()
return model
def get_adm_model(learning_rate, l2_reg, price_min, price_max, price_interval,
embedd_size):
price_step = int(
math.floor((price_max - price_min + K.epsilon()) / price_interval))
total_len = 0
input_tensors = []
# Input Embedding Layers
embedding_tensors = []
# composing embedding layers
for column_name, count_value, dimension_length in embedding_paras:
total_len += embedd_size # or dimension_length
input_tensor = Input(name='{}_index'.format(column_name),
shape=(1, ),
dtype='int64')
embedding_tensor = Embedding(
count_value,
embedd_size, # or dimension_length
input_length=1,
embeddings_initializer='glorot_normal',
name='{}_embedding'.format(column_name))(input_tensor)
embedding_tensor = Flatten()(embedding_tensor)
embedding_tensors.append(embedding_tensor)
input_tensors.append(input_tensor)
# Input Numerical Layers
numerical_tensors = []
numerical_embedd_tensors = []
for column_name in numerical_paras:
total_len += 1
input_tensor = Input(name='{}'.format(column_name),
shape=(1, ),
dtype='float32')
numerical_tensors.append(input_tensor)
input_tensors.append(input_tensor)
input_embedd_tensor = Lambda(lambda x: tf.tile(x, [1, embedd_size]))(
input_tensor)
numerical_embedd_tensors.append(input_embedd_tensor)
feature_num = len(input_tensors) # 44
# features (embedding + numeric)
## 1-order
x_o1_tensor = Concatenate()(embedding_tensors + numerical_tensors)
## 2-order
x_o2_tensor = Lambda(lambda x: tf.stack(x, axis=1))(
embedding_tensors +
numerical_embedd_tensors) # (None, feature_num, dim)
x_o2_tensor = InteractingLayer(att_embedding_size=8,
head_num=2)(x_o2_tensor)
x_o2_tensor = Flatten()(x_o2_tensor)
## high-order
x_oh_tensor = Dense(
total_len / 2,
activation='relu',
kernel_regularizer=regularizers.l2(l2_reg))(x_o1_tensor)
x_oh_tensor = Dense(
total_len / 4,
activation='relu',
kernel_regularizer=regularizers.l2(l2_reg))(x_oh_tensor)
# output layer
output_tensor = Concatenate(axis=1)(
[x_o1_tensor, x_o2_tensor, x_oh_tensor])
output_tensor = Dense(
price_step, kernel_regularizer=regularizers.l2(l2_reg))(output_tensor)
output_tensor = Softmax(name='mixture_price_3')(output_tensor)
model = Model(inputs=input_tensors, outputs=[output_tensor])
adam = optimizers.Adam(lr=learning_rate)
optimizer = hvd.DistributedOptimizer(adam)
model.compile(loss=neighborhood_likelihood_loss,
optimizer=optimizer,
experimental_run_tf_function=False)
return model
def mobile_net_v2(input_shape=(224,224,3), num_classes=1000):
"""
mobile net v2 based on https://arxiv.org/pdf/1801.04381.pdf
Args:
input_shape (tuple): input shape
num_classes (int): number of categories
Returns: mobile net v2 model
"""
input = layers.Input(shape=input_shape)
x = layers.Conv2D(
32,
3,
2,
padding='same',
use_bias=False
)(input)
x = layers.BatchNormalization()(x)
x = layers.ReLU(6.0)(x)
x = conv_block(x, 16, 1, 1, expand=False)
x = conv_block(x, 24, 2, 6)
x = conv_block(x, 24, 1, 6)
x = conv_block(x, 32, 2, 6)
x = conv_block(x, 32, 1, 6)
x = conv_block(x, 32, 1, 6)
x = conv_block(x, 64, 2, 6)
x = conv_block(x, 64, 1, 6)
x = conv_block(x, 64, 1, 6)
x = conv_block(x, 64, 1, 6)
x = conv_block(x, 96, 1, 6)
x = conv_block(x, 96, 1, 6)
x = conv_block(x, 96, 1, 6)
x = conv_block(x, 160, 2, 6)
x = conv_block(x, 160, 1, 6)
x = conv_block(x, 160, 1, 6)
x = conv_block(x, 320, 1, 6)
x = layers.Conv2D(
1280,
1,
1,
padding='same',
use_bias=False
)(x)
x = layers.BatchNormalization()(x)
x = layers.ReLU(6.0)(x)
x = layers.GlobalAveragePooling2D()(x)
x = layers.Dense(num_classes)(x)
x = layers.Activation('softmax')(x)
model = Model(inputs=input, outputs=x)
model.summary()
return model
filters=8,
kernel_size=(
3,
3),
padding='same',
activation='relu',
use_bias=False,
connectivity_level=built_in_sparsity.pop(0) or None)(layer)
layer = Flatten()(layer)
layer = Dropout(0.01)(layer)
layer = Sparse(units=10,
activation='softmax',
use_bias=False,
connectivity_level=built_in_sparsity.pop(0) or None)(layer)
model = Model(input_layer, layer)
model.summary()
model.compile(
NoisySGD(
lr=0.01),
'categorical_crossentropy',
['accuracy'])
# Train model with backprop.
model.fit(x_train, y_train, batch_size=64, epochs=5, verbose=2,
validation_data=(x_test, y_test),
callbacks=callback_list)
# Store model so SNN Toolbox can find it.
x_train = scaler.transform(x_train)
x_test = scaler.transform(x_test)
print(np.max(x), np.min(x))
print(np.max(x[0]))
#2. model
from tensorflow.keras import Model
from tensorflow.keras.layers import Input, Dense
input1 = Input(shape=(13, ))
dense1 = Dense(128, activation='relu')(input1)
dense1 = Dense(64, activation='relu')(dense1)
dense1 = Dense(64, activation='relu')(dense1)
dense1 = Dense(64, activation='relu')(dense1)
output1 = Dense(1)(dense1)
model = Model(inputs=input1, outputs=output1)
#3. compile and fit
model.compile(optimizer='adam', loss='mse', metrics=['mae'])
model.fit(x_train,
y_train,
batch_size=4,
epochs=150,
verbose=1,
validation_split=0.2)
#4. evalutate and predict
mse, mae = model.evaluate(x_test, y_test, batch_size=4)
print("mse :", mse, "\nmae :", mae)
y_predict = model.predict(x_test)
def hifis_rnn_mlp(cfg,
input_dim,
metrics,
metadata,
output_bias=None,
hparams=None):
'''
Defines a Keras model for HIFIS hybrid recurrent neural network and multilayer perceptron model (i.e. HIFIS-v3)
:param cfg: A dictionary of parameters associated with the model architecture
:param input_dim: The shape of the model input
:param metrics: Metrics to track model's performance
:param metadata: Dict containing prediction horizon, time series feature info
:param output_bias: initial bias applied to output layer
:param hparams: dict of hyperparameters
:return: a Keras model object with the architecture defined in this method
'''
# Set hyperparameters
if hparams is None:
nodes_dense0 = cfg['DENSE']['DENSE0']
nodes_dense1 = cfg['DENSE']['DENSE1']
layers = cfg['LAYERS']
dropout = cfg['DROPOUT']
l2_lambda = cfg['L2_LAMBDA']
lr = cfg['LR']
optimizer = Adam(learning_rate=lr)
lstm_units = cfg['LSTM']['UNITS']
else:
nodes_dense0 = hparams['NODES0']
nodes_dense1 = hparams['NODES1']
layers = hparams['LAYERS']
dropout = hparams['DROPOUT']
lr = 10**hparams['LR'] # Random sampling on logarithmic scale
beta_1 = 1 - 10**hparams['BETA_1']
beta_2 = 1 - 10**hparams['BETA_2']
l2_lambda = 10**hparams['L2_LAMBDA']
lstm_units = hparams['LSTM_UNITS']
if hparams['OPTIMIZER'] == 'adam':
optimizer = Adam(learning_rate=lr, beta_1=beta_1, beta_2=beta_2)
elif hparams['OPTIMIZER'] == 'sgd':
optimizer = SGD(learning_rate=lr)
if output_bias is not None:
output_bias = Constant(output_bias)
# Receive input to model and split into 2 tensors containing dynamic and static features respectively
X_input = Input(shape=input_dim)
split_idx = metadata['NUM_TS_FEATS'] * metadata['T_X']
X_dynamic, X_static = split(X_input,
[split_idx, X_input.shape[1] - split_idx],
axis=1)
# Define RNN component of model using LSTM cells. LSTM input shape is [batch_size, timesteps, features]
lstm_input_shape = (metadata['T_X'], metadata['NUM_TS_FEATS'])
X_dynamic = Reshape(lstm_input_shape)(X_dynamic)
X_dynamic = LSTM(lstm_units, activation='tanh',
return_sequences=True)(X_dynamic)
X_dynamic = Flatten()(X_dynamic)
# Define MLP component of model
X = concatenate([X_dynamic,
X_static]) # Combine output of LSTM with static features
X = Dense(nodes_dense0,
input_shape=input_dim,
activation='relu',
kernel_regularizer=l2(l2_lambda),
bias_regularizer=l2(l2_lambda),
name="dense0")(X)
X = Dropout(dropout, name='dropout0')(X)
for i in range(1, layers):
X = Dense(nodes_dense1,
activation='relu',
kernel_regularizer=l2(l2_lambda),
bias_regularizer=l2(l2_lambda),
name='dense%d' % i)(X)
X = Dropout(dropout, name='dropout%d' % i)(X)
Y = Dense(1,
activation='sigmoid',
name="output",
bias_initializer=output_bias)(X)
# Define model with inputs and outputs
model = Model(inputs=X_input,
outputs=Y,
name='HIFIS-rnn-mlp_' + str(metadata['N_WEEKS']) + '-weeks')
# Set model loss function, optimizer, metrics.
model.compile(loss=f1_loss(4.5), optimizer=optimizer, metrics=metrics)
# Print summary of model and return model object
if hparams is None:
model.summary()
return model
x_test = x_test / 255
x_train = x_train.reshape(x_train.shape[0], 28 * 28)
x_test = x_test.reshape(x_test.shape[0], 28 * 28)
# x_train=x_train.reshape(x_train.shape[0], x_train.shape[1], x_train.shape[2], 1)
# x_test=x_test.reshape(x_test.shape[0], x_test.shape[1], x_test.shape[2], 1)
y_train = keras.utils.to_categorical(y_train, 10)
y_test = keras.utils.to_categorical(y_test, 10)
input_layer = Input(28 * 28)
d1 = Dense(units=128, activation='sigmoid')(input_layer)
# c1=Conv2D(32, kernel_size=(3, 3), activation='relu')(input_layer)
# p1=MaxPool2D(pool_size=(2, 2))(c1)
# f1=Flatten()(p1)
d2 = Dense(units=10, activation='softmax')(d1)
model = Model(inputs=input_layer, outputs=d2)
model.compile(
# optimizer=keras.optimizers.SGD(learning_rate=LEARNINT_RATE,),
optimizer=keras.optimizers.SGD(learning_rate=LEARNINT_RATE),
loss='categorical_crossentropy',
metrics=['acc'],
)
train_jistory = model.fit(
x=x_train,
y=y_train,
batch_size=BATCH_SIZE,
epochs=EPOCHS,
verbose=1,
validation_split=0.2,
)
# Training constants
n_weights = 80
lr = 3e-4
alpha = 0.1
train_dataset_size = 1000 # Get subset of data due to slowness
val_dataset_size = 100
n_epochs = 100
# Tensorflow Keras model
Y_input = Input(shape=(solver.n_obs, ), name='Y_input')
layer1 = Dense(n_weights, input_shape=(solver.dofs, ), activation='relu')
layer1out = layer1(Y_input)
layer2 = Dense(n_weights, activation='relu')
layer2out = layer2(layer1out)
U_output = Dense(solver.dofs, name='U_output')(layer2out)
model = Model(inputs=Y_input, outputs=U_output)
L = tf.reduce_mean(tf.square(Y_input - G(tf.math.exp(U_output))))
grad_bias_1 = gradients(L, model.trainable_weights[1])[0]
dummy_inp = np.random.randn(batch_size, solver.n_obs)
grad_bias_1_eval = get_session().run(grad_bias_1,
feed_dict={Y_input: dummy_inp})
bias_1_vals = layer1.get_weights()[1]
kernel_vals = layer1.get_weights()[0]
L_0 = get_session().run(L, feed_dict={Y_input: dummy_inp})
eps_bias = np.random.randn(
n_weights) #Random direction to perturn bias weights
eps_bias = eps_bias / np.linalg.norm(eps_bias)
dir_grad = np.dot(grad_bias_1_eval, eps_bias)
n_eps = 32
def YOLOv3Net(cfgfile, model_size, num_classes):
#Konstruisanje YOLO v3 modela.
#Parametri:
# cfgfile: Ime i putanja do konfiguracione datoteke
# model_size: Veličina ulaznih podataka u model (visina, širina, dubina)
# num_classes: broj klasa
#Povratna vrednost:
# model: YOLO v3 model
# učitavanje informacija iz konfiguracionog mrežnog fajla
blocks = parse_cfg(cfgfile)
outputs = {}
output_filters = []
filters = []
out_pred = []
scale = 0
inputs = input_image = Input(shape=model_size)
inputs = inputs / 255.0
# prolazak kroz slojeve i parametre slojeva i konstruisanje TensorFlow modela
for i, block in enumerate(blocks[1:]):
# Ako je u pitanju konvolucioni blok
if (block["type"] == "convolutional"):
activation = block["activation"]
filters = int(block["filters"])
kernel_size = int(block["size"])
strides = int(block["stride"])
if strides > 1:
inputs = ZeroPadding2D(((1, 0), (1, 0)))(inputs)
inputs = Conv2D(filters,
kernel_size,
strides=strides,
padding='valid' if strides > 1 else 'same',
name='conv_' + str(i),
use_bias=False if
("batch_normalize" in block) else True)(inputs)
if "batch_normalize" in block:
inputs = BatchNormalization(name='bnorm_' + str(i))(inputs)
inputs = LeakyReLU(alpha=0.1, name='leaky_' + str(i))(inputs)
# Ako je u pitanju blok povećanja dimenzionalnosti
elif (block["type"] == "upsample"):
stride = int(block["stride"])
inputs = UpSampling2D(stride)(inputs)
# Ako je u pitanju [route] blok
elif (block["type"] == "route"):
block["layers"] = block["layers"].split(',')
start = int(block["layers"][0])
if len(block["layers"]) > 1:
end = int(block["layers"][1]) - i
filters = output_filters[i + start] + output_filters[end]
inputs = tf.concat([outputs[i + start], outputs[i + end]],
axis=-1)
else:
filters = output_filters[i + start]
inputs = outputs[i + start]
# Ako je u pitanju [shortcut] blok
elif block["type"] == "shortcut":
from_ = int(block["from"])
inputs = outputs[i - 1] + outputs[i + from_]
# Yolo detekcioni sloj (nakon konvolucionih obrada)
elif block["type"] == "yolo":
mask = block["mask"].split(",")
mask = [int(x) for x in mask]
anchors = block["anchors"].split(",")
anchors = [int(a) for a in anchors]
anchors = [(anchors[i], anchors[i + 1])
for i in range(0, len(anchors), 2)]
anchors = [anchors[i] for i in mask]
n_anchors = len(anchors)
out_shape = inputs.get_shape().as_list()
inputs = tf.reshape(inputs, [-1, n_anchors * out_shape[1] * out_shape[2], \
5 + num_classes])
box_centers = inputs[:, :, 0:2]
box_shapes = inputs[:, :, 2:4]
confidence = inputs[:, :, 4:5]
classes = inputs[:, :, 5:num_classes + 5]
box_centers = tf.sigmoid(box_centers)
confidence = tf.sigmoid(confidence)
classes = tf.sigmoid(classes)
anchors = tf.tile(anchors, [out_shape[1] * out_shape[2], 1])
box_shapes = tf.exp(box_shapes) * tf.cast(anchors,
dtype=tf.float32)
x = tf.range(out_shape[1], dtype=tf.float32)
y = tf.range(out_shape[2], dtype=tf.float32)
cx, cy = tf.meshgrid(x, y)
cx = tf.reshape(cx, (-1, 1))
cy = tf.reshape(cy, (-1, 1))
cxy = tf.concat([cx, cy], axis=-1)
cxy = tf.tile(cxy, [1, n_anchors])
cxy = tf.reshape(cxy, [1, -1, 2])
strides = (input_image.shape[1] // out_shape[1], \
input_image.shape[2] // out_shape[2])
box_centers = (box_centers + cxy) * strides
prediction = tf.concat(
[box_centers, box_shapes, confidence, classes], axis=-1)
if scale:
out_pred = tf.concat([out_pred, prediction], axis=1)
else:
out_pred = prediction
scale = 1
outputs[i] = inputs
output_filters.append(filters)
# Ulazna slika predstavlja početnu tačku
# Predikcije iz YOLO sloja predstavljaju krajnju tačku modela
model = Model(input_image, out_pred)
# Ispis svih slojeva i parametara modela u konzolu
# model.summary()
return model
def build_model(days: int,
posts: int,
embedding_len: int,
words: int,
target_len: int,
threshold: float = None) -> Model:
"""
Build Keras model using Functional API. Model consists of three sub modules: Post Sub Model, Day Sub Model and
Fortnight Sub Model.
:param days: Number of days in each row. Determines the number of Day Sub Models
:param posts: Number of posts in each day. Determines the number of Post Sub Models
:param embedding_len: Length of embedding of a single post
:param words: Number of words per one post, important to specify the input shape
:param target_len: Length of the target vector. Determines the number of units in the last, dense layer.
:return: Returns a ready Keras model with empty weights
"""
# Post sub model parameters
# Convolution
post_highlights_filters = 80
post_highlights_kernel = 5
post_highlights_stride = 1
# Maxpool
post_maxpool_size = 10
post_maxpool_stride = 5
# Day sub model parameters
# Convolution
day_highlights_filters = 80
day_highlights_kernel = 3
day_highlights_stride = 1
# Maxpool
day_maxpool_size = 5
day_maxpool_stride = 3
# Fortnight sub model parameters
lstm_units = 150
input_shape = Input(shape=(days, posts, words, embedding_len))
'''
Choose 'relu' as activation function, as it supposedly performs fine. Will try other functions as necessary, if
the network does not learn anything.
Resources: https://towardsdatascience.com/visualizing-intermediate-activation-in-convolutional-neural-networks-with-keras-260b36d60d0
and: https://ai.stackexchange.com/questions/7088/how-to-choose-an-activation-function
and: https://missinglink.ai/guides/neural-network-concepts/7-types-neural-network-activation-functions-right/
'''
post_highlights_convolution = Conv1D(filters=post_highlights_filters,
kernel_size=post_highlights_kernel,
strides=post_highlights_stride,
activation=LeakyReLU(),
name='post_highlights_convolution')
post_permute = Permute(dims=(2, 1))
shared_post_maxpool = MaxPooling1D(pool_size=post_maxpool_size,
strides=post_maxpool_stride,
data_format='channels_first',
name='shared_post_maxpool')
day_highlights_convolution = Conv1D(filters=day_highlights_filters,
kernel_size=day_highlights_kernel,
strides=day_highlights_stride,
activation=LeakyReLU(),
name='day_highlights_convolution')
shared_day_maxpool = MaxPooling1D(pool_size=day_maxpool_size,
strides=day_maxpool_stride,
data_format='channels_first',
name='shared_day_maxpool')
day_outputs = []
for i in range(days):
# Slice the input into different days. Take the second dimension, because first one on the left is 'batch_size'
day = Lambda(lambda x: x[:, i, :, :, :])(input_shape)
post_outputs = []
for j in range(posts):
# Slice the input into different posts. Take the second dimension,
# because first one on the left is 'batch_size'. What is left is [words:embeddings]
out = Lambda(lambda x: x[:, j, :, :])(day)
post_x = (post_highlights_convolution)(out)
post_x = (post_permute)(post_x)
post_x = (shared_post_maxpool)(post_x)
post_outputs.append(post_x)
day_x = Concatenate()(post_outputs)
day_x = (day_highlights_convolution)(day_x)
day_x = (shared_day_maxpool)(day_x)
day_outputs.append(day_x)
week_overview = Concatenate()(day_outputs)
output_x = LSTM(units=lstm_units)(week_overview)
'''
activation='sigmoid' is important here because of the multi-label classification problem.
'''
output_x = Dense(units=target_len, activation='sigmoid')(output_x)
model = Model(inputs=input_shape, outputs=output_x)
'''
optimizer=Adam is sort of arbitrary choice, but was recommended as well-performing.
Check here: https://www.dlology.com/blog/quick-notes-on-how-to-choose-optimizer-in-keras/
and here: https://datascience.stackexchange.com/questions/10523/guidelines-for-selecting-an-optimizer-for-training-neural-networks
loss='binary_crossentropy' is important here because of the multi-label classification problem.
Check here: https://stackoverflow.com/questions/44164749/how-does-keras-handle-multilabel-classification
and here: https://towardsdatascience.com/multi-label-image-classification-with-neural-network-keras-ddc1ab1afede
'''
if threshold:
recall_metric = tf.keras.metrics.Recall(thresholds=threshold)
accuracy_metric = tf.keras.metrics.BinaryAccuracy(threshold=threshold)
precision_metric = tf.keras.metrics.Precision(thresholds=threshold)
else:
recall_metric = tf.keras.metrics.Recall()
accuracy_metric = tf.keras.metrics.BinaryAccuracy()
precision_metric = tf.keras.metrics.Precision()
roc_curve_metric = tf.keras.metrics.AUC(num_thresholds=50,
multi_label=False)
model.compile(optimizer=Adam(learning_rate=0.01),
loss='binary_crossentropy',
metrics=[
roc_curve_metric, recall_metric, precision_metric,
accuracy_metric
])
print(model.summary())
return model
x = layers.MaxPooling2D(2)(x)
# Flatten feature map to a 1-dim tensor
x = layers.Flatten()(x)
# Create a fully connected layer with ReLU activation and 512 hidden units
x = layers.Dense(512, activation='relu')(x)
# Add a dropout rate of 0.5
x = layers.Dropout(0.5)(x)
# Create output layer with a single node and sigmoid activation
output = layers.Dense(1, activation='sigmoid')(x)
# Configure and compile the model
model = Model(img_input, output)
model.compile(loss='binary_crossentropy',
optimizer=RMSprop(lr=0.001),
metrics=['acc'])
# In[11]:
history = model.fit_generator(train_generator,
steps_per_epoch=100,
epochs=30,
validation_data=validation_generator,
validation_steps=50,
verbose=2)
# In[ ]:
main_branch = TimeDistributed(utter_embedding)(inputs)
main_branch = Bidirectional(LSTM(128, return_sequences=True,
activation="relu"))(main_branch)
main_branch = Bidirectional(LSTM(64, return_sequences=True,
activation="relu"))(main_branch)
da_output = Dense(9,
activation="softmax",
kernel_regularizer=regularizers.l2(0.01),
name="da_output")(main_branch)
st_output = Dense(1,
activation="sigmoid",
kernel_regularizer=regularizers.l2(0.01),
name="st_output")(main_branch)
model = Model(inputs=inputs, outputs=[da_output, st_output])
model.compile(optimizer=RMSprop(lr=0.2, decay=0.001),
loss={
"da_output": "categorical_crossentropy",
"st_output": "binary_crossentropy"
},
loss_weights={
"da_output": 0.5,
"st_output": 1.0
},
metrics=['accuracy'])
model.summary()
plot_model(model,
to_file='model_plot.png',
def compiled_tcn(
num_feat, # type: int
num_classes, # type: int
nb_filters, # type: int
kernel_size, # type: int
dilations, # type: List[int]
nb_stacks, # type: int
max_len, # type: int
padding='causal', # type: str
use_skip_connections=True, # type: bool
return_sequences=True,
regression=False, # type: bool
dropout_rate=0.05, # type: float
name='tcn', # type: str,
activation='linear', # type:str,
opt='adam',
lr=0.002,
metrics=[]):
# type: (...) -> keras.Model
"""Creates a compiled TCN model for a given task (i.e. regression or
classification). Classification uses a sparse categorical loss. Please
input class ids and not one-hot encodings.
Args:
num_feat: The number of features of your input, i.e. the last dimension of: (batch_size, timesteps, input_dim).
num_classes: The size of the final dense layer, how many classes we are predicting.
nb_filters: The number of filters to use in the convolutional layers.
kernel_size: The size of the kernel to use in each convolutional layer.
dilations: The list of the dilations. Example is: [1, 2, 4, 8, 16, 32, 64].
nb_stacks : The number of stacks of residual blocks to use.
max_len: The maximum sequence length, use None if the sequence length is dynamic.
padding: The padding to use in the convolutional layers.
use_skip_connections: Boolean. If we want to add skip connections from input to each residual block.
return_sequences: Boolean. Whether to return the last output in the output sequence, or the full sequence.
regression: Whether the output should be continuous or discrete.
dropout_rate: Float between 0 and 1. Fraction of the input units to drop.
name: Name of the model. Useful when having multiple TCN.
activation: The activation used in the residual blocks o = Activation(x + F(x)).
opt: Optimizer name.
lr: Learning rate.
Returns:
A compiled keras TCN.
"""
dilations = process_dilations(dilations)
# input_layer = Input(shape=(max_len, num_feat))
# --- Vincent Added to deal with reshaping input ---
total_input = max_len * num_feat # max_len~wind size and num_feat~selected columns
input_layer = Input(shape=(total_input, ))
reshape_layer = Reshape((max_len, num_feat),
input_shape=(total_input, ))(input_layer)
# --- Vincent Stopped breaking stuff here ---
x = TCN(nb_filters, kernel_size, nb_stacks, dilations, padding,
use_skip_connections, dropout_rate, return_sequences, activation,
name)(reshape_layer)
print('x.shape=', x.shape)
def get_opt():
if opt == 'adam':
return optimizers.Adam(lr=lr, clipnorm=1.)
elif opt == 'rmsprop':
return optimizers.RMSprop(lr=lr, clipnorm=1.)
else:
raise Exception('Only Adam and RMSProp are available here')
if not regression:
# classification
x = Dense(num_classes)(x)
x = Activation('softmax')(x)
output_layer = x
model = Model(input_layer, output_layer)
# https://github.com/keras-team/keras/pull/11373
# It's now in Keras@master but still not available with pip.
# TODO remove later.
def accuracy(y_true, y_pred):
# reshape in case it's in shape (num_samples, 1) instead of (num_samples,)
if K.ndim(y_true) == K.ndim(y_pred):
y_true = K.squeeze(y_true, -1)
# convert dense predictions to labels
y_pred_labels = K.argmax(y_pred, axis=-1)
y_pred_labels = K.cast(y_pred_labels, K.floatx())
return K.cast(K.equal(K.cast(y_true, K.floatx()), y_pred_labels),
K.floatx())
model.compile(get_opt(),
loss='sparse_categorical_crossentropy',
metrics=[accuracy] + metrics)
else:
# regression
x = Dense(1)(x)
x = Activation('linear')(x)
output_layer = x
model = Model(input_layer, output_layer)
model.compile(get_opt(), loss='mean_squared_error')
print(f'model.x = {input_layer.shape}')
print(f'model.y = {output_layer.shape}')
return model
def U_Net_3D_Xiao(image_shape, lr=0.0001, n_classes=2):
"""Create 3D U-Net.
Parameters
----------
image_shape : array of 3 int
Dimensions of the input image.
activation : str, optional
Keras available activation type.
lr : float, optional
Learning rate value.
n_classes : int, optional
Number of classes.
Returns
-------
model : Keras model
Xiao 3D U-Net type proposed network model.
Here is a picture of the network extracted from the original paper:
.. image:: img/xiao_network.jpg
:width: 100%
:align: center
"""
dinamic_dim = (None, ) * (len(image_shape) - 1) + (1, )
inputs = Input(dinamic_dim)
x = Conv3D(3, (3, 3, 3),
activation=None,
kernel_initializer='he_normal',
padding='same')(inputs)
x = BatchNormalization()(x)
x = ELU()(x)
# Encoder
s1 = residual_block(x, 32)
x = MaxPooling3D((2, 2, 1), padding='same')(s1)
s2 = residual_block(x, 48)
x = MaxPooling3D((2, 2, 1), padding='same')(s2)
s3 = residual_block(x, 64)
x = MaxPooling3D((2, 2, 1), padding='same')(s3)
x = residual_block(x, 80)
# Decoder
x = UpSampling3D((2, 2, 1))(x)
x = Conv3D(64, (1, 1, 1),
activation=None,
kernel_initializer='he_normal',
padding='same')(x)
x = BatchNormalization()(x)
x = ELU()(x)
x = Add()([s3, x])
x = residual_block(x, 64)
x = UpSampling3D((2, 2, 1))(x)
# Auxiliary ouput 1
a1 = UpSampling3D((2, 2, 1))(x)
a1 = Conv3D(n_classes, (1, 1, 1),
activation=None,
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(0.01),
name='aux1')(a1)
a1 = Activation('softmax')(a1)
x = Conv3D(48, (1, 1, 1),
activation=None,
kernel_initializer='he_normal',
padding='same')(x)
x = BatchNormalization()(x)
x = ELU()(x)
x = Add()([s2, x])
x = residual_block(x, 48)
x = UpSampling3D((2, 2, 1))(x)
x = Conv3D(32, (1, 1, 1),
activation=None,
kernel_initializer='he_normal',
padding='same')(x)
x = BatchNormalization()(x)
x = ELU()(x)
# Auxiliary ouput 2
a2 = Conv3D(n_classes, (1, 1, 1),
activation=None,
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(0.01),
name='aux2')(x)
a2 = Activation('softmax')(a2)
x = Add()([s1, x])
x = residual_block(x, 32)
x = Conv3D(3, (3, 3, 3),
activation=None,
kernel_initializer='he_normal',
padding='same',
kernel_regularizer=l2(0.01))(x)
x = BatchNormalization()(x)
x = ELU()(x)
# Adapt the output to use softmax pixel-wise
x = Conv3D(n_classes, (1, 1, 1),
activation=None,
kernel_initializer='he_normal',
padding='same')(x)
outputs = Activation('softmax', name='main_classifier')(x)
model = Model(inputs=[inputs], outputs=[a1, a2, outputs])
opt = tf.keras.optimizers.Adam(lr=lr,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-8,
decay=0.0,
amsgrad=False)
model.compile(optimizer=opt,
loss="categorical_crossentropy",
loss_weights=[0.15, 0.3, 1],
metrics=[jaccard_index_softmax])
return model
for residual in self.res_blocks:
x = residual(x)
return self.output_conv(x)
if __name__ == "__main__":
from tensorflow.keras.layers import Input
import numpy as np
encoder = Encoder(3,
1, [3, 3, 3], [32, 64, 128], [1, 1, 1],
initializer='he_normal')
decoder = Decoder(1,
3,
1, [3, 3, 3], [128, 64, 32], [1, 1, 1],
initializer='he_normal')
inputTensor = Input(shape=(28, 28, 1))
x = encoder(inputTensor)
x = decoder(x)
model = Model(inputs=inputTensor, outputs=x)
model.compile(loss='mse', optimizer='sgd')
model.summary()
filter_num, activation)
filter_num *= 2
previousLayer = layers.MaxPool2D((2, 2))(previousLayer)
temp_layer = previousLayer
previousLayer, temp_layer = shortcut_block(previousLayer, temp_layer,
filter_num, activation)
# average pool, flatten, then dense to instatntiate output layer
outputLayer = layers.AveragePooling2D((2, 2))(previousLayer)
outputLayer = layers.Flatten()(outputLayer)
outputLayer = layers.Dense(outputDimension,
activation='sigmoid',
kernel_regularizer=regularizers.l2(0.001),
name='outputLayer')(outputLayer)
#compile model
ourResnet = Model(inputs=inputLayer, outputs=[outputLayer], name='fashion_mlp')
ourResnet.compile(loss='categorical_crossentropy',
metrics=['acc'],
optimizer='adam')
print(ourResnet.summary())
# In[35]:
outputResults = {}
for x in range(1, 11):
# training the resnet model
trainingLabelsCategorical = utils.to_categorical(
trainingLabels, num_classes=outputDimension)
# ourResnet.fit(trainingData, trainingLabelsCategorical, epochs=1, validation_split=0.85)
# testing the resnet model
def ResNet18_6_Parted(classes, input_shape, weight_decay=1e-4):
input = Input(shape=input_shape)
x = input / 255
# x = conv2d_bn_relu(x, filters=64, kernel_size=(7, 7), weight_decay=weight_decay, strides=(2, 2))
# x = MaxPool2D(pool_size=(3, 3), strides=(2, 2), padding='same')(x)
x = conv2d_bn_relu(x,
filters=64,
kernel_size=(3, 3),
weight_decay=weight_decay,
strides=(1, 1))
# # conv 2
x = ResidualBlock(x,
filters=64,
kernel_size=(3, 3),
weight_decay=weight_decay,
downsample=False)
x = ResidualBlock(x,
filters=64,
kernel_size=(3, 3),
weight_decay=weight_decay,
downsample=False)
# # conv 3
x = ResidualBlock(x,
filters=128,
kernel_size=(3, 3),
weight_decay=weight_decay,
downsample=True)
x = ResidualBlock(x,
filters=128,
kernel_size=(3, 3),
weight_decay=weight_decay,
downsample=False)
# # conv 4
x = ResidualBlock(x,
filters=256,
kernel_size=(3, 3),
weight_decay=weight_decay,
downsample=True)
x = ResidualBlock(x,
filters=256,
kernel_size=(3, 3),
weight_decay=weight_decay,
downsample=False)
# # conv 5
x = ResidualBlock(x,
filters=512,
kernel_size=(3, 3),
weight_decay=weight_decay,
downsample=True)
x = ResidualBlock(x,
filters=512,
kernel_size=(3, 3),
weight_decay=weight_decay,
downsample=False)
x = AveragePooling2D(pool_size=(5, 15), padding='valid')(x)
x = Flatten()(x)
x1 = Dense(classes,
name="digit1",
activation='softmax',
kernel_initializer='he_normal')(x)
x2 = Dense(classes,
name="digit2",
activation='softmax',
kernel_initializer='he_normal')(x)
x3 = Dense(classes,
name="digit3",
activation='softmax',
kernel_initializer='he_normal')(x)
x4 = Dense(classes,
name="digit4",
activation='softmax',
kernel_initializer='he_normal')(x)
x5 = Dense(classes,
name="digit5",
activation='softmax',
kernel_initializer='he_normal')(x)
x6 = Dense(classes,
name="digit6",
activation='softmax',
kernel_initializer='he_normal')(x)
model = Model(input, outputs=[x1, x2, x3, x4, x5, x6], name='ResNet18')
return model
def retinanet_resnet(inputs, K, A):
"""Retinanet with resnet backbone. Classification and regression networks share weights across feature pyramid
layers"""
# --- Define kwargs dictionary
kwargs1 = {
'kernel_size': (1, 1, 1),
'padding': 'valid',
}
kwargs3 = {
'kernel_size': (1, 3, 3),
'padding': 'same',
}
kwargs7 = {
'kernel_size': (1, 7, 7),
'padding': 'valid',
}
# --- Define block components
conv1 = lambda x, filters, strides: layers.Conv3D(
filters=filters, strides=strides, **kwargs1)(x)
conv3 = lambda x, filters, strides: layers.Conv3D(
filters=filters, strides=strides, **kwargs3)(x)
relu = lambda x: layers.LeakyReLU()(x)
conv7 = lambda x, filters, strides: layers.Conv3D(
filters=filters, strides=strides, **kwargs7)(x)
max_pool = lambda x, pool_size, strides: layers.MaxPooling3D(
pool_size=pool_size, strides=strides, padding='valid')(x)
norm = lambda x: layers.BatchNormalization()(x)
add = lambda x, y: layers.Add()([x, y])
zeropad = lambda x, padding: layers.ZeroPadding3D(padding=padding)(x)
upsamp2x = lambda x: layers.UpSampling3D(size=(1, 2, 2))(x)
# --- Define stride-1, stride-2 blocks
# conv1 = lambda filters, x : relu(conv(x, filters, strides=1))
# conv2 = lambda filters, x : relu(conv(x, filters, strides=(2, 2)))
# --- Residual blocks
# conv blocks
conv_1 = lambda filters, x, strides: relu(
norm(conv1(x, filters, strides=strides)))
conv_2 = lambda filters, x: relu(norm(conv3(x, filters, strides=1)))
conv_3 = lambda filters, x: norm(conv1(x, filters, strides=1))
conv_sc = lambda filters, x, strides: norm(
conv1(x, filters, strides=strides))
conv_block = lambda filters1, filters2, x, strides: relu(
add(conv_3(filters2, conv_2(filters1, conv_1(filters1, x, strides))),
conv_sc(filters2, x, strides)))
# identity blocks
identity_1 = lambda filters, x: relu(norm(conv1(x, filters, strides=1)))
identity_2 = lambda filters, x: relu(norm(conv3(x, filters, strides=1)))
identity_3 = lambda filters, x: norm(conv1(x, filters, strides=1))
identity_block = lambda filters1, filters2, x: relu(
add(
identity_3(filters2, identity_2(filters1, identity_1(filters1, x))
), x))
# --- feature pyramid blocks
fp_block = lambda x, y: add(upsamp2x(x), conv1(y, 256, strides=1))
# --- classification head
class_subnet = classification_head(K, A)
# --- regression head
box_subnet = regression_head(A)
# --- ResNet-50 backbone
# stage 1 c2 1/4
res1 = max_pool(zeropad(
relu(
norm(
conv7(zeropad(inputs['dat'], (0, 3, 3)), 64,
strides=(1, 2, 2)))), (0, 1, 1)), (1, 3, 3),
strides=(1, 2, 2))
# stage 2 c2 1/4
res2 = identity_block(
64, 256, identity_block(64, 256, conv_block(64, 256, res1, strides=1)))
# stage 3 c3 1/8
res3 = identity_block(
128, 512,
identity_block(
128, 512,
identity_block(128, 512,
conv_block(128, 512, res2, strides=(1, 2, 2)))))
# stage 4 c4 1/16
res4 = identity_block(
256, 1024,
identity_block(
256, 1024,
identity_block(
256, 1024,
identity_block(
256, 1024,
identity_block(
256, 1024,
conv_block(256, 1024, res3, strides=(1, 2, 2)))))))
# stage 5 c5 1/32
res5 = identity_block(
512, 2048,
identity_block(512, 2048, conv_block(512,
2048,
res4,
strides=(1, 2, 2))))
# --- Feature Pyramid Network architecture
# p5 1/32
fp5 = conv1(res5, 256, strides=1)
# p4 1/16
fp4 = fp_block(fp5, res4)
p4 = conv3(fp4, 256, strides=1)
# p3 1/8
fp3 = fp_block(fp4, res3)
p3 = conv3(fp3, 256, strides=1)
# p6 1/4
# p6 = conv3(fp5, 256, strides=(2, 2))
# p7 1/2
# p7 = conv3(relu(p6), 256, strides=(2, 2))
feature_pyramid = [p3, p4, fp5]
# lambda layer that allows multiple outputs from a shared model to have specific names
# layers.Lambda(lambda x:x, name=name)()
# --- Class subnet
class_outputs = [class_subnet(features) for features in feature_pyramid]
# --- Box subnet
box_outputs = [box_subnet(features) for features in feature_pyramid]
# --- put class and box outputs in dictionary
logits = {
'cls-c3': layers.Lambda(lambda x: x, name='cls-c3')(class_outputs[0]),
'reg-c3': layers.Lambda(lambda x: x, name='reg-c3')(box_outputs[0]),
'cls-c4': layers.Lambda(lambda x: x, name='cls-c4')(class_outputs[1]),
'reg-c4': layers.Lambda(lambda x: x, name='reg-c4')(box_outputs[1]),
'cls-c5': layers.Lambda(lambda x: x, name='cls-c5')(class_outputs[2]),
'reg-c5': layers.Lambda(lambda x: x, name='reg-c5')(box_outputs[2])
}
model = Model(inputs=inputs, outputs=logits)
return model
