def get_filtered_filenames(voc_root_folder="../data/VOCdevkit/",
filter=CLASSES) -> list:
"""
Get a list of filtered filenames (no extension), based on a filter.
:param voc_root_folder: root folder of the PASCAL dataset
:return:
"""
log("Building list of filtered filenames.", lvl=2)
# Get annotation files (XML)
annotation_folder = os.path.join(voc_root_folder, "VOC2009/Annotations/")
annotation_files = os.listdir(annotation_folder)
filtered_filenames = []
# Go over all annotation files
for a_f in annotation_files:
# Parse each file
tree = ET.parse(os.path.join(annotation_folder, a_f))
# Check for presence of each of the filter classes
check = [
tag.text == filt for tag in tree.iterfind(".//name")
for filt in filter
]
# If any of them are present, add it to the list (and disregard the extension)
if np.any(check):
filtered_filenames.append(a_f[:-4])
return filtered_filenames
def evaluate_classifier(classifier: Model,
x: np.ndarray,
y: np.ndarray,
threshold=.5,
joint=False) -> dict:
"""
Evaluate classifier on number of metrics.
:param classifier: the model that is to be evaluated.
:param x: training (input) data which will lead to predictions
:param y: expected outcomes
:param threshold: boundary for prediction value to be considered 0 or 1
:return:
"""
log("Evaluating classifier.", lvl=2)
# Store metrics in dictionary
metrics = {}
# Get probabilities
if joint:
y_prob = classifier.predict(x)[1]
else:
y_prob = classifier.predict(x)
# ... and extract for predictions
y_pred = y_prob
super_threshold_indices = y_prob > threshold
y_pred[super_threshold_indices] = 1
y_pred[np.invert(super_threshold_indices)] = 0
pp = PrettyPrinter(indent=4)
metrics['Hamming loss'] = hamming_loss(y, y_pred)
metrics['Exact match ratio'] = accuracy_score(y, y_pred)
metrics['Hamming score'] = hamming_score(y, y_pred)
# metrics['Precision score (micro)'] = precision_score(y, y_pred, average='micro')
# metrics['Precision score (macro)'] = precision_score(y, y_pred, average='macro')
# metrics['F1 score (micro)'] = f1_score(y, y_pred, average='micro')
# metrics['Label-based accuracy'] = label_based_accuracy(y, y_pred)
metrics['Jaccard similarity index'] = jaccard_similarity_score(y, y_pred)
metrics['F1 score (macro)'] = f1_score(y, y_pred, average='macro')
"""In a multi-class classification setup, micro-average is preferable if you suspect
there might be class imbalance (i.e you may have many more examples of one class
than of other classes) -> We know this is not the case."""
# pp.pprint(metrics)
return metrics
def lift(x: np.ndarray) -> np.ndarray:
"""
Expand input matrix (n image vectors of size image_dim x image_dim x 3).
:param x: input matrix
:return: expanded input matrix
"""
if not len(x.shape) > 2:
image_size = x.shape[1]
image_dim = round(sqrt(image_size / 3))
return x.reshape((len(x), image_dim, image_dim, 3))
else:
log("Failed to lift x - already expanded.", lvl=3)
return x
def squash(x: np.ndarray) -> np.ndarray:
"""
Flatten input matrix (n images of size image_dim x image_dim x 3).
:param x: input matrix (
:return: flattened input matrix
"""
if len(x.shape) > 2:
image_dim = x.shape[1]
image_size = image_dim**2 * 3
return x.reshape((len(x), image_size))
else:
log("Failed to squash x - already flattened.", lvl=3)
return x
def build_autoencoder(model_name: str, convolutional: bool, train: bool,
x_tr: np.ndarray, x_va: np.ndarray,
compression_factor: int, epochs=100,
optimizer="adam", loss="mean_squared_error",
patience=5) -> \
(Sequential, Sequential, float):
"""
Build an autoencoder.
:param model_name: name used in file creation
:param x_tr: training images
:param x_va: validation images
:param compression_factor: what's the size of the bottleneck?
:param epochs: number of epochs to train for
:param optimizer: optimizer to use in training
:param loss: loss function to use in training
:param patience: number of epochs without improvement before early stop
:return: autoencoder model, encoder model, decoder model, compression factor
"""
# Model parameters
image_dim = x_tr.shape[1]
image_size = image_dim**2 * 3
encoding_dim = image_size // compression_factor
# Set parameters
batch_size = 128
if hlp.LOG_LEVEL == 3:
verbose = 1
elif hlp.LOG_LEVEL == 2:
verbose = 2
else:
verbose = 0
# Full model name for file output
full_model_name = model_name + '_im_dim' + str(image_dim) + '-comp' + str(
int(compression_factor))
# Build model path
model_path = os.path.join(os.pardir, "models", "autoencoders",
full_model_name + ".h5")
architecture_path = os.path.join(os.pardir, "models", "autoencoders",
"architecture",
full_model_name + "_architecture.png")
# Keep track of history
history = []
history_path = os.path.join(os.pardir, "models", "autoencoders", "history",
full_model_name + "_history.npy")
# Try loading the model, ...
try:
autoencoder = load_model(model_path)
log("Found model \'", model_name, "\' locally.", lvl=3)
history = np.load(file=history_path).tolist()
log("Found model \'", model_name, "\' history locally.", lvl=3)
# ... otherwise, create it
except:
if convolutional:
log("Building deep convolutional autoencoder.", lvl=2)
autoencoder = convolutional_auto_architecture(
image_dim=image_dim,
optimizer=optimizer,
loss=loss,
compression_factor=compression_factor)
else:
log("Building linear autoencoder.", lvl=2)
autoencoder = linear_auto_architecture(
image_size=image_size,
optimizer=optimizer,
loss=loss,
compression_factor=compression_factor)
# Print model info
log("Network parameters: image dimension {}, image size {}, encoding dimension {}, compression factor {}."
.format(image_dim, image_size, encoding_dim, compression_factor),
lvl=3)
# Train the model (either continue training the old model, or train the new one)
if train:
log("Training autoencoder.", lvl=2)
# Flatten image data for linear model
if not convolutional:
# Flatten
x_tr = squash(x_tr)
x_va = squash(x_va)
# Training parameters
es = EarlyStopping(monitor='val_loss',
patience=patience,
verbose=verbose)
mc = ModelCheckpoint(filepath=model_path,
monitor='val_loss',
verbose=verbose,
save_best_only=True)
tb = TensorBoard(log_dir='/tmp/' + model_name + '_im' +
str(image_dim) + 'comp' +
str(int(compression_factor)))
# Data augmentation to get the most out of our images
'''datagen = ImageDataGenerator(
rotation_range=15,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
)
datagen.fit(x_tr)'''
# Train model
history.append(
autoencoder.fit(x_tr,
x_tr,
epochs=epochs,
batch_size=batch_size,
verbose=verbose,
validation_data=(x_va, x_va),
callbacks=[es, mc, tb]))
# Save model and history
# autoencoder.save(autoencoder_path) # <- already stored at checkpoint
np.save(file=history_path, arr=history)
# Visual aid
plot_model(autoencoder,
to_file=architecture_path,
show_layer_names=True,
show_shapes=True)
# Get intermediate output at encoded layer
if convolutional:
encoder = Model(inputs=autoencoder.input,
outputs=autoencoder.get_layer(name='encoder').output)
else:
encoder = Model(inputs=autoencoder.input,
outputs=autoencoder.get_layer(index=0).output)
# See effect of encoded representations on output (eigenfaces?)
'''decoder = Model(inputs=Input(shape=(encoding_dim,)), outputs=autoencoder.output)
plot_model(autoencoder, to_file=os.path.join(os.pardir, "models", "decoder.png"), show_layer_names=True, show_shapes=True)'''
# Size of encoded representation
log("Compression factor is {}, encoded vector length is {}.".format(
compression_factor, encoding_dim),
lvl=3)
return autoencoder, encoder, history
# Show 'n tell
if save: fig.savefig(plot_path, dpi=fig.dpi)
if plot: plt.show()
return ax
########
# MAIN #
########
if __name__ == "__main__":
"""Main function."""
# Let's go
log("AUTOENCODERS", title=True)
# Set parameters
hlp.LOG_LEVEL = 3
tf.logging.set_verbosity(tf.logging.ERROR)
# Check folders
set_up_model_directory('autoencoders')
# Model approaches
linear, linear_train = True, True
convolutional, convolutional_train = True, True
# Loop over these dimensions
'''image_dims = (48,)
compression_factors = (48,)
def build_unet_segmentation_network(model_name: str,
x_tr: np.ndarray,
y_tr: np.ndarray,
x_va: np.ndarray,
y_va: np.ndarray,
optimizer='adam',
loss='mean_squared_error',
epochs=200,
patience=25,
verbose=1) -> (Model, History):
"""
Build and train U-net inspired segmentation network.
:param x_tr: training images
:param y_tr: training segmentations
:param x_va: validation images
:param y_va: validation segmentations
:param optimizer: (sic)
:param loss: (sic)
:param epochs: (sic)
:param patience: (sic)
:param verbose: (sic)
:return: U-net model and its training history
"""
# Set parameters
image_dim = x_tr.shape[1]
image_size = image_dim**2 * 3
input_shape = (image_dim, image_dim, 3)
# Build model path
model_name = "{}_im{}".format(model_name, image_dim)
model_path = os.path.join(os.pardir, "models", "segmentation",
model_name + ".h5")
architecture_path = os.path.join(os.pardir, "models", "segmentation",
"architecture",
model_name + "_architecture.png")
history_path = os.path.join(os.pardir, "models", "segmentation", "history",
model_name + "_history.npy")
# Try loading the model, ...
try:
unet = load_model(model_path)
log("Found model \'", model_name, "\' locally.", lvl=3)
history = np.load(file=history_path).tolist()
log("Found model \'", model_name, "\' history locally.", lvl=3)
# ... otherwise, create it
except Exception as e:
log("Exception:", e, lvl=3)
# Build U-Net model
image_input = Input(shape=input_shape)
c1 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(image_input)
c1 = Dropout(0.1)(c1)
c1 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c1)
p1 = MaxPooling2D((2, 2))(c1)
c2 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p1)
c2 = Dropout(0.1)(c2)
c2 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c2)
p2 = MaxPooling2D((2, 2))(c2)
c3 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p2)
c3 = Dropout(0.2)(c3)
c3 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c3)
p3 = MaxPooling2D((2, 2))(c3)
c4 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p3)
c4 = Dropout(0.2)(c4)
c4 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c4)
p4 = MaxPooling2D(pool_size=(2, 2))(c4)
c5 = Conv2D(256, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p4)
c5 = Dropout(0.3)(c5)
c5 = Conv2D(256, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c5)
u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)
u6 = concatenate([u6, c4])
c6 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u6)
c6 = Dropout(0.2)(c6)
c6 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c6)
u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)
u7 = concatenate([u7, c3])
c7 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u7)
c7 = Dropout(0.2)(c7)
c7 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c7)
u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)
u8 = concatenate([u8, c2])
c8 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u8)
c8 = Dropout(0.1)(c8)
c8 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c8)
u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)
u9 = concatenate([u9, c1], axis=3)
c9 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u9)
c9 = Dropout(0.1)(c9)
c9 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c9)
outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)
unet = Model(inputs=[image_input], outputs=[outputs])
unet.compile(optimizer=optimizer, loss=loss)
unet.summary()
plot_model(unet, to_file=architecture_path, show_shapes=True)
# Callbacks
if patience == 0:
patience = epochs
es = EarlyStopping(monitor='val_loss',
patience=patience,
verbose=verbose)
mc = ModelCheckpoint(filepath=model_path,
monitor='val_loss',
verbose=verbose,
save_best_only=True)
tb = TensorBoard(log_dir="/tmp/segm_unet_im{}".format(image_dim))
# Data augmentation to get the most out of our images
datagen = ImageDataGenerator(
rotation_range=15,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
)
datagen.fit(x_tr)
# Train model using data augmentation
history = unet.fit_generator(datagen.flow(x_tr, y_tr, batch_size=128),
epochs=epochs,
steps_per_epoch=x_tr.shape[0] // 128,
verbose=verbose,
validation_data=(x_va, y_va),
callbacks=[es, mc, tb])
# Save model and history
np.save(file=history_path, arr=history)
plot_model(unet, to_file=architecture_path, show_shapes=True)
return unet, history
steps_per_epoch=x_tr.shape[0] // 128,
verbose=verbose,
validation_data=(x_va, y_va),
callbacks=[es, mc, tb])
# Save model and history
np.save(file=history_path, arr=history)
plot_model(unet, to_file=architecture_path, show_shapes=True)
return unet, history
if __name__ == "__main__":
# Let's go
log("U-NET SEGMENTATION NETWORK", title=True)
# Set logging parameters
hlp.LOG_LEVEL = 3
tf.logging.set_verbosity(tf.logging.ERROR)
# Check folders
set_up_model_directory('segmentation')
# Dashboard
image_dim = 96
compression_factor = 24
epochs = 500
patience = 0
loss = 'mean_squared_error'
def joint_model(model_name: str,
image_dim: int,
compression_factor: int,
verbose=1) -> (Model, History):
"""
Build and train a joint model: autoencoder plus classification training.
:param model_name: name for file output
:param image_dim: dimension of input image
:param compression_factor: (sic)
:param verbose: verbosity during training.
:return: joint model, plus its history
"""
# Full model name for file output
full_model_name = "{}_im{}comp{}".format(model_name, image_dim,
compression_factor)
# Build model paths
model_path = os.path.join(os.pardir, "models", "classifiers",
full_model_name + ".h5")
architecture_path = os.path.join(os.pardir, "models", "classifiers",
"architecture",
full_model_name + "_architecture.png")
history_path = os.path.join(os.pardir, "models", "classifiers", "history",
full_model_name + "_history.npy")
# Try loading the model, ...
try:
joint = load_model(model_path)
log("Found model \'", model_name, "\' locally.", lvl=3)
history = np.load(file=history_path)
log("Found model \'", model_name, "\' history locally.", lvl=3)
# ... otherwise, create it
except:
# Model parameters
input_shape = (image_dim, image_dim, 3)
image_size = np.prod(input_shape)
encoding_dim = image_size // compression_factor
# Build model base (encoder)
image_input = Input(shape=input_shape)
conv_1 = Conv2D(image_dim, (3, 3),
padding='same',
activation='relu',
kernel_initializer='random_uniform',
bias_initializer='zeros')(image_input)
batch_1 = BatchNormalization()(conv_1)
max_1 = MaxPooling2D((2, 2), padding='same')(batch_1)
conv_2 = Conv2D(image_dim // (2 * compression_factor), (3, 3),
padding='same',
activation='relu',
kernel_initializer='random_uniform',
bias_initializer='zeros')(max_1)
batch_2 = BatchNormalization()(conv_2)
encoded = MaxPooling2D((2, 2), padding='same')(batch_2)
# Build autoencoder
conv_3 = Conv2D(image_dim // (2 * compression_factor), (3, 3),
padding='same',
activation='relu',
kernel_initializer='random_uniform',
bias_initializer='zeros')(encoded)
batch_3 = BatchNormalization()(conv_3)
up_1 = UpSampling2D((2, 2))(batch_3)
conv_4 = Conv2D(image_dim, (3, 3),
padding='same',
activation='relu',
kernel_initializer='random_uniform',
bias_initializer='zeros')(up_1)
batch_4 = BatchNormalization()(conv_4)
up_2 = UpSampling2D((2, 2))(batch_4)
autoencoder = Conv2D(3, (10, 10),
padding='same',
activation='sigmoid',
kernel_initializer='random_uniform',
bias_initializer='zeros',
name='autoencoder')(up_2)
# Build classifier
flatten = Flatten()(encoded)
dense_1 = Dense(encoding_dim, activation='relu')(flatten)
drop_1 = Dropout(.5)(dense_1)
classifier = Dense(5, activation='sigmoid', name='classifier')(drop_1)
# Build joint model
joint = Model(inputs=image_input, outputs=[autoencoder, classifier])
# Save model architecture visualization
plot_model(joint, to_file=architecture_path)
joint.compile(loss={
'classifier': 'binary_crossentropy',
'autoencoder': 'mean_squared_error'
},
optimizer='adam',
metrics={'classifier': 'accuracy'})
# Callbacks
es = EarlyStopping(monitor='val_loss',
patience=patience,
verbose=verbose)
mc = ModelCheckpoint(filepath=model_path,
monitor='val_loss',
verbose=verbose,
save_best_only=True)
tb = TensorBoard(log_dir="/tmp/{}_im{}comp{}".format(
model_name, image_dim, compression_factor))
history = joint.fit(x=x_tr,
y={
'classifier': y_tr,
'autoencoder': x_tr
},
batch_size=batch_size,
epochs=epochs,
validation_data=(x_te, {
'classifier': y_te,
'autoencoder': x_te
}),
verbose=1,
callbacks=[es, mc, tb])
np.save(file=history_path, arr=history)
return joint, history
def plot_model_history(history: History,
image_dim: int,
model_type="segmentation",
save=True,
plot=True) -> plt.Axes:
"""
Plot the histories of the model metrics 'loss' and 'val loss'.
:param histories: histories of trained models
:param image_dim: image dimension
:param compression_factor: (sic)
:param save: save plot to png
:param plot: show plot
:return: plot axes
"""
log("Plotting model metrics.", lvl=2)
# Set font
font = {
'fontname': 'Times New Roman Bold',
'fontfamily': 'serif',
'weight': 'bold'
}
# Set style
sns.set_style('whitegrid')
colors = sns.color_palette('pastel', 2)
# Initialize axes
fig = plt.figure(figsize=(8, 5), dpi=150)
ax = plt.axes()
# Fill axes
n_epochs = len(history.history['loss'])
y_label = 'Loss (mean squared error)'
# Plot training & validation accuracy values
ax.plot(history.history["loss"], label="Training", color=colors[0])
ax.plot(history.history["val_loss"], label="Validation", color=colors[1])
# Add vertical line to indicate early stopping
ax.axvline(x=n_epochs - 1, linestyle='--', color=colors[0])
# Set a title, the correct y-label, and the y-limit
ax.set_ylabel(y_label, fontdict={'fontname': 'Times New Roman'})
ax.set_ylim(0.1, 1.2)
ax.set_yscale("log")
ax.set_xlabel('Epoch', fontdict={'fontname': 'Times New Roman'})
ax.set_xlim(0, 500)
ax.legend(loc='best', prop={'family': 'Serif'})
# Build model path
evaluation_label = 'histories_segm_im_dim{}'.format(image_dim)
plot_path = os.path.join(os.pardir, "models", model_type, "plots",
evaluation_label + ".png")
# Show 'n tell
if save: fig.savefig(plot_path, dpi=fig.dpi)
if plot: plt.show()
return fig
def build_classifier(image_dim: int,
compression_factor: int,
x_tr: np.ndarray,
y_tr: np.ndarray,
x_va: np.ndarray,
y_va: np.ndarray,
from_scratch=False,
all_trainable=False,
loss='binary_crossentropy',
epochs=200,
patience=25) -> (Sequential, History):
"""
Build a deep convolutional classifier network, either from scratch, or from a pretrained autoencoder.
:param image_dim: dimension of training and validation images
:param compression_factor: size of bottleneck in network
:param x_tr: training images
:param y_tr: training labels
:param x_va: validation images
:param y_va: validation labels
:param from_scratch: do we start fresh, or from earlier weights?
:param all_trainable: are all weights trainable, or do we freeze the encoder part?
:param epochs: number of epochs to train for
:param patience: number of epochs to wait before we stop training, when seeing no improvement in validation loss
:return: classifier model, training history
"""
# Model parameters
image_size = image_dim**2 * 3
encoding_dim = image_size // compression_factor
# Set verbosity
if hlp.LOG_LEVEL == 3:
verbose = 1
elif hlp.LOG_LEVEL == 2:
verbose = 2
else:
verbose = 0
# Start building new Sequential model
classifier = Sequential()
# Use previously trained model (we could just load weights, too).
if not from_scratch:
# Build encoder
_, encoder, _ = build_autoencoder(
model_name='conv_auto',
convolutional=True,
train=False,
x_tr=x_tr,
x_va=x_va,
compression_factor=compression_factor)
# Freeze, dirtbag (or not, I'm not dirty Harry)
conv_layers = [1, 2, 4, 5]
for i in conv_layers:
encoder.get_layer(index=i).trainable = all_trainable
# Add encoding part of autoencoder to our classifier
classifier.add(encoder)
# ... unless we just use the architecture
else:
input_shape = (image_dim, image_dim, 3)
image_size = np.prod(input_shape)
# Build model
classifier = Sequential()
classifier.add(
Conv2D(image_dim, (3, 3),
padding='same',
activation='relu',
input_shape=input_shape,
kernel_initializer='random_uniform',
bias_initializer='zeros'))
classifier.add(BatchNormalization())
classifier.add(MaxPooling2D((2, 2), padding='same'))
classifier.add(
Conv2D(image_dim // compression_factor, (3, 3),
padding='same',
activation='relu',
kernel_initializer='random_uniform',
bias_initializer='zeros'))
classifier.add(BatchNormalization())
classifier.add(MaxPooling2D((2, 2), padding='same', name='encoder'))
# Add classification layers
classifier.add(Flatten())
classifier.add(Dense(encoding_dim))
classifier.add(Activation('relu'))
classifier.add(Dropout(0.5))
classifier.add(Dense(
5, activation='sigmoid')) # <-- multilabel (for multiclass: softmax)
classifier.compile(optimizer='adam', loss=loss, metrics=['accuracy'])
# Plot model
full_classifier_name = "{}_im_dim{}-comp{}_full{}_scratch{}_loss{}".format(
classifier_name, image_dim, compression_factor, all_trainable,
from_scratch, loss)
architecture_path = os.path.join(os.pardir, "models", "classifiers",
"architecture",
full_classifier_name + ".png")
plot_model(classifier,
to_file=architecture_path,
show_layer_names=True,
show_shapes=True)
# Data augmentation to get the most out of our images
datagen = ImageDataGenerator(
rotation_range=15,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
)
datagen.fit(x_tr)
# Callbacks
if patience == 0:
patience = epochs
es = EarlyStopping(monitor='val_loss', patience=patience, verbose=verbose)
mc = ModelCheckpoint(filepath=classifier_path,
monitor='val_loss',
verbose=verbose,
save_best_only=True)
tb = TensorBoard(
log_dir='/tmp/{}_class_im{}comp{}_full{}_scratch{}'.format(
classifier_name, image_dim, compression_factor, all_trainable,
from_scratch))
# Print model info
log("Network parameters: image dimension {}, image size {}, compression factor {}."
.format(image_dim, image_size, compression_factor),
lvl=3)
# Train model using data augmentation
history = classifier.fit_generator(datagen.flow(x_tr, y_tr, batch_size=32),
epochs=epochs,
steps_per_epoch=x_tr.shape[0] // 32,
verbose=verbose,
validation_data=(x_va, y_va),
callbacks=[es, mc, tb])
'''history = classifier.fit(x=x_tr, y=y_tr,
batch_size=32,
epochs=epochs,
verbose=verbose,
callbacks=[es, mc, tb],
validation_data=(x_va, y_va))'''
# Save model and history
classifier.save(classifier_path)
return classifier, history
# Show 'n tell
if save: plt.savefig(model_path, dpi=150)
if plot: plt.show()
return fig, subplot_axes
########
# MAIN #
########
if __name__ == "__main__":
# Let's go
log("CLASSIFIERS", title=True)
# Set logging parameters
hlp.LOG_LEVEL = 3
tf.logging.set_verbosity(tf.logging.ERROR)
# Check folders
set_up_model_directory('classifiers')
# Dashboard
image_dim = 96
compression_factor = 24
loss = 'binary_crossentropy'
epochs = 300
patience = 50
def plot_model_histories(histories: dict,
image_dim: int,
compression_factor: int,
save=True,
plot=True) -> plt.Axes:
"""
Plot the histories of the model metrics 'loss' and 'accuracy'.
:param histories: histories of trained models
:param image_dim: image dimension
:param compression_factor: (sic)
:param save: save plot to png
:param plot: show plot
:return: plot axes
"""
log("Plotting model metrics.", lvl=2)
# Set style
sns.set_style('whitegrid')
colors = sns.color_palette('pastel', n_colors=3)
# Initialize axes
fig, subplot_axes = plt.subplots(2,
2,
squeeze=False,
sharex='none',
sharey='none',
figsize=(11, 10),
constrained_layout=True)
# Fill axes
for col in range(2):
train_or_val = 'Training' if col == 0 else 'Validation'
for row in range(2):
ax = subplot_axes[row][col]
color_counter = 0
for label, history in histories.items():
n_epochs = len(history.history['loss'])
if row == 0:
key = 'acc' if col == 0 else 'val_acc'
title = '{} accuracy'.format(train_or_val)
y_label = 'Accuracy'
y_limit = (.7, .9)
else:
key = 'loss' if col == 0 else 'val_loss'
title = '{} loss'.format(train_or_val)
y_label = 'Loss (binary cross entropy)'
y_limit = (.2, .7)
# Plot training & validation accuracy values
ax.plot(history.history[key],
label="Model: {}".format(label),
color=colors[color_counter])
# Add vertical line to indicate early stopping
ax.axvline(x=n_epochs,
linestyle='--',
color=colors[color_counter])
# Set a title, the correct y-label, and the y-limit
ax.set_title(title, fontdict={'fontweight': 'semibold'})
ax.set_ylabel(y_label)
ax.set_ylim(y_limit)
color_counter += 1
if row == 1: ax.set_xlabel('Epoch')
ax.set_xlim(0, 200)
ax.legend(loc='best')
# Title
plt.suptitle(
"Training histories of classifier models (image dim {} - compression {})"
.format(image_dim, compression_factor),
fontweight='bold')
# Build model path
evaluation_label = 'histories_im_dim{}comp{}'.format(
image_dim, compression_factor)
plot_path = os.path.join(os.pardir, "models", "autoencoders", "plots",
evaluation_label + ".png")
# Show 'n tell
if save: fig.savefig(plot_path, dpi=fig.dpi)
if plot: plt.show()
return ax
def pipeline(image_dim=214,
filter=CLASSES,
class_data=True,
segm_data=False,
mono=True,
voc_root_folder="../data/VOCdevkit/") -> (dict, dict):
"""
Pipeline to extract training and validation datasets from full dataset.
:param image_dim: images will be resized to image_dim x image_dim pixels
:param filter: classes to take into account (out of total of 20)
:param class_data: load classification data
:param segm_data: load segmentation data
:param voc_root_folder: root folder of PASCAL dataset
:return: data dicts
"""
if not (class_data or segm_data):
raise Exception(
"No data requested! Choose classification data, segmentation data, or both."
)
log("Preparing data: image dimension {imdim}x{imdim}.".format(
imdim=image_dim),
lvl=1)
# Make folders, should they not exist yet
make_folders("data", "scripts",
os.path.join('data', 'image_dim_' + str(image_dim)))
# Try to load data sets locally
data_path = os.path.join(os.pardir, 'data', 'image_dim_' + str(image_dim),
"data_" + str(image_dim) + ".npy")
data_segm_path = os.path.join(
os.pardir, 'data', 'image_dim_' + str(image_dim),
"data_segm_" + str(image_dim) + ('mono' if mono else '') + ".npy")
# Build (x,y) for TRAIN/TRAINVAL/VAL (classification)
log("Building filtered dataset.", lvl=2)
classes_folder = os.path.join(voc_root_folder, "VOC2009", "ImageSets",
"Main")
classes_files = os.listdir(classes_folder)
# Only take training and validation images that pass filter
train_files = [
os.path.join(classes_folder, c_f) for filt in filter
for c_f in classes_files if filt in c_f and '_train.txt' in c_f
]
val_files = [
os.path.join(classes_folder, c_f) for filt in filter
for c_f in classes_files if filt in c_f and '_val.txt' in c_f
]
# If classification data is requested
if class_data:
# Try to load the data from disk
log("Getting classification data.", lvl=3)
try:
data = np.load(file=data_path).item()
log("Loaded classification data.", lvl=2)
train_images = data['train_images']
val_images = data['val_images']
except (NameError, FileNotFoundError):
log("Failed to load one of the {}-sized datasets. Rebuilding.".
format(image_dim),
lvl=1)
x_train, y_train, train_images = build_classification_dataset(
list_of_files=train_files,
image_dim=image_dim,
filter=filter,
voc_root_folder=voc_root_folder)
log("Extracted {} training images from {} classes.".format(
x_train.shape[0], y_train.shape[1]),
lvl=3)
x_val, y_val, val_images = build_classification_dataset(
list_of_files=val_files,
image_dim=image_dim,
filter=filter,
voc_root_folder=voc_root_folder)
log("Extracted {} validation images from {} classes.".format(
x_val.shape[0], y_train.shape[1]),
lvl=3)
# Keep random sample as test
test_indices = rnd.sample(range(len(x_val)), k=len(x_val) // 2)
val_indices = [
i for i in range(len(x_val)) if i not in test_indices
]
x_test = x_val[test_indices]
y_test = y_val[test_indices]
x_val = x_val[val_indices]
y_val = y_val[val_indices]
data = {
'x_train': x_train,
'y_train': y_train,
'x_val': x_val,
'y_val': y_val,
'x_test': x_test,
'y_test': y_test,
'train_images': train_images,
'val_images': val_images
}
# Save locally, because this takes a while
log("Storing classification data.", lvl=3)
np.save(data_path, data)
else:
data = {}
# If segmentation data is requested
if segm_data:
# Try to load segmentation data from disk
log("Getting segmentation data.", lvl=3)
try:
data_segm = np.load(file=data_segm_path).item()
log("Loaded segmentation data.", lvl=2)
# ... else build datasets
except (FileNotFoundError, NameError):
if not class_data:
raise Exception(
"Failed to load segmentation data. Set 'class_data' to true to build datasets!"
)
log("Failed to load segmentation data. Rebuilding.", lvl=3)
data_segm = build_segmentation_dataset(
train_list=train_images,
val_list=val_images,
image_dim=image_dim,
voc_root_folder=voc_root_folder,
mono=mono)
log("Storing segmentation data.", lvl=3)
np.save(data_segm_path, data_segm)
else:
data_segm = {}
# Return data
return data, data_segm
evaluation_label + ".png")
# Show 'n tell
if save: fig.savefig(plot_path, dpi=fig.dpi)
if plot: plt.show()
return ax
########
# MAIN #
########
if __name__ == "__main__":
# Let's go
log("SEGMENTATION", title=True)
# Set logging parameters
hlp.LOG_LEVEL = 3
tf.logging.set_verbosity(tf.logging.ERROR)
# Check folders
set_up_model_directory('segmentation')
# Dashboard
image_dim = 96
compression_factor = 24
train = False
epochs = 500
patience = 50
loss = 'mean_squared_error'
def plot_classifier_prediction_comparison(classifiers: list,
names: list,
model_name: str,
compression_factor: float,
loss: str,
x: np.ndarray,
y_true: np.ndarray,
examples=5,
random=True,
save=True,
plot=True,
indices=None):
"""
Show some images, their true class labels, and the predicted class labels, for a given classifier.
:param classifier: model used to predict labels
:param model_name:
:param compression_factor:
:param x: images
:param y_true: true labels
:param examples: number of images to show
:param random: random selection of training images, or sequential (i.e. first #examples)
:param save: save to disk?
:param plot: show plot?
:return: SubplotAxes
"""
log("Plotting classifier prediction comparison.")
sns.set_style('white')
# Set font
font = {
'fontname': 'Times New Roman Bold',
'fontfamily': 'serif',
'weight': 'bold'
}
# Set y-labels
if names == [] or len(names) != len(classifiers):
names = ['' for i in range(len(classifiers))]
# Set indices
if not indices:
rnd.seed(919)
indices = rnd.sample(range(
len(x)), examples) if random else [i for i in range(examples)]
else:
indices = indices
# Take subsets
x_sample = x[indices]
y_true_sample = y_true[indices]
y_pred_sample = []
# Make predictions
for index, classifier in enumerate(classifiers):
if index == 4:
y_pred_sample.append(classifier.predict(x=x_sample)[1])
else:
y_pred_sample.append(classifier.predict(x=x_sample))
# Get image dimension
image_dim = x.shape[1]
# Plot parameters
row_count = len(classifiers) + 1
col_count = examples
plot_count = row_count * col_count
# Initialize axes
fig, subplot_axes = plt.subplots(row_count,
col_count,
squeeze=True,
sharey='row',
figsize=(15, 18),
constrained_layout=True)
# Set colors
colors = sns.color_palette('pastel', n_colors=len(dat.CLASSES))
# Fill axes
for i in range(plot_count):
row = i // col_count
col = i % col_count
original_image = x_sample[col]
ax = subplot_axes[row][col]
# First row: show original images
if row == 0:
labels = [
label
for got_label, label in zip(y_true_sample[col], dat.CLASSES)
if got_label == 1
]
ax.set_title("Image label: {}".format(labels), fontdict=font)
ax.imshow(original_image)
ax.axis('off')
# Second, third, ... row: show predictions
else:
if row == 1:
ax.set_title("Predictions", fontdict=font)
else:
ax.set_title("", fontdict=font)
# sns.barplot(y=y_pred_sample[col], hue=colors)
ax.bar(x=range(len(dat.CLASSES)),
height=y_pred_sample[row - 1][col],
color=colors)
ax.axhline(y=.5, linestyle='--', color='black')
ax.set_xticks(ticks=range(len(dat.CLASSES)))
ax.set_xticklabels(dat.CLASSES)
ax.set_ylim(0, 1)
if col == 0:
ax.set_ylabel(names[row - 1], fontdict=font)
else:
ax.set_ylabel("")
ax.set_yticklabels([])
# ax.set_aspect(2)
for tick in ax.get_xticklabels():
tick.set_rotation(45)
for i in range(len(x_sample)):
subplot_axes[1][i].set_ylim(0, 1.0)
# General make-up
plt.tight_layout()
# Full model name for file output
full_model_name = "prediction_comparison" + model_name + '_im_dim' + str(
image_dim) + '-comp' + str(int(compression_factor)) + 'loss' + loss
# Build model path
model_path = os.path.join(os.pardir, "models", "classifiers", "plots",
full_model_name + ".png")
# Title
'''plt.suptitle(
"Predictions for <{}> approach (image dim {} - compression {})".format(approach, image_dim, compression_factor),
fontweight='bold')'''
# Show 'n tell
if save: plt.savefig(model_path, dpi=150)
if plot: plt.show()
return fig, subplot_axes
def build_segm_net(model_name: str,
train: bool,
x_tr: np.ndarray,
y_tr: np.ndarray,
x_va: np.ndarray,
y_va: np.ndarray,
mono=True,
compression_factor=32,
epochs=100,
optimizer="adam",
loss='mean_squared_error',
patience=5,
auto=False) -> (Sequential, History):
"""
Build and train segmentation net
:param model_name: name to be used in file output
:param train: (continue and) train the model?
:param x_tr: training images
:param y_tr: training targets
:param x_va: validation images
:param y_va: validation targets
:param compression_factor: (sic)
:param epochs: training duration
:param optimizer: (sic)
:param loss: (sic)
:param patience: epochs to wait without seeing validation improvement
:return: model and its history
"""
# Model parameters
image_dim = x_tr.shape[1]
image_size = image_dim**2 * 3
encoding_dim = int(image_size / compression_factor)
# Set parameters
batch_size = 128
if hlp.LOG_LEVEL == 3:
verbose = 1
elif hlp.LOG_LEVEL == 2:
verbose = 2
else:
verbose = 0
# Full model name for file output
full_model_name = "{}_im_dim{}-comp{}_{}_mono{}".format(
model_name, image_dim, compression_factor, loss, mono)
if auto:
full_model_name += "auto"
# Build model path
model_path = os.path.join(os.pardir, "models", "segmentation",
full_model_name + ".h5")
architecture_path = os.path.join(os.pardir, "models", "segmentation",
"architecture",
full_model_name + "_architecture.png")
# Keep track of history
history_path = os.path.join(os.pardir, "models", "segmentation", "history",
full_model_name + "_history.npy")
# Try loading the model, ...
try:
segnet = load_model(model_path)
log("Found model \'", model_name, "\' locally.", lvl=3)
history = np.load(file=history_path).tolist()
log("Found model \'", model_name, "\' history locally.", lvl=3)
# ... otherwise, create it
except Exception as e:
# log(e)
log("Building segmentation network.")
segnet = convolutional_segm_architecture(image_dim=image_dim,
optimizer=optimizer,
loss=loss,
encoding_dim=encoding_dim,
auto=auto)
# Print model info
log("Network parameters: image dimension {}, image size {}, compression factor {}."
.format(image_dim, image_size, compression_factor),
lvl=3)
# Train the model (either continue training the old model, or train the new one)
log("Training deep convolutional segmentation network.", lvl=2)
# Callbacks
if patience == 0:
patience = epochs
es = EarlyStopping(monitor='val_loss',
patience=patience,
verbose=verbose)
mc = ModelCheckpoint(filepath=model_path,
monitor='val_loss',
verbose=verbose,
save_best_only=True)
tb = TensorBoard(log_dir='/tmp/' + model_name + '_im' +
str(image_dim) + 'comp' +
str(int(compression_factor)) + 'mono' + str(mono))
# Data augmentation to get the most out of our images
datagen = ImageDataGenerator(
rotation_range=15,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
)
datagen.fit(x_tr)
# Train model using data augmentation
history = segnet.fit_generator(datagen.flow(x_tr, y_tr, batch_size=32),
epochs=epochs,
steps_per_epoch=x_tr.shape[0] // 32,
verbose=verbose,
validation_data=(x_va, y_va),
callbacks=[es, mc, tb])
# Save model and history
# autoencoder.save(autoencoder_path) # <- already stored at checkpoint
np.save(file=history_path, arr=history)
# Visual aid
plot_model(segnet,
to_file=architecture_path,
show_layer_names=True,
show_shapes=True)
# Size of encoded representation
log("Compression factor is {}, encoded vector length is {}.".format(
compression_factor, encoding_dim),
lvl=3)
return segnet, history
epochs=epochs,
validation_data=(x_te, {
'classifier': y_te,
'autoencoder': x_te
}),
verbose=1,
callbacks=[es, mc, tb])
np.save(file=history_path, arr=history)
return joint, history
if __name__ == "__main__":
# Let's go
log("JOINT MODEL - AUTOENCODER/CLASSIFIER", title=True)
# Set logging parameters
hlp.LOG_LEVEL = 3
tf.logging.set_verbosity(tf.logging.ERROR)
# Check folders
set_up_model_directory('classifiers')
# Dashboard
image_dim = 96
compression_factor = 24
epochs = 300
patience = 50
batch_size = 128
def plot_model_histories(histories: dict,
image_dim: int,
compression_factor: int,
loss: str,
save=True,
plot=True) -> plt.Axes:
"""
Plot the histories of the model metrics 'loss' and 'accuracy'.
:param histories: histories of trained models
:param image_dim: image dimension
:param compression_factor: (sic)
:param save: save plot to png
:param plot: show plot
:return: plot axes
"""
log("Plotting model metrics.", lvl=2)
# Set style
sns.set_style('whitegrid')
colors = sns.color_palette('pastel', n_colors=len(histories))
# Initialize axes
fig, subplot_axes = plt.subplots(1,
2,
squeeze=False,
sharex='none',
sharey='none',
figsize=(11, 4),
constrained_layout=True)
fig.dpi = 150
# Fill axes
for col in range(2):
train_or_val = 'Training' if col == 0 else 'Validation'
ax = subplot_axes[0][col]
color_counter = 0
for label, history in histories.items():
n_epochs = len(history.history['loss'])
key = 'loss' if col == 0 else 'val_loss'
title = '{} loss'.format(train_or_val)
y_label = 'Loss (mean squared error)' if col == 0 else ""
y_limit = (0, 1)
# Plot label
plot_label = "{}".format(label.capitalize())
# Plot training & validation accuracy values
ax.plot(history.history[key],
label=plot_label,
color=colors[color_counter])
# Add vertical line to indicate early stopping
ax.axvline(x=n_epochs - 1,
linestyle='--',
color=colors[color_counter])
# Set a title, the correct y-label, and the y-limit
ax.set_title(title,
fontdict={
'fontweight': 'semibold',
'family': 'serif'
})
ax.set_ylabel(y_label, fontdict={'family': 'serif'})
ax.set_ylim(y_limit)
color_counter += 1
ax.set_yscale("log")
if col == 0:
ax.legend(loc='best', prop={'family': 'serif'})
ax.set_xlabel('Epoch', fontdict={'family': 'serif'})
# ax.set_xlim(0, 100)
# fig.legend(loc='best', prop={'weight': 'bold', 'family':'serif'})
# Title
'''plt.suptitle(
"Training histories of classifier models (image dim {} - compression {})".format(image_dim, compression_factor),
fontweight='bold')'''
# Build model path
evaluation_label = 'histories_segm_im_dim{}comp{}loss{}'.format(
image_dim, compression_factor, loss)
plot_path = os.path.join(os.pardir, "models", "segmentation", "plots",
evaluation_label + ".png")
# Show 'n tell
if save: fig.savefig(plot_path, dpi=fig.dpi)
if plot: plt.show()
return ax
evaluation_label + ".png")
# Show 'n tell
if save: fig.savefig(plot_path, dpi=fig.dpi)
if plot: plt.show()
return ax
########
# MAIN #
########
if __name__ == '__main__':
# Let's go
log("DATA PREPARATION", title=True)
# Set logging parameters
hlp.LOG_LEVEL = 3
# Get data
data, _ = pipeline(image_dim=48,
class_data=True,
segm_data=True,
mono=True)
x_tr, y_tr = data['x_train'], data['y_train']
x_va, y_va = data['x_val'], data['y_val']
x_te, y_te = data['x_test'], data['y_test']
train_images = data['train_images']