def get_filtered_filenames(voc_root_folder="../data/VOCdevkit/", filter=CLASSES) -> list: """ Get a list of filtered filenames (no extension), based on a filter. :param voc_root_folder: root folder of the PASCAL dataset :return: """ log("Building list of filtered filenames.", lvl=2) # Get annotation files (XML) annotation_folder = os.path.join(voc_root_folder, "VOC2009/Annotations/") annotation_files = os.listdir(annotation_folder) filtered_filenames = [] # Go over all annotation files for a_f in annotation_files: # Parse each file tree = ET.parse(os.path.join(annotation_folder, a_f)) # Check for presence of each of the filter classes check = [ tag.text == filt for tag in tree.iterfind(".//name") for filt in filter ] # If any of them are present, add it to the list (and disregard the extension) if np.any(check): filtered_filenames.append(a_f[:-4]) return filtered_filenames
def evaluate_classifier(classifier: Model, x: np.ndarray, y: np.ndarray, threshold=.5, joint=False) -> dict: """ Evaluate classifier on number of metrics. :param classifier: the model that is to be evaluated. :param x: training (input) data which will lead to predictions :param y: expected outcomes :param threshold: boundary for prediction value to be considered 0 or 1 :return: """ log("Evaluating classifier.", lvl=2) # Store metrics in dictionary metrics = {} # Get probabilities if joint: y_prob = classifier.predict(x)[1] else: y_prob = classifier.predict(x) # ... and extract for predictions y_pred = y_prob super_threshold_indices = y_prob > threshold y_pred[super_threshold_indices] = 1 y_pred[np.invert(super_threshold_indices)] = 0 pp = PrettyPrinter(indent=4) metrics['Hamming loss'] = hamming_loss(y, y_pred) metrics['Exact match ratio'] = accuracy_score(y, y_pred) metrics['Hamming score'] = hamming_score(y, y_pred) # metrics['Precision score (micro)'] = precision_score(y, y_pred, average='micro') # metrics['Precision score (macro)'] = precision_score(y, y_pred, average='macro') # metrics['F1 score (micro)'] = f1_score(y, y_pred, average='micro') # metrics['Label-based accuracy'] = label_based_accuracy(y, y_pred) metrics['Jaccard similarity index'] = jaccard_similarity_score(y, y_pred) metrics['F1 score (macro)'] = f1_score(y, y_pred, average='macro') """In a multi-class classification setup, micro-average is preferable if you suspect there might be class imbalance (i.e you may have many more examples of one class than of other classes) -> We know this is not the case.""" # pp.pprint(metrics) return metrics
def lift(x: np.ndarray) -> np.ndarray: """ Expand input matrix (n image vectors of size image_dim x image_dim x 3). :param x: input matrix :return: expanded input matrix """ if not len(x.shape) > 2: image_size = x.shape[1] image_dim = round(sqrt(image_size / 3)) return x.reshape((len(x), image_dim, image_dim, 3)) else: log("Failed to lift x - already expanded.", lvl=3) return x
def squash(x: np.ndarray) -> np.ndarray: """ Flatten input matrix (n images of size image_dim x image_dim x 3). :param x: input matrix ( :return: flattened input matrix """ if len(x.shape) > 2: image_dim = x.shape[1] image_size = image_dim**2 * 3 return x.reshape((len(x), image_size)) else: log("Failed to squash x - already flattened.", lvl=3) return x
def build_autoencoder(model_name: str, convolutional: bool, train: bool, x_tr: np.ndarray, x_va: np.ndarray, compression_factor: int, epochs=100, optimizer="adam", loss="mean_squared_error", patience=5) -> \ (Sequential, Sequential, float): """ Build an autoencoder. :param model_name: name used in file creation :param x_tr: training images :param x_va: validation images :param compression_factor: what's the size of the bottleneck? :param epochs: number of epochs to train for :param optimizer: optimizer to use in training :param loss: loss function to use in training :param patience: number of epochs without improvement before early stop :return: autoencoder model, encoder model, decoder model, compression factor """ # Model parameters image_dim = x_tr.shape[1] image_size = image_dim**2 * 3 encoding_dim = image_size // compression_factor # Set parameters batch_size = 128 if hlp.LOG_LEVEL == 3: verbose = 1 elif hlp.LOG_LEVEL == 2: verbose = 2 else: verbose = 0 # Full model name for file output full_model_name = model_name + '_im_dim' + str(image_dim) + '-comp' + str( int(compression_factor)) # Build model path model_path = os.path.join(os.pardir, "models", "autoencoders", full_model_name + ".h5") architecture_path = os.path.join(os.pardir, "models", "autoencoders", "architecture", full_model_name + "_architecture.png") # Keep track of history history = [] history_path = os.path.join(os.pardir, "models", "autoencoders", "history", full_model_name + "_history.npy") # Try loading the model, ... try: autoencoder = load_model(model_path) log("Found model \'", model_name, "\' locally.", lvl=3) history = np.load(file=history_path).tolist() log("Found model \'", model_name, "\' history locally.", lvl=3) # ... otherwise, create it except: if convolutional: log("Building deep convolutional autoencoder.", lvl=2) autoencoder = convolutional_auto_architecture( image_dim=image_dim, optimizer=optimizer, loss=loss, compression_factor=compression_factor) else: log("Building linear autoencoder.", lvl=2) autoencoder = linear_auto_architecture( image_size=image_size, optimizer=optimizer, loss=loss, compression_factor=compression_factor) # Print model info log("Network parameters: image dimension {}, image size {}, encoding dimension {}, compression factor {}." .format(image_dim, image_size, encoding_dim, compression_factor), lvl=3) # Train the model (either continue training the old model, or train the new one) if train: log("Training autoencoder.", lvl=2) # Flatten image data for linear model if not convolutional: # Flatten x_tr = squash(x_tr) x_va = squash(x_va) # Training parameters es = EarlyStopping(monitor='val_loss', patience=patience, verbose=verbose) mc = ModelCheckpoint(filepath=model_path, monitor='val_loss', verbose=verbose, save_best_only=True) tb = TensorBoard(log_dir='/tmp/' + model_name + '_im' + str(image_dim) + 'comp' + str(int(compression_factor))) # Data augmentation to get the most out of our images '''datagen = ImageDataGenerator( rotation_range=15, width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True, ) datagen.fit(x_tr)''' # Train model history.append( autoencoder.fit(x_tr, x_tr, epochs=epochs, batch_size=batch_size, verbose=verbose, validation_data=(x_va, x_va), callbacks=[es, mc, tb])) # Save model and history # autoencoder.save(autoencoder_path) # <- already stored at checkpoint np.save(file=history_path, arr=history) # Visual aid plot_model(autoencoder, to_file=architecture_path, show_layer_names=True, show_shapes=True) # Get intermediate output at encoded layer if convolutional: encoder = Model(inputs=autoencoder.input, outputs=autoencoder.get_layer(name='encoder').output) else: encoder = Model(inputs=autoencoder.input, outputs=autoencoder.get_layer(index=0).output) # See effect of encoded representations on output (eigenfaces?) '''decoder = Model(inputs=Input(shape=(encoding_dim,)), outputs=autoencoder.output) plot_model(autoencoder, to_file=os.path.join(os.pardir, "models", "decoder.png"), show_layer_names=True, show_shapes=True)''' # Size of encoded representation log("Compression factor is {}, encoded vector length is {}.".format( compression_factor, encoding_dim), lvl=3) return autoencoder, encoder, history
# Show 'n tell if save: fig.savefig(plot_path, dpi=fig.dpi) if plot: plt.show() return ax ######## # MAIN # ######## if __name__ == "__main__": """Main function.""" # Let's go log("AUTOENCODERS", title=True) # Set parameters hlp.LOG_LEVEL = 3 tf.logging.set_verbosity(tf.logging.ERROR) # Check folders set_up_model_directory('autoencoders') # Model approaches linear, linear_train = True, True convolutional, convolutional_train = True, True # Loop over these dimensions '''image_dims = (48,) compression_factors = (48,)
def build_unet_segmentation_network(model_name: str, x_tr: np.ndarray, y_tr: np.ndarray, x_va: np.ndarray, y_va: np.ndarray, optimizer='adam', loss='mean_squared_error', epochs=200, patience=25, verbose=1) -> (Model, History): """ Build and train U-net inspired segmentation network. :param x_tr: training images :param y_tr: training segmentations :param x_va: validation images :param y_va: validation segmentations :param optimizer: (sic) :param loss: (sic) :param epochs: (sic) :param patience: (sic) :param verbose: (sic) :return: U-net model and its training history """ # Set parameters image_dim = x_tr.shape[1] image_size = image_dim**2 * 3 input_shape = (image_dim, image_dim, 3) # Build model path model_name = "{}_im{}".format(model_name, image_dim) model_path = os.path.join(os.pardir, "models", "segmentation", model_name + ".h5") architecture_path = os.path.join(os.pardir, "models", "segmentation", "architecture", model_name + "_architecture.png") history_path = os.path.join(os.pardir, "models", "segmentation", "history", model_name + "_history.npy") # Try loading the model, ... try: unet = load_model(model_path) log("Found model \'", model_name, "\' locally.", lvl=3) history = np.load(file=history_path).tolist() log("Found model \'", model_name, "\' history locally.", lvl=3) # ... otherwise, create it except Exception as e: log("Exception:", e, lvl=3) # Build U-Net model image_input = Input(shape=input_shape) c1 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(image_input) c1 = Dropout(0.1)(c1) c1 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c1) p1 = MaxPooling2D((2, 2))(c1) c2 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p1) c2 = Dropout(0.1)(c2) c2 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c2) p2 = MaxPooling2D((2, 2))(c2) c3 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p2) c3 = Dropout(0.2)(c3) c3 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c3) p3 = MaxPooling2D((2, 2))(c3) c4 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p3) c4 = Dropout(0.2)(c4) c4 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c4) p4 = MaxPooling2D(pool_size=(2, 2))(c4) c5 = Conv2D(256, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p4) c5 = Dropout(0.3)(c5) c5 = Conv2D(256, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c5) u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5) u6 = concatenate([u6, c4]) c6 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u6) c6 = Dropout(0.2)(c6) c6 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c6) u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6) u7 = concatenate([u7, c3]) c7 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u7) c7 = Dropout(0.2)(c7) c7 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c7) u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7) u8 = concatenate([u8, c2]) c8 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u8) c8 = Dropout(0.1)(c8) c8 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c8) u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8) u9 = concatenate([u9, c1], axis=3) c9 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u9) c9 = Dropout(0.1)(c9) c9 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c9) outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9) unet = Model(inputs=[image_input], outputs=[outputs]) unet.compile(optimizer=optimizer, loss=loss) unet.summary() plot_model(unet, to_file=architecture_path, show_shapes=True) # Callbacks if patience == 0: patience = epochs es = EarlyStopping(monitor='val_loss', patience=patience, verbose=verbose) mc = ModelCheckpoint(filepath=model_path, monitor='val_loss', verbose=verbose, save_best_only=True) tb = TensorBoard(log_dir="/tmp/segm_unet_im{}".format(image_dim)) # Data augmentation to get the most out of our images datagen = ImageDataGenerator( rotation_range=15, width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True, ) datagen.fit(x_tr) # Train model using data augmentation history = unet.fit_generator(datagen.flow(x_tr, y_tr, batch_size=128), epochs=epochs, steps_per_epoch=x_tr.shape[0] // 128, verbose=verbose, validation_data=(x_va, y_va), callbacks=[es, mc, tb]) # Save model and history np.save(file=history_path, arr=history) plot_model(unet, to_file=architecture_path, show_shapes=True) return unet, history
steps_per_epoch=x_tr.shape[0] // 128, verbose=verbose, validation_data=(x_va, y_va), callbacks=[es, mc, tb]) # Save model and history np.save(file=history_path, arr=history) plot_model(unet, to_file=architecture_path, show_shapes=True) return unet, history if __name__ == "__main__": # Let's go log("U-NET SEGMENTATION NETWORK", title=True) # Set logging parameters hlp.LOG_LEVEL = 3 tf.logging.set_verbosity(tf.logging.ERROR) # Check folders set_up_model_directory('segmentation') # Dashboard image_dim = 96 compression_factor = 24 epochs = 500 patience = 0 loss = 'mean_squared_error'
def joint_model(model_name: str, image_dim: int, compression_factor: int, verbose=1) -> (Model, History): """ Build and train a joint model: autoencoder plus classification training. :param model_name: name for file output :param image_dim: dimension of input image :param compression_factor: (sic) :param verbose: verbosity during training. :return: joint model, plus its history """ # Full model name for file output full_model_name = "{}_im{}comp{}".format(model_name, image_dim, compression_factor) # Build model paths model_path = os.path.join(os.pardir, "models", "classifiers", full_model_name + ".h5") architecture_path = os.path.join(os.pardir, "models", "classifiers", "architecture", full_model_name + "_architecture.png") history_path = os.path.join(os.pardir, "models", "classifiers", "history", full_model_name + "_history.npy") # Try loading the model, ... try: joint = load_model(model_path) log("Found model \'", model_name, "\' locally.", lvl=3) history = np.load(file=history_path) log("Found model \'", model_name, "\' history locally.", lvl=3) # ... otherwise, create it except: # Model parameters input_shape = (image_dim, image_dim, 3) image_size = np.prod(input_shape) encoding_dim = image_size // compression_factor # Build model base (encoder) image_input = Input(shape=input_shape) conv_1 = Conv2D(image_dim, (3, 3), padding='same', activation='relu', kernel_initializer='random_uniform', bias_initializer='zeros')(image_input) batch_1 = BatchNormalization()(conv_1) max_1 = MaxPooling2D((2, 2), padding='same')(batch_1) conv_2 = Conv2D(image_dim // (2 * compression_factor), (3, 3), padding='same', activation='relu', kernel_initializer='random_uniform', bias_initializer='zeros')(max_1) batch_2 = BatchNormalization()(conv_2) encoded = MaxPooling2D((2, 2), padding='same')(batch_2) # Build autoencoder conv_3 = Conv2D(image_dim // (2 * compression_factor), (3, 3), padding='same', activation='relu', kernel_initializer='random_uniform', bias_initializer='zeros')(encoded) batch_3 = BatchNormalization()(conv_3) up_1 = UpSampling2D((2, 2))(batch_3) conv_4 = Conv2D(image_dim, (3, 3), padding='same', activation='relu', kernel_initializer='random_uniform', bias_initializer='zeros')(up_1) batch_4 = BatchNormalization()(conv_4) up_2 = UpSampling2D((2, 2))(batch_4) autoencoder = Conv2D(3, (10, 10), padding='same', activation='sigmoid', kernel_initializer='random_uniform', bias_initializer='zeros', name='autoencoder')(up_2) # Build classifier flatten = Flatten()(encoded) dense_1 = Dense(encoding_dim, activation='relu')(flatten) drop_1 = Dropout(.5)(dense_1) classifier = Dense(5, activation='sigmoid', name='classifier')(drop_1) # Build joint model joint = Model(inputs=image_input, outputs=[autoencoder, classifier]) # Save model architecture visualization plot_model(joint, to_file=architecture_path) joint.compile(loss={ 'classifier': 'binary_crossentropy', 'autoencoder': 'mean_squared_error' }, optimizer='adam', metrics={'classifier': 'accuracy'}) # Callbacks es = EarlyStopping(monitor='val_loss', patience=patience, verbose=verbose) mc = ModelCheckpoint(filepath=model_path, monitor='val_loss', verbose=verbose, save_best_only=True) tb = TensorBoard(log_dir="/tmp/{}_im{}comp{}".format( model_name, image_dim, compression_factor)) history = joint.fit(x=x_tr, y={ 'classifier': y_tr, 'autoencoder': x_tr }, batch_size=batch_size, epochs=epochs, validation_data=(x_te, { 'classifier': y_te, 'autoencoder': x_te }), verbose=1, callbacks=[es, mc, tb]) np.save(file=history_path, arr=history) return joint, history
def plot_model_history(history: History, image_dim: int, model_type="segmentation", save=True, plot=True) -> plt.Axes: """ Plot the histories of the model metrics 'loss' and 'val loss'. :param histories: histories of trained models :param image_dim: image dimension :param compression_factor: (sic) :param save: save plot to png :param plot: show plot :return: plot axes """ log("Plotting model metrics.", lvl=2) # Set font font = { 'fontname': 'Times New Roman Bold', 'fontfamily': 'serif', 'weight': 'bold' } # Set style sns.set_style('whitegrid') colors = sns.color_palette('pastel', 2) # Initialize axes fig = plt.figure(figsize=(8, 5), dpi=150) ax = plt.axes() # Fill axes n_epochs = len(history.history['loss']) y_label = 'Loss (mean squared error)' # Plot training & validation accuracy values ax.plot(history.history["loss"], label="Training", color=colors[0]) ax.plot(history.history["val_loss"], label="Validation", color=colors[1]) # Add vertical line to indicate early stopping ax.axvline(x=n_epochs - 1, linestyle='--', color=colors[0]) # Set a title, the correct y-label, and the y-limit ax.set_ylabel(y_label, fontdict={'fontname': 'Times New Roman'}) ax.set_ylim(0.1, 1.2) ax.set_yscale("log") ax.set_xlabel('Epoch', fontdict={'fontname': 'Times New Roman'}) ax.set_xlim(0, 500) ax.legend(loc='best', prop={'family': 'Serif'}) # Build model path evaluation_label = 'histories_segm_im_dim{}'.format(image_dim) plot_path = os.path.join(os.pardir, "models", model_type, "plots", evaluation_label + ".png") # Show 'n tell if save: fig.savefig(plot_path, dpi=fig.dpi) if plot: plt.show() return fig
def build_classifier(image_dim: int, compression_factor: int, x_tr: np.ndarray, y_tr: np.ndarray, x_va: np.ndarray, y_va: np.ndarray, from_scratch=False, all_trainable=False, loss='binary_crossentropy', epochs=200, patience=25) -> (Sequential, History): """ Build a deep convolutional classifier network, either from scratch, or from a pretrained autoencoder. :param image_dim: dimension of training and validation images :param compression_factor: size of bottleneck in network :param x_tr: training images :param y_tr: training labels :param x_va: validation images :param y_va: validation labels :param from_scratch: do we start fresh, or from earlier weights? :param all_trainable: are all weights trainable, or do we freeze the encoder part? :param epochs: number of epochs to train for :param patience: number of epochs to wait before we stop training, when seeing no improvement in validation loss :return: classifier model, training history """ # Model parameters image_size = image_dim**2 * 3 encoding_dim = image_size // compression_factor # Set verbosity if hlp.LOG_LEVEL == 3: verbose = 1 elif hlp.LOG_LEVEL == 2: verbose = 2 else: verbose = 0 # Start building new Sequential model classifier = Sequential() # Use previously trained model (we could just load weights, too). if not from_scratch: # Build encoder _, encoder, _ = build_autoencoder( model_name='conv_auto', convolutional=True, train=False, x_tr=x_tr, x_va=x_va, compression_factor=compression_factor) # Freeze, dirtbag (or not, I'm not dirty Harry) conv_layers = [1, 2, 4, 5] for i in conv_layers: encoder.get_layer(index=i).trainable = all_trainable # Add encoding part of autoencoder to our classifier classifier.add(encoder) # ... unless we just use the architecture else: input_shape = (image_dim, image_dim, 3) image_size = np.prod(input_shape) # Build model classifier = Sequential() classifier.add( Conv2D(image_dim, (3, 3), padding='same', activation='relu', input_shape=input_shape, kernel_initializer='random_uniform', bias_initializer='zeros')) classifier.add(BatchNormalization()) classifier.add(MaxPooling2D((2, 2), padding='same')) classifier.add( Conv2D(image_dim // compression_factor, (3, 3), padding='same', activation='relu', kernel_initializer='random_uniform', bias_initializer='zeros')) classifier.add(BatchNormalization()) classifier.add(MaxPooling2D((2, 2), padding='same', name='encoder')) # Add classification layers classifier.add(Flatten()) classifier.add(Dense(encoding_dim)) classifier.add(Activation('relu')) classifier.add(Dropout(0.5)) classifier.add(Dense( 5, activation='sigmoid')) # <-- multilabel (for multiclass: softmax) classifier.compile(optimizer='adam', loss=loss, metrics=['accuracy']) # Plot model full_classifier_name = "{}_im_dim{}-comp{}_full{}_scratch{}_loss{}".format( classifier_name, image_dim, compression_factor, all_trainable, from_scratch, loss) architecture_path = os.path.join(os.pardir, "models", "classifiers", "architecture", full_classifier_name + ".png") plot_model(classifier, to_file=architecture_path, show_layer_names=True, show_shapes=True) # Data augmentation to get the most out of our images datagen = ImageDataGenerator( rotation_range=15, width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True, ) datagen.fit(x_tr) # Callbacks if patience == 0: patience = epochs es = EarlyStopping(monitor='val_loss', patience=patience, verbose=verbose) mc = ModelCheckpoint(filepath=classifier_path, monitor='val_loss', verbose=verbose, save_best_only=True) tb = TensorBoard( log_dir='/tmp/{}_class_im{}comp{}_full{}_scratch{}'.format( classifier_name, image_dim, compression_factor, all_trainable, from_scratch)) # Print model info log("Network parameters: image dimension {}, image size {}, compression factor {}." .format(image_dim, image_size, compression_factor), lvl=3) # Train model using data augmentation history = classifier.fit_generator(datagen.flow(x_tr, y_tr, batch_size=32), epochs=epochs, steps_per_epoch=x_tr.shape[0] // 32, verbose=verbose, validation_data=(x_va, y_va), callbacks=[es, mc, tb]) '''history = classifier.fit(x=x_tr, y=y_tr, batch_size=32, epochs=epochs, verbose=verbose, callbacks=[es, mc, tb], validation_data=(x_va, y_va))''' # Save model and history classifier.save(classifier_path) return classifier, history
# Show 'n tell if save: plt.savefig(model_path, dpi=150) if plot: plt.show() return fig, subplot_axes ######## # MAIN # ######## if __name__ == "__main__": # Let's go log("CLASSIFIERS", title=True) # Set logging parameters hlp.LOG_LEVEL = 3 tf.logging.set_verbosity(tf.logging.ERROR) # Check folders set_up_model_directory('classifiers') # Dashboard image_dim = 96 compression_factor = 24 loss = 'binary_crossentropy' epochs = 300 patience = 50
def plot_model_histories(histories: dict, image_dim: int, compression_factor: int, save=True, plot=True) -> plt.Axes: """ Plot the histories of the model metrics 'loss' and 'accuracy'. :param histories: histories of trained models :param image_dim: image dimension :param compression_factor: (sic) :param save: save plot to png :param plot: show plot :return: plot axes """ log("Plotting model metrics.", lvl=2) # Set style sns.set_style('whitegrid') colors = sns.color_palette('pastel', n_colors=3) # Initialize axes fig, subplot_axes = plt.subplots(2, 2, squeeze=False, sharex='none', sharey='none', figsize=(11, 10), constrained_layout=True) # Fill axes for col in range(2): train_or_val = 'Training' if col == 0 else 'Validation' for row in range(2): ax = subplot_axes[row][col] color_counter = 0 for label, history in histories.items(): n_epochs = len(history.history['loss']) if row == 0: key = 'acc' if col == 0 else 'val_acc' title = '{} accuracy'.format(train_or_val) y_label = 'Accuracy' y_limit = (.7, .9) else: key = 'loss' if col == 0 else 'val_loss' title = '{} loss'.format(train_or_val) y_label = 'Loss (binary cross entropy)' y_limit = (.2, .7) # Plot training & validation accuracy values ax.plot(history.history[key], label="Model: {}".format(label), color=colors[color_counter]) # Add vertical line to indicate early stopping ax.axvline(x=n_epochs, linestyle='--', color=colors[color_counter]) # Set a title, the correct y-label, and the y-limit ax.set_title(title, fontdict={'fontweight': 'semibold'}) ax.set_ylabel(y_label) ax.set_ylim(y_limit) color_counter += 1 if row == 1: ax.set_xlabel('Epoch') ax.set_xlim(0, 200) ax.legend(loc='best') # Title plt.suptitle( "Training histories of classifier models (image dim {} - compression {})" .format(image_dim, compression_factor), fontweight='bold') # Build model path evaluation_label = 'histories_im_dim{}comp{}'.format( image_dim, compression_factor) plot_path = os.path.join(os.pardir, "models", "autoencoders", "plots", evaluation_label + ".png") # Show 'n tell if save: fig.savefig(plot_path, dpi=fig.dpi) if plot: plt.show() return ax
def pipeline(image_dim=214, filter=CLASSES, class_data=True, segm_data=False, mono=True, voc_root_folder="../data/VOCdevkit/") -> (dict, dict): """ Pipeline to extract training and validation datasets from full dataset. :param image_dim: images will be resized to image_dim x image_dim pixels :param filter: classes to take into account (out of total of 20) :param class_data: load classification data :param segm_data: load segmentation data :param voc_root_folder: root folder of PASCAL dataset :return: data dicts """ if not (class_data or segm_data): raise Exception( "No data requested! Choose classification data, segmentation data, or both." ) log("Preparing data: image dimension {imdim}x{imdim}.".format( imdim=image_dim), lvl=1) # Make folders, should they not exist yet make_folders("data", "scripts", os.path.join('data', 'image_dim_' + str(image_dim))) # Try to load data sets locally data_path = os.path.join(os.pardir, 'data', 'image_dim_' + str(image_dim), "data_" + str(image_dim) + ".npy") data_segm_path = os.path.join( os.pardir, 'data', 'image_dim_' + str(image_dim), "data_segm_" + str(image_dim) + ('mono' if mono else '') + ".npy") # Build (x,y) for TRAIN/TRAINVAL/VAL (classification) log("Building filtered dataset.", lvl=2) classes_folder = os.path.join(voc_root_folder, "VOC2009", "ImageSets", "Main") classes_files = os.listdir(classes_folder) # Only take training and validation images that pass filter train_files = [ os.path.join(classes_folder, c_f) for filt in filter for c_f in classes_files if filt in c_f and '_train.txt' in c_f ] val_files = [ os.path.join(classes_folder, c_f) for filt in filter for c_f in classes_files if filt in c_f and '_val.txt' in c_f ] # If classification data is requested if class_data: # Try to load the data from disk log("Getting classification data.", lvl=3) try: data = np.load(file=data_path).item() log("Loaded classification data.", lvl=2) train_images = data['train_images'] val_images = data['val_images'] except (NameError, FileNotFoundError): log("Failed to load one of the {}-sized datasets. Rebuilding.". format(image_dim), lvl=1) x_train, y_train, train_images = build_classification_dataset( list_of_files=train_files, image_dim=image_dim, filter=filter, voc_root_folder=voc_root_folder) log("Extracted {} training images from {} classes.".format( x_train.shape[0], y_train.shape[1]), lvl=3) x_val, y_val, val_images = build_classification_dataset( list_of_files=val_files, image_dim=image_dim, filter=filter, voc_root_folder=voc_root_folder) log("Extracted {} validation images from {} classes.".format( x_val.shape[0], y_train.shape[1]), lvl=3) # Keep random sample as test test_indices = rnd.sample(range(len(x_val)), k=len(x_val) // 2) val_indices = [ i for i in range(len(x_val)) if i not in test_indices ] x_test = x_val[test_indices] y_test = y_val[test_indices] x_val = x_val[val_indices] y_val = y_val[val_indices] data = { 'x_train': x_train, 'y_train': y_train, 'x_val': x_val, 'y_val': y_val, 'x_test': x_test, 'y_test': y_test, 'train_images': train_images, 'val_images': val_images } # Save locally, because this takes a while log("Storing classification data.", lvl=3) np.save(data_path, data) else: data = {} # If segmentation data is requested if segm_data: # Try to load segmentation data from disk log("Getting segmentation data.", lvl=3) try: data_segm = np.load(file=data_segm_path).item() log("Loaded segmentation data.", lvl=2) # ... else build datasets except (FileNotFoundError, NameError): if not class_data: raise Exception( "Failed to load segmentation data. Set 'class_data' to true to build datasets!" ) log("Failed to load segmentation data. Rebuilding.", lvl=3) data_segm = build_segmentation_dataset( train_list=train_images, val_list=val_images, image_dim=image_dim, voc_root_folder=voc_root_folder, mono=mono) log("Storing segmentation data.", lvl=3) np.save(data_segm_path, data_segm) else: data_segm = {} # Return data return data, data_segm
evaluation_label + ".png") # Show 'n tell if save: fig.savefig(plot_path, dpi=fig.dpi) if plot: plt.show() return ax ######## # MAIN # ######## if __name__ == "__main__": # Let's go log("SEGMENTATION", title=True) # Set logging parameters hlp.LOG_LEVEL = 3 tf.logging.set_verbosity(tf.logging.ERROR) # Check folders set_up_model_directory('segmentation') # Dashboard image_dim = 96 compression_factor = 24 train = False epochs = 500 patience = 50 loss = 'mean_squared_error'
def plot_classifier_prediction_comparison(classifiers: list, names: list, model_name: str, compression_factor: float, loss: str, x: np.ndarray, y_true: np.ndarray, examples=5, random=True, save=True, plot=True, indices=None): """ Show some images, their true class labels, and the predicted class labels, for a given classifier. :param classifier: model used to predict labels :param model_name: :param compression_factor: :param x: images :param y_true: true labels :param examples: number of images to show :param random: random selection of training images, or sequential (i.e. first #examples) :param save: save to disk? :param plot: show plot? :return: SubplotAxes """ log("Plotting classifier prediction comparison.") sns.set_style('white') # Set font font = { 'fontname': 'Times New Roman Bold', 'fontfamily': 'serif', 'weight': 'bold' } # Set y-labels if names == [] or len(names) != len(classifiers): names = ['' for i in range(len(classifiers))] # Set indices if not indices: rnd.seed(919) indices = rnd.sample(range( len(x)), examples) if random else [i for i in range(examples)] else: indices = indices # Take subsets x_sample = x[indices] y_true_sample = y_true[indices] y_pred_sample = [] # Make predictions for index, classifier in enumerate(classifiers): if index == 4: y_pred_sample.append(classifier.predict(x=x_sample)[1]) else: y_pred_sample.append(classifier.predict(x=x_sample)) # Get image dimension image_dim = x.shape[1] # Plot parameters row_count = len(classifiers) + 1 col_count = examples plot_count = row_count * col_count # Initialize axes fig, subplot_axes = plt.subplots(row_count, col_count, squeeze=True, sharey='row', figsize=(15, 18), constrained_layout=True) # Set colors colors = sns.color_palette('pastel', n_colors=len(dat.CLASSES)) # Fill axes for i in range(plot_count): row = i // col_count col = i % col_count original_image = x_sample[col] ax = subplot_axes[row][col] # First row: show original images if row == 0: labels = [ label for got_label, label in zip(y_true_sample[col], dat.CLASSES) if got_label == 1 ] ax.set_title("Image label: {}".format(labels), fontdict=font) ax.imshow(original_image) ax.axis('off') # Second, third, ... row: show predictions else: if row == 1: ax.set_title("Predictions", fontdict=font) else: ax.set_title("", fontdict=font) # sns.barplot(y=y_pred_sample[col], hue=colors) ax.bar(x=range(len(dat.CLASSES)), height=y_pred_sample[row - 1][col], color=colors) ax.axhline(y=.5, linestyle='--', color='black') ax.set_xticks(ticks=range(len(dat.CLASSES))) ax.set_xticklabels(dat.CLASSES) ax.set_ylim(0, 1) if col == 0: ax.set_ylabel(names[row - 1], fontdict=font) else: ax.set_ylabel("") ax.set_yticklabels([]) # ax.set_aspect(2) for tick in ax.get_xticklabels(): tick.set_rotation(45) for i in range(len(x_sample)): subplot_axes[1][i].set_ylim(0, 1.0) # General make-up plt.tight_layout() # Full model name for file output full_model_name = "prediction_comparison" + model_name + '_im_dim' + str( image_dim) + '-comp' + str(int(compression_factor)) + 'loss' + loss # Build model path model_path = os.path.join(os.pardir, "models", "classifiers", "plots", full_model_name + ".png") # Title '''plt.suptitle( "Predictions for <{}> approach (image dim {} - compression {})".format(approach, image_dim, compression_factor), fontweight='bold')''' # Show 'n tell if save: plt.savefig(model_path, dpi=150) if plot: plt.show() return fig, subplot_axes
def build_segm_net(model_name: str, train: bool, x_tr: np.ndarray, y_tr: np.ndarray, x_va: np.ndarray, y_va: np.ndarray, mono=True, compression_factor=32, epochs=100, optimizer="adam", loss='mean_squared_error', patience=5, auto=False) -> (Sequential, History): """ Build and train segmentation net :param model_name: name to be used in file output :param train: (continue and) train the model? :param x_tr: training images :param y_tr: training targets :param x_va: validation images :param y_va: validation targets :param compression_factor: (sic) :param epochs: training duration :param optimizer: (sic) :param loss: (sic) :param patience: epochs to wait without seeing validation improvement :return: model and its history """ # Model parameters image_dim = x_tr.shape[1] image_size = image_dim**2 * 3 encoding_dim = int(image_size / compression_factor) # Set parameters batch_size = 128 if hlp.LOG_LEVEL == 3: verbose = 1 elif hlp.LOG_LEVEL == 2: verbose = 2 else: verbose = 0 # Full model name for file output full_model_name = "{}_im_dim{}-comp{}_{}_mono{}".format( model_name, image_dim, compression_factor, loss, mono) if auto: full_model_name += "auto" # Build model path model_path = os.path.join(os.pardir, "models", "segmentation", full_model_name + ".h5") architecture_path = os.path.join(os.pardir, "models", "segmentation", "architecture", full_model_name + "_architecture.png") # Keep track of history history_path = os.path.join(os.pardir, "models", "segmentation", "history", full_model_name + "_history.npy") # Try loading the model, ... try: segnet = load_model(model_path) log("Found model \'", model_name, "\' locally.", lvl=3) history = np.load(file=history_path).tolist() log("Found model \'", model_name, "\' history locally.", lvl=3) # ... otherwise, create it except Exception as e: # log(e) log("Building segmentation network.") segnet = convolutional_segm_architecture(image_dim=image_dim, optimizer=optimizer, loss=loss, encoding_dim=encoding_dim, auto=auto) # Print model info log("Network parameters: image dimension {}, image size {}, compression factor {}." .format(image_dim, image_size, compression_factor), lvl=3) # Train the model (either continue training the old model, or train the new one) log("Training deep convolutional segmentation network.", lvl=2) # Callbacks if patience == 0: patience = epochs es = EarlyStopping(monitor='val_loss', patience=patience, verbose=verbose) mc = ModelCheckpoint(filepath=model_path, monitor='val_loss', verbose=verbose, save_best_only=True) tb = TensorBoard(log_dir='/tmp/' + model_name + '_im' + str(image_dim) + 'comp' + str(int(compression_factor)) + 'mono' + str(mono)) # Data augmentation to get the most out of our images datagen = ImageDataGenerator( rotation_range=15, width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True, ) datagen.fit(x_tr) # Train model using data augmentation history = segnet.fit_generator(datagen.flow(x_tr, y_tr, batch_size=32), epochs=epochs, steps_per_epoch=x_tr.shape[0] // 32, verbose=verbose, validation_data=(x_va, y_va), callbacks=[es, mc, tb]) # Save model and history # autoencoder.save(autoencoder_path) # <- already stored at checkpoint np.save(file=history_path, arr=history) # Visual aid plot_model(segnet, to_file=architecture_path, show_layer_names=True, show_shapes=True) # Size of encoded representation log("Compression factor is {}, encoded vector length is {}.".format( compression_factor, encoding_dim), lvl=3) return segnet, history
epochs=epochs, validation_data=(x_te, { 'classifier': y_te, 'autoencoder': x_te }), verbose=1, callbacks=[es, mc, tb]) np.save(file=history_path, arr=history) return joint, history if __name__ == "__main__": # Let's go log("JOINT MODEL - AUTOENCODER/CLASSIFIER", title=True) # Set logging parameters hlp.LOG_LEVEL = 3 tf.logging.set_verbosity(tf.logging.ERROR) # Check folders set_up_model_directory('classifiers') # Dashboard image_dim = 96 compression_factor = 24 epochs = 300 patience = 50 batch_size = 128
def plot_model_histories(histories: dict, image_dim: int, compression_factor: int, loss: str, save=True, plot=True) -> plt.Axes: """ Plot the histories of the model metrics 'loss' and 'accuracy'. :param histories: histories of trained models :param image_dim: image dimension :param compression_factor: (sic) :param save: save plot to png :param plot: show plot :return: plot axes """ log("Plotting model metrics.", lvl=2) # Set style sns.set_style('whitegrid') colors = sns.color_palette('pastel', n_colors=len(histories)) # Initialize axes fig, subplot_axes = plt.subplots(1, 2, squeeze=False, sharex='none', sharey='none', figsize=(11, 4), constrained_layout=True) fig.dpi = 150 # Fill axes for col in range(2): train_or_val = 'Training' if col == 0 else 'Validation' ax = subplot_axes[0][col] color_counter = 0 for label, history in histories.items(): n_epochs = len(history.history['loss']) key = 'loss' if col == 0 else 'val_loss' title = '{} loss'.format(train_or_val) y_label = 'Loss (mean squared error)' if col == 0 else "" y_limit = (0, 1) # Plot label plot_label = "{}".format(label.capitalize()) # Plot training & validation accuracy values ax.plot(history.history[key], label=plot_label, color=colors[color_counter]) # Add vertical line to indicate early stopping ax.axvline(x=n_epochs - 1, linestyle='--', color=colors[color_counter]) # Set a title, the correct y-label, and the y-limit ax.set_title(title, fontdict={ 'fontweight': 'semibold', 'family': 'serif' }) ax.set_ylabel(y_label, fontdict={'family': 'serif'}) ax.set_ylim(y_limit) color_counter += 1 ax.set_yscale("log") if col == 0: ax.legend(loc='best', prop={'family': 'serif'}) ax.set_xlabel('Epoch', fontdict={'family': 'serif'}) # ax.set_xlim(0, 100) # fig.legend(loc='best', prop={'weight': 'bold', 'family':'serif'}) # Title '''plt.suptitle( "Training histories of classifier models (image dim {} - compression {})".format(image_dim, compression_factor), fontweight='bold')''' # Build model path evaluation_label = 'histories_segm_im_dim{}comp{}loss{}'.format( image_dim, compression_factor, loss) plot_path = os.path.join(os.pardir, "models", "segmentation", "plots", evaluation_label + ".png") # Show 'n tell if save: fig.savefig(plot_path, dpi=fig.dpi) if plot: plt.show() return ax
evaluation_label + ".png") # Show 'n tell if save: fig.savefig(plot_path, dpi=fig.dpi) if plot: plt.show() return ax ######## # MAIN # ######## if __name__ == '__main__': # Let's go log("DATA PREPARATION", title=True) # Set logging parameters hlp.LOG_LEVEL = 3 # Get data data, _ = pipeline(image_dim=48, class_data=True, segm_data=True, mono=True) x_tr, y_tr = data['x_train'], data['y_train'] x_va, y_va = data['x_val'], data['y_val'] x_te, y_te = data['x_test'], data['y_test'] train_images = data['train_images']