def main():
"""Main function, estimates the performance of a model on the training,
validation und test datasets."""
# handle arguments from command line
parser = argparse.ArgumentParser()
parser.add_argument("identifier", help="Identifier of the experiment of " \
"which to load the weights.")
parser.add_argument("--images", required=True,
help="Filepath to the 'faces/' subdirectory in the " \
"'Labeled Faces in the Wild grayscaled and " \
"cropped' dataset.")
args = parser.parse_args()
validate_identifier(args.identifier, must_exist=True)
if not os.path.isdir(args.images):
raise Exception("The provided filepath to the dataset seems to not " \
"exist.")
# Load:
# 1. Validation set,
# 2. Training set,
# 3. Test set
# We will test on each one of them.
# Results from training and validation set are already known, but will be
# shown in more detail here.
# Additionally, we need to load train and val datasets to make sure that
# no image contained in them is contained in the test set.
print("Loading validation set...")
pairs_val = get_image_pairs(args.images, VALIDATION_COUNT_EXAMPLES,
pairs_of_same_imgs=False, ignore_order=True,
exclude_images=list(), seed=SEED,
verbose=False)
assert len(pairs_val) == VALIDATION_COUNT_EXAMPLES
X_val, y_val = image_pairs_to_xy(pairs_val, height=INPUT_HEIGHT,
width=INPUT_WIDTH)
print("Loading training set...")
pairs_train = get_image_pairs(args.images, TRAIN_COUNT_EXAMPLES,
pairs_of_same_imgs=False, ignore_order=True,
exclude_images=pairs_val, seed=SEED,
verbose=False)
assert len(pairs_train) == TRAIN_COUNT_EXAMPLES
X_train, y_train = image_pairs_to_xy(pairs_train, height=INPUT_HEIGHT,
width=INPUT_WIDTH)
print("Loading test set...")
pairs_test = get_image_pairs(args.images, TEST_COUNT_EXAMPLES,
pairs_of_same_imgs=False, ignore_order=True,
exclude_images=pairs_val+pairs_train,
seed=SEED, verbose=False)
assert len(pairs_test) == TEST_COUNT_EXAMPLES
X_test, y_test = image_pairs_to_xy(pairs_test, height=INPUT_HEIGHT,
width=INPUT_WIDTH)
print("")
# Plot dataset skew
print("Plotting dataset skew. (Only for pairs of images showing the " \
"same person.)")
print("More unequal bars mean that the dataset is more skewed (towards " \
"very few people).")
print("Close the chart to continue.")
plot_dataset_skew(
pairs_train, pairs_val, pairs_test,
only_y_same=True,
show_plot_windows=SHOW_PLOT_WINDOWS
)
print("Creating model...")
model, _ = create_model()
(success, last_epoch) = load_weights(model, SAVE_WEIGHTS_DIR,
args.identifier)
if not success:
raise Exception("Could not successfully load model weights")
print("Loaded model weights of epoch '%s'" % (str(last_epoch)))
# If we just do one run over a set (training/val/test) we will not augment
# the images (ia_noop). If we do multiple runs, we will augment images in
# each run (ia).
ia_noop = ImageAugmenter(INPUT_WIDTH, INPUT_HEIGHT)
ia = ImageAugmenter(INPUT_WIDTH, INPUT_HEIGHT,
hflip=True, vflip=False,
scale_to_percent=1.1,
scale_axis_equally=False,
rotation_deg=20,
shear_deg=6,
translation_x_px=4,
translation_y_px=4)
# ---------------
# Run the tests on the train/val/test sets.
# We will do a standard testing with one run over each set.
# We will also do a non-standard test for val and test set where we do
# multiple runs over the same images and augment them each time. Then
# we will average the predictions for each pair over the runs to come
# to a final conclusion.
# Using augmentation seems to improve the results very slightly
# (<1% difference).
# ---------------
# only 1 run for training set, as 10 or more runs would take quite long
# when tested, 10 runs seemed to improve the accuracy by a tiny amount
print("-------------")
print("Training set results (averaged over 1 run)")
print("-------------")
evaluate_model(model, X_train, y_train, ia_noop, 1)
print("")
print("-------------")
print("Validation set results (averaged over 1 run)")
print("-------------")
evaluate_model(model, X_val, y_val, ia_noop, 1)
print("")
print("-------------")
print("Validation set results (averaged over 50 runs)")
print("-------------")
evaluate_model(model, X_val, y_val, ia, 50)
if TEST_COUNT_EXAMPLES > 0:
print("-------------")
print("Test set results (averaged over 1 run)")
print("-------------")
evaluate_model(model, X_test, y_test, ia_noop, 1)
print("")
print("-------------")
print("Test set results (averaged over 50 runs)")
print("-------------")
evaluate_model(model, X_test, y_test, ia, 25)
print("Finished.")
def main():
"""Main function, estimates the performance of a model on the training,
validation und test datasets."""
# handle arguments from command line
parser = argparse.ArgumentParser()
parser.add_argument("identifier", help="Identifier of the experiment of " \
"which to load the weights.")
parser.add_argument("--images", required=True,
help="Filepath to the 'faces/' subdirectory in the " \
"'Labeled Faces in the Wild grayscaled and " \
"cropped' dataset.")
args = parser.parse_args()
validate_identifier(args.identifier, must_exist=True)
if not os.path.isdir(args.images):
raise Exception("The provided filepath to the dataset seems to not " \
"exist.")
# Load:
# 1. Validation set,
# 2. Training set,
# 3. Test set
# We will test on each one of them.
# Results from training and validation set are already known, but will be
# shown in more detail here.
# Additionally, we need to load train and val datasets to make sure that
# no image contained in them is contained in the test set.
print("Loading validation set...")
pairs_val = get_image_pairs(args.images,
VALIDATION_COUNT_EXAMPLES,
pairs_of_same_imgs=False,
ignore_order=True,
exclude_images=list(),
seed=SEED,
verbose=False)
assert len(pairs_val) == VALIDATION_COUNT_EXAMPLES
X_val, y_val = image_pairs_to_xy(pairs_val,
height=INPUT_HEIGHT,
width=INPUT_WIDTH)
print("Loading training set...")
pairs_train = get_image_pairs(args.images,
TRAIN_COUNT_EXAMPLES,
pairs_of_same_imgs=False,
ignore_order=True,
exclude_images=pairs_val,
seed=SEED,
verbose=False)
assert len(pairs_train) == TRAIN_COUNT_EXAMPLES
X_train, y_train = image_pairs_to_xy(pairs_train,
height=INPUT_HEIGHT,
width=INPUT_WIDTH)
print("Loading test set...")
pairs_test = get_image_pairs(args.images,
TEST_COUNT_EXAMPLES,
pairs_of_same_imgs=False,
ignore_order=True,
exclude_images=pairs_val + pairs_train,
seed=SEED,
verbose=False)
assert len(pairs_test) == TEST_COUNT_EXAMPLES
X_test, y_test = image_pairs_to_xy(pairs_test,
height=INPUT_HEIGHT,
width=INPUT_WIDTH)
print("")
# Plot dataset skew
print("Plotting dataset skew. (Only for pairs of images showing the " \
"same person.)")
print("More unequal bars mean that the dataset is more skewed (towards " \
"very few people).")
print("Close the chart to continue.")
plot_dataset_skew(pairs_train,
pairs_val,
pairs_test,
only_y_same=True,
show_plot_windows=SHOW_PLOT_WINDOWS)
print("Creating model...")
model, _ = create_model()
(success, last_epoch) = load_weights(model, SAVE_WEIGHTS_DIR,
args.identifier)
if not success:
raise Exception("Could not successfully load model weights")
print("Loaded model weights of epoch '%s'" % (str(last_epoch)))
# If we just do one run over a set (training/val/test) we will not augment
# the images (ia_noop). If we do multiple runs, we will augment images in
# each run (ia).
ia_noop = ImageAugmenter(INPUT_WIDTH, INPUT_HEIGHT)
ia = ImageAugmenter(INPUT_WIDTH,
INPUT_HEIGHT,
hflip=True,
vflip=False,
scale_to_percent=1.1,
scale_axis_equally=False,
rotation_deg=20,
shear_deg=6,
translation_x_px=4,
translation_y_px=4)
# ---------------
# Run the tests on the train/val/test sets.
# We will do a standard testing with one run over each set.
# We will also do a non-standard test for val and test set where we do
# multiple runs over the same images and augment them each time. Then
# we will average the predictions for each pair over the runs to come
# to a final conclusion.
# Using augmentation seems to improve the results very slightly
# (<1% difference).
# ---------------
# only 1 run for training set, as 10 or more runs would take quite long
# when tested, 10 runs seemed to improve the accuracy by a tiny amount
print("-------------")
print("Training set results (averaged over 1 run)")
print("-------------")
evaluate_model(model, X_train, y_train, ia_noop, 1)
print("")
print("-------------")
print("Validation set results (averaged over 1 run)")
print("-------------")
evaluate_model(model, X_val, y_val, ia_noop, 1)
print("")
print("-------------")
print("Validation set results (averaged over 50 runs)")
print("-------------")
evaluate_model(model, X_val, y_val, ia, 50)
if TEST_COUNT_EXAMPLES > 0:
print("-------------")
print("Test set results (averaged over 1 run)")
print("-------------")
evaluate_model(model, X_test, y_test, ia_noop, 1)
print("")
print("-------------")
print("Test set results (averaged over 50 runs)")
print("-------------")
evaluate_model(model, X_test, y_test, ia, 25)
print("Finished.")
def main():
"""Main function.
1. Handle console arguments,
2. Load datasets,
3. Initialize network,
4. Initialize training looper
5. Train (+validate)."""
# handle arguments from command line
parser = argparse.ArgumentParser()
parser.add_argument("identifier",
help="A short name/identifier for your experiment, " \
"e.g. 'ex42b_more_dropout'.")
parser.add_argument("--images", required=True,
help="Filepath to the 'faces/' subdirectory in the " \
"'Labeled Faces in the Wild grayscaled and " \
"cropped' dataset.")
parser.add_argument("--load", required=False,
help="Identifier of a previous experiment that you " \
"want to continue (loads weights, optimizer state "
"and history).")
parser.add_argument("--dropout", required=False,
help="Dropout rate (0.0 - 1.0) after the last " \
"conv-layer and after the GRU layer. Default " \
"is 0.5.")
parser.add_argument("--augmul", required=False,
help="Multiplicator for the augmentation " \
"(0.0=no augmentation, 1.0=normal aug., " \
"2.0=rather strong aug.). Default is 1.5.")
args = parser.parse_args()
validate_identifier(args.identifier, must_exist=False)
if not os.path.isdir(args.images):
raise Exception(
"The provided filepath to the dataset seems to not exist.")
if args.load:
validate_identifier(args.load)
if identifier_exists(args.identifier):
if args.identifier != args.load:
agreed = ask_continue("[WARNING] Identifier '%s' already exists and " \
"is different from load-identifier '%s'. It " \
"will be overwritten. Continue? [y/n] " \
% (args.identifier, args.load))
if not agreed:
return
if args.augmul is None:
args.augmul = 1.5
# load validation set
# we load this before the training set so that it is less skewed (otherwise
# most images of people with only one image would be lost to the training set)
print("-----------------------")
print("Loading validation dataset...")
print("-----------------------")
print("")
pairs_val = get_image_pairs(args.images,
VALIDATION_COUNT_EXAMPLES,
pairs_of_same_imgs=False,
ignore_order=True,
exclude_images=list(),
seed=SEED,
verbose=True)
# load training set
print("-----------------------")
print("Loading training dataset...")
print("-----------------------")
print("")
pairs_train = get_image_pairs(args.images,
TRAIN_COUNT_EXAMPLES,
pairs_of_same_imgs=False,
ignore_order=True,
exclude_images=pairs_val,
seed=SEED,
verbose=True)
print("-----------------------")
# check if more pairs have been requested than can be generated
assert len(pairs_val) == VALIDATION_COUNT_EXAMPLES
assert len(pairs_train) == TRAIN_COUNT_EXAMPLES
# we loaded pairs of filepaths so far, now load the contents
print("Loading image contents from hard drive...")
X_val, y_val = image_pairs_to_xy(pairs_val)
X_train, y_train = image_pairs_to_xy(pairs_train)
# Plot dataset skew
print("Plotting dataset skew. (Only for pairs of images showing the same " \
"person.)")
print("More unequal bars mean that the dataset is more skewed (towards very " \
"few people).")
print("Close the chart to continue.")
plot_dataset_skew(pairs_train,
pairs_val, [],
only_y_same=True,
show_plot_windows=SHOW_PLOT_WINDOWS,
save_to_filepath=SAVE_DISTRIBUTION_PLOT_FILEPATH.format(
identifier=args.identifier))
# initialize the network
print("Creating model...")
model, optimizer = create_model(args.dropout)
# Calling the compile method seems to mess with the seeds (theano problem?)
# Therefore they are reset here (numpy seeds seem to be unaffected)
# (Seems to still not make runs reproducible.)
random.seed(SEED)
# -------------------
# Training loop part
# -------------------
# initialize the plotter for loss and accuracy
sp_fpath = SAVE_PLOT_FILEPATH.format(identifier=args.identifier)
la_plotter = LossAccPlotter(save_to_filepath=sp_fpath)
# intialize the image augmenters
# they are going to rotate, shift etc. the images
augmul = float(args.augmul)
ia_train = ImageAugmenter(64,
64,
hflip=True,
vflip=False,
scale_to_percent=1.0 + (0.075 * augmul),
scale_axis_equally=False,
rotation_deg=int(7 * augmul),
shear_deg=int(3 * augmul),
translation_x_px=int(3 * augmul),
translation_y_px=int(3 * augmul))
# prefill the training augmenter with lots of random affine transformation
# matrices, so that they can be reused many times
ia_train.pregenerate_matrices(15000)
# we dont want any augmentations for the validation set
ia_val = ImageAugmenter(64, 64)
# load previous data if requested
# includes: weights (works only if new and old model are identical),
# optimizer state (works only for same optimizer, seems to cause errors for adam),
# history (loss and acc values per epoch),
# old plot (will be continued)
if args.load:
print("Loading previous model...")
epoch_start, history = \
load_previous_model(args.load, model, optimizer, la_plotter,
SAVE_OPTIMIZER_STATE_DIR, SAVE_WEIGHTS_DIR,
SAVE_CSV_FILEPATH)
else:
epoch_start = 0
history = History()
# run the training loop
print("Training...")
train_loop(args.identifier, model, optimizer, epoch_start, history,
la_plotter, ia_train, ia_val, X_train, y_train, X_val, y_val)
print("Finished.")
def main():
"""Main function.
1. Handle console arguments,
2. Load datasets,
3. Initialize network,
4. Initialize training looper
5. Train (+validate)."""
# handle arguments from command line
parser = argparse.ArgumentParser()
parser.add_argument("identifier",
help="A short name/identifier for your experiment, " \
"e.g. 'ex42b_more_dropout'.")
parser.add_argument("--images", required=True,
help="Filepath to the 'faces/' subdirectory in the " \
"'Labeled Faces in the Wild grayscaled and " \
"cropped' dataset.")
parser.add_argument("--load", required=False,
help="Identifier of a previous experiment that you " \
"want to continue (loads weights, optimizer state "
"and history).")
parser.add_argument("--dropout", required=False,
help="Dropout rate (0.0 - 1.0) after the last " \
"conv-layer and after the GRU layer. Default " \
"is 0.5.")
parser.add_argument("--augmul", required=False,
help="Multiplicator for the augmentation " \
"(0.0=no augmentation, 1.0=normal aug., " \
"2.0=rather strong aug.). Default is 1.5.")
args = parser.parse_args()
validate_identifier(args.identifier, must_exist=False)
if not os.path.isdir(args.images):
raise Exception("The provided filepath to the dataset seems to not exist.")
if args.load:
validate_identifier(args.load)
if identifier_exists(args.identifier):
if args.identifier != args.load:
agreed = ask_continue("[WARNING] Identifier '%s' already exists and " \
"is different from load-identifier '%s'. It " \
"will be overwritten. Continue? [y/n] " \
% (args.identifier, args.load))
if not agreed:
return
if args.augmul is None:
args.augmul = 1.5
# load validation set
# we load this before the training set so that it is less skewed (otherwise
# most images of people with only one image would be lost to the training set)
print("-----------------------")
print("Loading validation dataset...")
print("-----------------------")
print("")
pairs_val = get_image_pairs(args.images, VALIDATION_COUNT_EXAMPLES,
pairs_of_same_imgs=False, ignore_order=True,
exclude_images=list(), seed=SEED, verbose=True)
# load training set
print("-----------------------")
print("Loading training dataset...")
print("-----------------------")
print("")
pairs_train = get_image_pairs(args.images, TRAIN_COUNT_EXAMPLES,
pairs_of_same_imgs=False, ignore_order=True,
exclude_images=pairs_val, seed=SEED, verbose=True)
print("-----------------------")
# check if more pairs have been requested than can be generated
assert len(pairs_val) == VALIDATION_COUNT_EXAMPLES
assert len(pairs_train) == TRAIN_COUNT_EXAMPLES
# we loaded pairs of filepaths so far, now load the contents
print("Loading image contents from hard drive...")
X_val, y_val = image_pairs_to_xy(pairs_val)
X_train, y_train = image_pairs_to_xy(pairs_train)
# Plot dataset skew
print("Plotting dataset skew. (Only for pairs of images showing the same " \
"person.)")
print("More unequal bars mean that the dataset is more skewed (towards very " \
"few people).")
print("Close the chart to continue.")
plot_dataset_skew(
pairs_train, pairs_val, [],
only_y_same=True,
show_plot_windows=SHOW_PLOT_WINDOWS,
save_to_filepath=SAVE_DISTRIBUTION_PLOT_FILEPATH.format(identifier=args.identifier)
)
# initialize the network
print("Creating model...")
model, optimizer = create_model(args.dropout)
# Calling the compile method seems to mess with the seeds (theano problem?)
# Therefore they are reset here (numpy seeds seem to be unaffected)
# (Seems to still not make runs reproducible.)
random.seed(SEED)
# -------------------
# Training loop part
# -------------------
# initialize the plotter for loss and accuracy
sp_fpath = SAVE_PLOT_FILEPATH.format(identifier=args.identifier)
la_plotter = LossAccPlotter(save_to_filepath=sp_fpath)
# intialize the image augmenters
# they are going to rotate, shift etc. the images
augmul = float(args.augmul)
ia_train = ImageAugmenter(64, 64, hflip=True, vflip=False,
scale_to_percent=1.0 + (0.075*augmul),
scale_axis_equally=False,
rotation_deg=int(7*augmul),
shear_deg=int(3*augmul),
translation_x_px=int(3*augmul),
translation_y_px=int(3*augmul))
# prefill the training augmenter with lots of random affine transformation
# matrices, so that they can be reused many times
ia_train.pregenerate_matrices(15000)
# we dont want any augmentations for the validation set
ia_val = ImageAugmenter(64, 64)
# load previous data if requested
# includes: weights (works only if new and old model are identical),
# optimizer state (works only for same optimizer, seems to cause errors for adam),
# history (loss and acc values per epoch),
# old plot (will be continued)
if args.load:
print("Loading previous model...")
epoch_start, history = \
load_previous_model(args.load, model, optimizer, la_plotter,
SAVE_OPTIMIZER_STATE_DIR, SAVE_WEIGHTS_DIR,
SAVE_CSV_FILEPATH)
else:
epoch_start = 0
history = History()
# run the training loop
print("Training...")
train_loop(args.identifier, model, optimizer, epoch_start, history,
la_plotter, ia_train, ia_val, X_train, y_train, X_val, y_val)
print("Finished.")
def main():
"""Main function.
1. Handle console arguments,
2. Load datasets,
3. Initialize network,
4. Initialize training looper
5. Train (+validate)."""
# handle arguments from command line
parser = argparse.ArgumentParser()
parser.add_argument("identifier",
help="A short name/identifier for your experiment, " \
"e.g. 'ex42b_more_dropout'.")
parser.add_argument("--images", required=True,
help="Filepath to the 'faces/' subdirectory in the " \
"'Labeled Faces in the Wild grayscaled and " \
"cropped' dataset.")
parser.add_argument("--load", required=False,
help="Identifier of a previous experiment that you " \
"want to continue (loads weights, optimizer state "
"and history).")
args = parser.parse_args()
validate_identifier(args.identifier, must_exist=False)
if not os.path.isdir(args.images):
raise Exception("The provided filepath to the dataset seems to not " \
"exist.")
if not args.images.endswith("/faces"):
print("[WARNING] Filepath to the dataset is expected to usually end " \
"in '/faces', i.e. the default directory containing all face " \
"images in the lfwcrop_grey dataset.")
if args.load:
validate_identifier(args.load)
if identifier_exists(args.identifier):
if args.identifier != args.load:
agreed = ask_continue("[WARNING] Identifier '%s' already exists " \
"and is different from load-identifier " \
"'%s'. It will be overwritten. Continue? " \
"[y/n] " % (args.identifier, args.load))
if not agreed:
return
# load validation set
# we load this before the training set so that it is less skewed (otherwise
# most images of people with only one image would be lost to the training
# set)
print("-----------------------")
print("Loading validation dataset...")
print("-----------------------")
print("")
pairs_val = get_image_pairs(args.images, VALIDATION_COUNT_EXAMPLES,
pairs_of_same_imgs=False, ignore_order=True,
exclude_images=list(), seed=SEED, verbose=True)
# load training set
print("-----------------------")
print("Loading training dataset...")
print("-----------------------")
print("")
pairs_train = get_image_pairs(args.images, TRAIN_COUNT_EXAMPLES,
pairs_of_same_imgs=False, ignore_order=True,
exclude_images=pairs_val, seed=SEED,
verbose=True)
print("-----------------------")
# check if more pairs have been requested than can be generated
assert len(pairs_val) == VALIDATION_COUNT_EXAMPLES
assert len(pairs_train) == TRAIN_COUNT_EXAMPLES
# we loaded pairs of filepaths so far, now load the contents
print("Loading image contents from hard drive...")
X_val, y_val = image_pairs_to_xy(pairs_val, height=INPUT_HEIGHT,
width=INPUT_WIDTH)
X_train, y_train = image_pairs_to_xy(pairs_train, height=INPUT_HEIGHT,
width=INPUT_WIDTH)
# Plot dataset skew
print("Saving dataset skew plot to file...")
plot_dataset_skew(
pairs_train, pairs_val, [],
only_y_same=True,
show_plot_windows=SHOW_PLOT_WINDOWS,
save_to_filepath=SAVE_DISTRIBUTION_PLOT_FILEPATH.format(
identifier=args.identifier
)
)
# initialize the network
print("Creating model...")
model, optimizer = create_model()
# Calling the compile method seems to mess with the seeds (theano problem?)
# Therefore they are reset here (numpy seeds seem to be unaffected)
# (Seems to still not make runs reproducible.)
random.seed(SEED)
# -------------------
# Training loop part
# -------------------
# initialize the plotter for loss and accuracy
sp_fpath = SAVE_PLOT_FILEPATH.format(identifier=args.identifier)
la_plotter = LossAccPlotter(save_to_filepath=sp_fpath,
show_plot_window=SHOW_PLOT_WINDOWS)
# initialize the image augmenter for training images
ia_train = ImageAugmenter(INPUT_WIDTH, INPUT_HEIGHT,
hflip=True, vflip=False,
scale_to_percent=1.1,
scale_axis_equally=False,
rotation_deg=20,
shear_deg=6,
translation_x_px=4,
translation_y_px=4)
# prefill the training augmenter with lots of random affine transformation
# matrices, so that they can be reused many times
ia_train.pregenerate_matrices(15000)
# we dont want any augmentations for the validation set
ia_val = ImageAugmenter(INPUT_WIDTH, INPUT_HEIGHT)
# load previous data if requested
# includes: weights (works only if new and old model are identical),
# history (loss and acc values per epoch),
# old plot (will be continued)
if args.load:
print("Loading previous model...")
epoch_start, history = \
load_previous_model(args.load, model, la_plotter,
SAVE_WEIGHTS_DIR, SAVE_CSV_FILEPATH)
else:
epoch_start = 0
history = History()
print("Model summary:")
model.summary()
# run the training loop
print("Training...")
train_loop(args.identifier, model, optimizer, epoch_start, history,
la_plotter, ia_train, ia_val, X_train, y_train, X_val, y_val)
print("Finished.")