Beispiel #1
0
def main():
    """Main function, estimates the performance of a model on the training,
    validation und test datasets."""
    # handle arguments from command line
    parser = argparse.ArgumentParser()
    parser.add_argument("identifier", help="Identifier of the experiment of " \
                                           "which to load the weights.")
    parser.add_argument("--images", required=True,
                        help="Filepath to the 'faces/' subdirectory in the " \
                             "'Labeled Faces in the Wild grayscaled and " \
                             "cropped' dataset.")
    args = parser.parse_args()
    validate_identifier(args.identifier, must_exist=True)

    if not os.path.isdir(args.images):
        raise Exception("The provided filepath to the dataset seems to not " \
                        "exist.")

    # Load:
    #  1. Validation set,
    #  2. Training set,
    #  3. Test set
    # We will test on each one of them.
    # Results from training and validation set are already known, but will be
    # shown in more detail here.
    # Additionally, we need to load train and val datasets to make sure that
    # no image contained in them is contained in the test set.
    print("Loading validation set...")
    pairs_val = get_image_pairs(args.images, VALIDATION_COUNT_EXAMPLES,
                                pairs_of_same_imgs=False, ignore_order=True,
                                exclude_images=list(), seed=SEED,
                                verbose=False)
    assert len(pairs_val) == VALIDATION_COUNT_EXAMPLES
    X_val, y_val = image_pairs_to_xy(pairs_val, height=INPUT_HEIGHT,
                                     width=INPUT_WIDTH)

    print("Loading training set...")
    pairs_train = get_image_pairs(args.images, TRAIN_COUNT_EXAMPLES,
                                  pairs_of_same_imgs=False, ignore_order=True,
                                  exclude_images=pairs_val, seed=SEED,
                                  verbose=False)
    assert len(pairs_train) == TRAIN_COUNT_EXAMPLES
    X_train, y_train = image_pairs_to_xy(pairs_train, height=INPUT_HEIGHT,
                                         width=INPUT_WIDTH)

    print("Loading test set...")
    pairs_test = get_image_pairs(args.images, TEST_COUNT_EXAMPLES,
                                 pairs_of_same_imgs=False, ignore_order=True,
                                 exclude_images=pairs_val+pairs_train,
                                 seed=SEED, verbose=False)
    assert len(pairs_test) == TEST_COUNT_EXAMPLES
    X_test, y_test = image_pairs_to_xy(pairs_test, height=INPUT_HEIGHT,
                                       width=INPUT_WIDTH)
    print("")

    # Plot dataset skew
    print("Plotting dataset skew. (Only for pairs of images showing the " \
          "same person.)")
    print("More unequal bars mean that the dataset is more skewed (towards " \
          "very few people).")
    print("Close the chart to continue.")
    plot_dataset_skew(
        pairs_train, pairs_val, pairs_test,
        only_y_same=True,
        show_plot_windows=SHOW_PLOT_WINDOWS
    )

    print("Creating model...")
    model, _ = create_model()
    (success, last_epoch) = load_weights(model, SAVE_WEIGHTS_DIR,
                                         args.identifier)
    if not success:
        raise Exception("Could not successfully load model weights")
    print("Loaded model weights of epoch '%s'" % (str(last_epoch)))

    # If we just do one run over a set (training/val/test) we will not augment
    # the images (ia_noop). If we do multiple runs, we will augment images in
    # each run (ia).
    ia_noop = ImageAugmenter(INPUT_WIDTH, INPUT_HEIGHT)
    ia = ImageAugmenter(INPUT_WIDTH, INPUT_HEIGHT,
                        hflip=True, vflip=False,
                        scale_to_percent=1.1,
                        scale_axis_equally=False,
                        rotation_deg=20,
                        shear_deg=6,
                        translation_x_px=4,
                        translation_y_px=4)

    # ---------------
    # Run the tests on the train/val/test sets.
    # We will do a standard testing with one run over each set.
    # We will also do a non-standard test for val and test set where we do
    # multiple runs over the same images and augment them each time. Then
    # we will average the predictions for each pair over the runs to come
    # to a final conclusion.
    # Using augmentation seems to improve the results very slightly
    # (<1% difference).
    # ---------------

    # only 1 run for training set, as 10 or more runs would take quite long
    # when tested, 10 runs seemed to improve the accuracy by a tiny amount
    print("-------------")
    print("Training set results (averaged over 1 run)")
    print("-------------")
    evaluate_model(model, X_train, y_train, ia_noop, 1)
    print("")

    print("-------------")
    print("Validation set results (averaged over 1 run)")
    print("-------------")
    evaluate_model(model, X_val, y_val, ia_noop, 1)
    print("")

    print("-------------")
    print("Validation set results (averaged over 50 runs)")
    print("-------------")
    evaluate_model(model, X_val, y_val, ia, 50)

    if TEST_COUNT_EXAMPLES > 0:
        print("-------------")
        print("Test set results (averaged over 1 run)")
        print("-------------")
        evaluate_model(model, X_test, y_test, ia_noop, 1)
        print("")

        print("-------------")
        print("Test set results (averaged over 50 runs)")
        print("-------------")
        evaluate_model(model, X_test, y_test, ia, 25)

    print("Finished.")
Beispiel #2
0
def main():
    """Main function, estimates the performance of a model on the training,
    validation und test datasets."""
    # handle arguments from command line
    parser = argparse.ArgumentParser()
    parser.add_argument("identifier", help="Identifier of the experiment of " \
                                           "which to load the weights.")
    parser.add_argument("--images", required=True,
                        help="Filepath to the 'faces/' subdirectory in the " \
                             "'Labeled Faces in the Wild grayscaled and " \
                             "cropped' dataset.")
    args = parser.parse_args()
    validate_identifier(args.identifier, must_exist=True)

    if not os.path.isdir(args.images):
        raise Exception("The provided filepath to the dataset seems to not " \
                        "exist.")

    # Load:
    #  1. Validation set,
    #  2. Training set,
    #  3. Test set
    # We will test on each one of them.
    # Results from training and validation set are already known, but will be
    # shown in more detail here.
    # Additionally, we need to load train and val datasets to make sure that
    # no image contained in them is contained in the test set.
    print("Loading validation set...")
    pairs_val = get_image_pairs(args.images,
                                VALIDATION_COUNT_EXAMPLES,
                                pairs_of_same_imgs=False,
                                ignore_order=True,
                                exclude_images=list(),
                                seed=SEED,
                                verbose=False)
    assert len(pairs_val) == VALIDATION_COUNT_EXAMPLES
    X_val, y_val = image_pairs_to_xy(pairs_val,
                                     height=INPUT_HEIGHT,
                                     width=INPUT_WIDTH)

    print("Loading training set...")
    pairs_train = get_image_pairs(args.images,
                                  TRAIN_COUNT_EXAMPLES,
                                  pairs_of_same_imgs=False,
                                  ignore_order=True,
                                  exclude_images=pairs_val,
                                  seed=SEED,
                                  verbose=False)
    assert len(pairs_train) == TRAIN_COUNT_EXAMPLES
    X_train, y_train = image_pairs_to_xy(pairs_train,
                                         height=INPUT_HEIGHT,
                                         width=INPUT_WIDTH)

    print("Loading test set...")
    pairs_test = get_image_pairs(args.images,
                                 TEST_COUNT_EXAMPLES,
                                 pairs_of_same_imgs=False,
                                 ignore_order=True,
                                 exclude_images=pairs_val + pairs_train,
                                 seed=SEED,
                                 verbose=False)
    assert len(pairs_test) == TEST_COUNT_EXAMPLES
    X_test, y_test = image_pairs_to_xy(pairs_test,
                                       height=INPUT_HEIGHT,
                                       width=INPUT_WIDTH)
    print("")

    # Plot dataset skew
    print("Plotting dataset skew. (Only for pairs of images showing the " \
          "same person.)")
    print("More unequal bars mean that the dataset is more skewed (towards " \
          "very few people).")
    print("Close the chart to continue.")
    plot_dataset_skew(pairs_train,
                      pairs_val,
                      pairs_test,
                      only_y_same=True,
                      show_plot_windows=SHOW_PLOT_WINDOWS)

    print("Creating model...")
    model, _ = create_model()
    (success, last_epoch) = load_weights(model, SAVE_WEIGHTS_DIR,
                                         args.identifier)
    if not success:
        raise Exception("Could not successfully load model weights")
    print("Loaded model weights of epoch '%s'" % (str(last_epoch)))

    # If we just do one run over a set (training/val/test) we will not augment
    # the images (ia_noop). If we do multiple runs, we will augment images in
    # each run (ia).
    ia_noop = ImageAugmenter(INPUT_WIDTH, INPUT_HEIGHT)
    ia = ImageAugmenter(INPUT_WIDTH,
                        INPUT_HEIGHT,
                        hflip=True,
                        vflip=False,
                        scale_to_percent=1.1,
                        scale_axis_equally=False,
                        rotation_deg=20,
                        shear_deg=6,
                        translation_x_px=4,
                        translation_y_px=4)

    # ---------------
    # Run the tests on the train/val/test sets.
    # We will do a standard testing with one run over each set.
    # We will also do a non-standard test for val and test set where we do
    # multiple runs over the same images and augment them each time. Then
    # we will average the predictions for each pair over the runs to come
    # to a final conclusion.
    # Using augmentation seems to improve the results very slightly
    # (<1% difference).
    # ---------------

    # only 1 run for training set, as 10 or more runs would take quite long
    # when tested, 10 runs seemed to improve the accuracy by a tiny amount
    print("-------------")
    print("Training set results (averaged over 1 run)")
    print("-------------")
    evaluate_model(model, X_train, y_train, ia_noop, 1)
    print("")

    print("-------------")
    print("Validation set results (averaged over 1 run)")
    print("-------------")
    evaluate_model(model, X_val, y_val, ia_noop, 1)
    print("")

    print("-------------")
    print("Validation set results (averaged over 50 runs)")
    print("-------------")
    evaluate_model(model, X_val, y_val, ia, 50)

    if TEST_COUNT_EXAMPLES > 0:
        print("-------------")
        print("Test set results (averaged over 1 run)")
        print("-------------")
        evaluate_model(model, X_test, y_test, ia_noop, 1)
        print("")

        print("-------------")
        print("Test set results (averaged over 50 runs)")
        print("-------------")
        evaluate_model(model, X_test, y_test, ia, 25)

    print("Finished.")
Beispiel #3
0
def main():
    """Main function.
    1. Handle console arguments,
    2. Load datasets,
    3. Initialize network,
    4. Initialize training looper
    5. Train (+validate)."""

    # handle arguments from command line
    parser = argparse.ArgumentParser()
    parser.add_argument("identifier",
                        help="A short name/identifier for your experiment, " \
                             "e.g. 'ex42b_more_dropout'.")
    parser.add_argument("--images", required=True,
                        help="Filepath to the 'faces/' subdirectory in the " \
                             "'Labeled Faces in the Wild grayscaled and " \
                             "cropped' dataset.")
    parser.add_argument("--load", required=False,
                        help="Identifier of a previous experiment that you " \
                             "want to continue (loads weights, optimizer state "
                             "and history).")
    parser.add_argument("--dropout", required=False,
                        help="Dropout rate (0.0 - 1.0) after the last " \
                             "conv-layer and after the GRU layer. Default " \
                             "is 0.5.")
    parser.add_argument("--augmul", required=False,
                        help="Multiplicator for the augmentation " \
                             "(0.0=no augmentation, 1.0=normal aug., " \
                             "2.0=rather strong aug.). Default is 1.5.")
    args = parser.parse_args()
    validate_identifier(args.identifier, must_exist=False)

    if not os.path.isdir(args.images):
        raise Exception(
            "The provided filepath to the dataset seems to not exist.")

    if args.load:
        validate_identifier(args.load)

    if identifier_exists(args.identifier):
        if args.identifier != args.load:
            agreed = ask_continue("[WARNING] Identifier '%s' already exists and " \
                                  "is different from load-identifier '%s'. It " \
                                  "will be overwritten. Continue? [y/n] " \
                                  % (args.identifier, args.load))
            if not agreed:
                return

    if args.augmul is None:
        args.augmul = 1.5

    # load validation set
    # we load this before the training set so that it is less skewed (otherwise
    # most images of people with only one image would be lost to the training set)
    print("-----------------------")
    print("Loading validation dataset...")
    print("-----------------------")
    print("")
    pairs_val = get_image_pairs(args.images,
                                VALIDATION_COUNT_EXAMPLES,
                                pairs_of_same_imgs=False,
                                ignore_order=True,
                                exclude_images=list(),
                                seed=SEED,
                                verbose=True)

    # load training set
    print("-----------------------")
    print("Loading training dataset...")
    print("-----------------------")
    print("")
    pairs_train = get_image_pairs(args.images,
                                  TRAIN_COUNT_EXAMPLES,
                                  pairs_of_same_imgs=False,
                                  ignore_order=True,
                                  exclude_images=pairs_val,
                                  seed=SEED,
                                  verbose=True)
    print("-----------------------")

    # check if more pairs have been requested than can be generated
    assert len(pairs_val) == VALIDATION_COUNT_EXAMPLES
    assert len(pairs_train) == TRAIN_COUNT_EXAMPLES

    # we loaded pairs of filepaths so far, now load the contents
    print("Loading image contents from hard drive...")
    X_val, y_val = image_pairs_to_xy(pairs_val)
    X_train, y_train = image_pairs_to_xy(pairs_train)

    # Plot dataset skew
    print("Plotting dataset skew. (Only for pairs of images showing the same " \
          "person.)")
    print("More unequal bars mean that the dataset is more skewed (towards very " \
          "few people).")
    print("Close the chart to continue.")
    plot_dataset_skew(pairs_train,
                      pairs_val, [],
                      only_y_same=True,
                      show_plot_windows=SHOW_PLOT_WINDOWS,
                      save_to_filepath=SAVE_DISTRIBUTION_PLOT_FILEPATH.format(
                          identifier=args.identifier))

    # initialize the network
    print("Creating model...")
    model, optimizer = create_model(args.dropout)

    # Calling the compile method seems to mess with the seeds (theano problem?)
    # Therefore they are reset here (numpy seeds seem to be unaffected)
    # (Seems to still not make runs reproducible.)
    random.seed(SEED)

    # -------------------
    # Training loop part
    # -------------------
    # initialize the plotter for loss and accuracy
    sp_fpath = SAVE_PLOT_FILEPATH.format(identifier=args.identifier)
    la_plotter = LossAccPlotter(save_to_filepath=sp_fpath)

    # intialize the image augmenters
    # they are going to rotate, shift etc. the images
    augmul = float(args.augmul)
    ia_train = ImageAugmenter(64,
                              64,
                              hflip=True,
                              vflip=False,
                              scale_to_percent=1.0 + (0.075 * augmul),
                              scale_axis_equally=False,
                              rotation_deg=int(7 * augmul),
                              shear_deg=int(3 * augmul),
                              translation_x_px=int(3 * augmul),
                              translation_y_px=int(3 * augmul))
    # prefill the training augmenter with lots of random affine transformation
    # matrices, so that they can be reused many times
    ia_train.pregenerate_matrices(15000)

    # we dont want any augmentations for the validation set
    ia_val = ImageAugmenter(64, 64)

    # load previous data if requested
    # includes: weights (works only if new and old model are identical),
    # optimizer state (works only for same optimizer, seems to cause errors for adam),
    # history (loss and acc values per epoch),
    # old plot (will be continued)
    if args.load:
        print("Loading previous model...")
        epoch_start, history = \
            load_previous_model(args.load, model, optimizer, la_plotter,
                                SAVE_OPTIMIZER_STATE_DIR, SAVE_WEIGHTS_DIR,
                                SAVE_CSV_FILEPATH)
    else:
        epoch_start = 0
        history = History()

    # run the training loop
    print("Training...")
    train_loop(args.identifier, model, optimizer, epoch_start, history,
               la_plotter, ia_train, ia_val, X_train, y_train, X_val, y_val)

    print("Finished.")
Beispiel #4
0
def main():
    """Main function.
    1. Handle console arguments,
    2. Load datasets,
    3. Initialize network,
    4. Initialize training looper
    5. Train (+validate)."""

    # handle arguments from command line
    parser = argparse.ArgumentParser()
    parser.add_argument("identifier",
                        help="A short name/identifier for your experiment, " \
                             "e.g. 'ex42b_more_dropout'.")
    parser.add_argument("--images", required=True,
                        help="Filepath to the 'faces/' subdirectory in the " \
                             "'Labeled Faces in the Wild grayscaled and " \
                             "cropped' dataset.")
    parser.add_argument("--load", required=False,
                        help="Identifier of a previous experiment that you " \
                             "want to continue (loads weights, optimizer state "
                             "and history).")
    parser.add_argument("--dropout", required=False,
                        help="Dropout rate (0.0 - 1.0) after the last " \
                             "conv-layer and after the GRU layer. Default " \
                             "is 0.5.")
    parser.add_argument("--augmul", required=False,
                        help="Multiplicator for the augmentation " \
                             "(0.0=no augmentation, 1.0=normal aug., " \
                             "2.0=rather strong aug.). Default is 1.5.")
    args = parser.parse_args()
    validate_identifier(args.identifier, must_exist=False)

    if not os.path.isdir(args.images):
        raise Exception("The provided filepath to the dataset seems to not exist.")

    if args.load:
        validate_identifier(args.load)

    if identifier_exists(args.identifier):
        if args.identifier != args.load:
            agreed = ask_continue("[WARNING] Identifier '%s' already exists and " \
                                  "is different from load-identifier '%s'. It " \
                                  "will be overwritten. Continue? [y/n] " \
                                  % (args.identifier, args.load))
            if not agreed:
                return

    if args.augmul is None:
        args.augmul = 1.5

    # load validation set
    # we load this before the training set so that it is less skewed (otherwise
    # most images of people with only one image would be lost to the training set)
    print("-----------------------")
    print("Loading validation dataset...")
    print("-----------------------")
    print("")
    pairs_val = get_image_pairs(args.images, VALIDATION_COUNT_EXAMPLES,
                                pairs_of_same_imgs=False, ignore_order=True,
                                exclude_images=list(), seed=SEED, verbose=True)

    # load training set
    print("-----------------------")
    print("Loading training dataset...")
    print("-----------------------")
    print("")
    pairs_train = get_image_pairs(args.images, TRAIN_COUNT_EXAMPLES,
                                  pairs_of_same_imgs=False, ignore_order=True,
                                  exclude_images=pairs_val, seed=SEED, verbose=True)
    print("-----------------------")

    # check if more pairs have been requested than can be generated
    assert len(pairs_val) == VALIDATION_COUNT_EXAMPLES
    assert len(pairs_train) == TRAIN_COUNT_EXAMPLES

    # we loaded pairs of filepaths so far, now load the contents
    print("Loading image contents from hard drive...")
    X_val, y_val = image_pairs_to_xy(pairs_val)
    X_train, y_train = image_pairs_to_xy(pairs_train)

    # Plot dataset skew
    print("Plotting dataset skew. (Only for pairs of images showing the same " \
          "person.)")
    print("More unequal bars mean that the dataset is more skewed (towards very " \
          "few people).")
    print("Close the chart to continue.")
    plot_dataset_skew(
        pairs_train, pairs_val, [],
        only_y_same=True,
        show_plot_windows=SHOW_PLOT_WINDOWS,
        save_to_filepath=SAVE_DISTRIBUTION_PLOT_FILEPATH.format(identifier=args.identifier)
    )

    # initialize the network
    print("Creating model...")
    model, optimizer = create_model(args.dropout)

    # Calling the compile method seems to mess with the seeds (theano problem?)
    # Therefore they are reset here (numpy seeds seem to be unaffected)
    # (Seems to still not make runs reproducible.)
    random.seed(SEED)

    # -------------------
    # Training loop part
    # -------------------
    # initialize the plotter for loss and accuracy
    sp_fpath = SAVE_PLOT_FILEPATH.format(identifier=args.identifier)
    la_plotter = LossAccPlotter(save_to_filepath=sp_fpath)

    # intialize the image augmenters
    # they are going to rotate, shift etc. the images
    augmul = float(args.augmul)
    ia_train = ImageAugmenter(64, 64, hflip=True, vflip=False,
                              scale_to_percent=1.0 + (0.075*augmul),
                              scale_axis_equally=False,
                              rotation_deg=int(7*augmul),
                              shear_deg=int(3*augmul),
                              translation_x_px=int(3*augmul),
                              translation_y_px=int(3*augmul))
    # prefill the training augmenter with lots of random affine transformation
    # matrices, so that they can be reused many times
    ia_train.pregenerate_matrices(15000)

    # we dont want any augmentations for the validation set
    ia_val = ImageAugmenter(64, 64)

    # load previous data if requested
    # includes: weights (works only if new and old model are identical),
    # optimizer state (works only for same optimizer, seems to cause errors for adam),
    # history (loss and acc values per epoch),
    # old plot (will be continued)
    if args.load:
        print("Loading previous model...")
        epoch_start, history = \
            load_previous_model(args.load, model, optimizer, la_plotter,
                                SAVE_OPTIMIZER_STATE_DIR, SAVE_WEIGHTS_DIR,
                                SAVE_CSV_FILEPATH)
    else:
        epoch_start = 0
        history = History()

    # run the training loop
    print("Training...")
    train_loop(args.identifier, model, optimizer, epoch_start, history,
               la_plotter, ia_train, ia_val, X_train, y_train, X_val, y_val)

    print("Finished.")
Beispiel #5
0
def main():
    """Main function.
    1. Handle console arguments,
    2. Load datasets,
    3. Initialize network,
    4. Initialize training looper
    5. Train (+validate)."""

    # handle arguments from command line
    parser = argparse.ArgumentParser()
    parser.add_argument("identifier",
                        help="A short name/identifier for your experiment, " \
                             "e.g. 'ex42b_more_dropout'.")
    parser.add_argument("--images", required=True,
                        help="Filepath to the 'faces/' subdirectory in the " \
                             "'Labeled Faces in the Wild grayscaled and " \
                             "cropped' dataset.")
    parser.add_argument("--load", required=False,
                        help="Identifier of a previous experiment that you " \
                             "want to continue (loads weights, optimizer state "
                             "and history).")
    args = parser.parse_args()
    validate_identifier(args.identifier, must_exist=False)

    if not os.path.isdir(args.images):
        raise Exception("The provided filepath to the dataset seems to not " \
                        "exist.")

    if not args.images.endswith("/faces"):
        print("[WARNING] Filepath to the dataset is expected to usually end " \
              "in '/faces', i.e. the default directory containing all face " \
              "images in the lfwcrop_grey dataset.")

    if args.load:
        validate_identifier(args.load)

    if identifier_exists(args.identifier):
        if args.identifier != args.load:
            agreed = ask_continue("[WARNING] Identifier '%s' already exists " \
                                  "and is different from load-identifier " \
                                  "'%s'. It will be overwritten. Continue? " \
                                  "[y/n] " % (args.identifier, args.load))
            if not agreed:
                return

    # load validation set
    # we load this before the training set so that it is less skewed (otherwise
    # most images of people with only one image would be lost to the training
    # set)
    print("-----------------------")
    print("Loading validation dataset...")
    print("-----------------------")
    print("")
    pairs_val = get_image_pairs(args.images, VALIDATION_COUNT_EXAMPLES,
                                pairs_of_same_imgs=False, ignore_order=True,
                                exclude_images=list(), seed=SEED, verbose=True)

    # load training set
    print("-----------------------")
    print("Loading training dataset...")
    print("-----------------------")
    print("")
    pairs_train = get_image_pairs(args.images, TRAIN_COUNT_EXAMPLES,
                                  pairs_of_same_imgs=False, ignore_order=True,
                                  exclude_images=pairs_val, seed=SEED,
                                  verbose=True)
    print("-----------------------")

    # check if more pairs have been requested than can be generated
    assert len(pairs_val) == VALIDATION_COUNT_EXAMPLES
    assert len(pairs_train) == TRAIN_COUNT_EXAMPLES

    # we loaded pairs of filepaths so far, now load the contents
    print("Loading image contents from hard drive...")
    X_val, y_val = image_pairs_to_xy(pairs_val, height=INPUT_HEIGHT,
                                     width=INPUT_WIDTH)
    X_train, y_train = image_pairs_to_xy(pairs_train, height=INPUT_HEIGHT,
                                         width=INPUT_WIDTH)

    # Plot dataset skew
    print("Saving dataset skew plot to file...")
    plot_dataset_skew(
        pairs_train, pairs_val, [],
        only_y_same=True,
        show_plot_windows=SHOW_PLOT_WINDOWS,
        save_to_filepath=SAVE_DISTRIBUTION_PLOT_FILEPATH.format(
            identifier=args.identifier
        )
    )

    # initialize the network
    print("Creating model...")
    model, optimizer = create_model()

    # Calling the compile method seems to mess with the seeds (theano problem?)
    # Therefore they are reset here (numpy seeds seem to be unaffected)
    # (Seems to still not make runs reproducible.)
    random.seed(SEED)

    # -------------------
    # Training loop part
    # -------------------
    # initialize the plotter for loss and accuracy
    sp_fpath = SAVE_PLOT_FILEPATH.format(identifier=args.identifier)
    la_plotter = LossAccPlotter(save_to_filepath=sp_fpath,
                                show_plot_window=SHOW_PLOT_WINDOWS)

    # initialize the image augmenter for training images
    ia_train = ImageAugmenter(INPUT_WIDTH, INPUT_HEIGHT,
                              hflip=True, vflip=False,
                              scale_to_percent=1.1,
                              scale_axis_equally=False,
                              rotation_deg=20,
                              shear_deg=6,
                              translation_x_px=4,
                              translation_y_px=4)

    # prefill the training augmenter with lots of random affine transformation
    # matrices, so that they can be reused many times
    ia_train.pregenerate_matrices(15000)

    # we dont want any augmentations for the validation set
    ia_val = ImageAugmenter(INPUT_WIDTH, INPUT_HEIGHT)

    # load previous data if requested
    # includes: weights (works only if new and old model are identical),
    # history (loss and acc values per epoch),
    # old plot (will be continued)
    if args.load:
        print("Loading previous model...")
        epoch_start, history = \
            load_previous_model(args.load, model, la_plotter,
                                SAVE_WEIGHTS_DIR, SAVE_CSV_FILEPATH)
    else:
        epoch_start = 0
        history = History()

    print("Model summary:")
    model.summary()

    # run the training loop
    print("Training...")
    train_loop(args.identifier, model, optimizer, epoch_start, history,
               la_plotter, ia_train, ia_val, X_train, y_train, X_val, y_val)

    print("Finished.")