def get_optional_arguments(): """ Put the arguments in a list so that they are accessible from both argparse and gui """ argument_list = [] argument_list.append({"opts": ("--serializer", ), "type": str.lower, "dest": "serializer", "default": "json", "choices": ("json", "pickle", "yaml"), "help": "Serializer for alignments file. If " "yaml is chosen and not available, then " "json will be used as the default " "fallback."}) argument_list.append({ "opts": ("-D", "--detector"), "type": str.lower, "choices": PluginLoader.get_available_extractors( "detect"), "default": "mtcnn", "help": "R|Detector to use." "\n'dlib-hog': uses least resources, but is the" "\n\tleast reliable." "\n'dlib-cnn': faster than mtcnn but detects" "\n\tfewer faces and fewer false positives." "\n'mtcnn': slower than dlib, but uses fewer" "\n\tresources whilst detecting more faces and" "\n\tmore false positives. Has superior" "\n\talignment to dlib"}) argument_list.append({ "opts": ("-A", "--aligner"), "type": str.lower, "choices": PluginLoader.get_available_extractors( "align"), "default": "fan", "help": "R|Aligner to use." "\n'dlib': Dlib Pose Predictor. Faster, less " "\n\tresource intensive, but less accurate." "\n'fan': Face Alignment Network. Best aligner." "\n\tGPU heavy, slow when not running on GPU"}) argument_list.append({"opts": ("-r", "--rotate-images"), "type": str, "dest": "rotate_images", "default": None, "help": "If a face isn't found, rotate the " "images to try to find a face. Can find " "more faces at the cost of extraction " "speed. Pass in a single number to use " "increments of that size up to 360, or " "pass in a list of numbers to enumerate " "exactly what angles to check"}) argument_list.append({"opts": ("-bt", "--blur-threshold"), "type": float, "action": Slider, "min_max": (0.0, 100.0), "rounding": 1, "dest": "blur_thresh", "default": 0.0, "help": "Automatically discard images blurrier than the specified " "threshold. Discarded images are moved into a \"blurry\" " "sub-folder. Lower values allow more blur. Set to 0.0 to " "turn off."}) argument_list.append({"opts": ("-mp", "--multiprocess"), "action": "store_true", "default": False, "help": "Run extraction in parallel. Offers " "speed up for some extractor/detector " "combinations, less so for others. " "Only has an effect if both the " "aligner and detector use the GPU, " "otherwise this is automatic."}) argument_list.append({"opts": ("-sz", "--size"), "type": int, "action": Slider, "min_max": (128, 512), "default": 256, "rounding": 64, "help": "The output size of extracted faces. Make sure that the " "model you intend to train supports your required size. " "This will only need to be changed for hi-res models."}) argument_list.append({"opts": ("-min", "--min-size"), "type": int, "action": Slider, "dest": "min_size", "min_max": (0, 1080), "default": 0, "rounding": 20, "help": "Filters out faces detected below this size. Length, in " "pixels across the diagonal of the bounding box. Set to 0 " "for off"}) argument_list.append({"opts": ("-s", "--skip-existing"), "action": "store_true", "dest": "skip_existing", "default": False, "help": "Skips frames that have already been " "extracted and exist in the alignments " "file"}) argument_list.append({"opts": ("-sf", "--skip-existing-faces"), "action": "store_true", "dest": "skip_faces", "default": False, "help": "Skip frames that already have " "detected faces in the alignments " "file"}) argument_list.append({"opts": ("-dl", "--debug-landmarks"), "action": "store_true", "dest": "debug_landmarks", "default": False, "help": "Draw landmarks on the ouput faces for " "debug"}) argument_list.append({"opts": ("-ae", "--align-eyes"), "action": "store_true", "dest": "align_eyes", "default": False, "help": "Perform extra alignment to ensure " "left/right eyes are at the same " "height"}) argument_list.append({"opts": ("-si", "--save-interval"), "dest": "save_interval", "type": int, "action": Slider, "min_max": (0, 1000), "rounding": 10, "default": 0, "help": "Automatically save the alignments file after a set amount " "of frames. Will only save at the end of extracting by " "default. WARNING: Don't interrupt the script when writing " "the file because it might get corrupted. Set to 0 to turn " "off"}) return argument_list
def _set_loss(self): """ Set the default loss options. Loss Documentation MAE https://heartbeat.fritz.ai/5-regression-loss-functions-all-machine -learners-should-know-4fb140e9d4b0 MSE https://heartbeat.fritz.ai/5-regression-loss-functions-all-machine -learners-should-know-4fb140e9d4b0 LogCosh https://heartbeat.fritz.ai/5-regression-loss-functions-all-machine -learners-should-know-4fb140e9d4b0 Smooth L1 https://arxiv.org/pdf/1701.03077.pdf L_inf_norm https://medium.com/@montjoile/l0-norm-l1-norm-l2-norm-l-infinity -norm-7a7d18a4f40c SSIM http://www.cns.nyu.edu/pub/eero/wang03-reprint.pdf GMSD https://arxiv.org/ftp/arxiv/papers/1308/1308.3052.pdf """ logger.debug("Setting Loss config") section = "global.loss" self.add_section(title=section, info="Loss configuration options\n" "Loss is the mechanism by which a Neural Network judges how well it " "thinks that it is recreating a face." + ADDITIONAL_INFO) self.add_item( section=section, title="loss_function", datatype=str, group="loss", default="ssim", choices=["mae", "mse", "logcosh", "smooth_loss", "l_inf_norm", "ssim", "gmsd", "pixel_gradient_diff"], info="The loss function to use." "\n\t MAE - Mean absolute error will guide reconstructions of each pixel " "towards its median value in the training dataset. Robust to outliers but as " "a median, it can potentially ignore some infrequent image types in the dataset." "\n\t MSE - Mean squared error will guide reconstructions of each pixel " "towards its average value in the training dataset. As an avg, it will be " "suspectible to outliers and typically produces slightly blurrier results." "\n\t LogCosh - log(cosh(x)) acts similiar to MSE for small errors and to " "MAE for large errors. Like MSE, it is very stable and prevents overshoots " "when errors are near zero. Like MAE, it is robust to outliers. NB: Due to a bug " "in PlaidML, this loss does not work on AMD cards." "\n\t Smooth_L1 --- Modification of the MAE loss to correct two of its " "disadvantages. This loss has improved stability and guidance for small errors." "\n\t L_inf_norm --- The L_inf norm will reduce the largest individual pixel " "error in an image. As each largest error is minimized sequentially, the " "overall error is improved. This loss will be extremely focused on outliers." "\n\t SSIM - Structural Similarity Index Metric is a perception-based " "loss that considers changes in texture, luminance, contrast, and local spatial " "statistics of an image. Potentially delivers more realistic looking images." "\n\t GMSD - Gradient Magnitude Similarity Deviation seeks to match " "the global standard deviation of the pixel to pixel differences between two " "images. Similiar in approach to SSIM. NB: This loss does not currently work on " "AMD cards." "\n\t Pixel_Gradient_Difference - Instead of minimizing the difference between " "the absolute value of each pixel in two reference images, compute the pixel to " "pixel spatial difference in each image and then minimize that difference " "between two images. Allows for large color shifts,but maintains the structure " "of the image.") self.add_item( section=section, title="mask_loss_function", datatype=str, group="loss", default="mse", choices=["mae", "mse"], info="The loss function to use when learning a mask." "\n\t MAE - Mean absolute error will guide reconstructions of each pixel " "towards its median value in the training dataset. Robust to outliers but as " "a median, it can potentially ignore some infrequent image types in the dataset." "\n\t MSE - Mean squared error will guide reconstructions of each pixel " "towards its average value in the training dataset. As an avg, it will be " "suspectible to outliers and typically produces slightly blurrier results.") self.add_item( section=section, title="l2_reg_term", datatype=int, group="loss", min_max=(0, 400), rounding=1, default=100, info="The amount of L2 Regularization to apply as a penalty to Structural Similarity " "loss functions.\n\nNB: You should only adjust this if you know what you are " "doing!\n\n" "L2 regularization applies a penalty term to the given Loss function. This " "penalty will only be applied if SSIM or GMSD is selected for the main loss " "function, otherwise it is ignored.\n\nThe value given here is as a percentage " "weight of the main loss function. For example:" "\n\t 100 - Will give equal weighting to the main loss and the penalty function. " "\n\t 25 - Will give the penalty function 1/4 of the weight of the main loss " "function. " "\n\t 400 - Will give the penalty function 4x as much importance as the main " "loss function." "\n\t 0 - Disables L2 Regularization altogether.") self.add_item( section=section, title="eye_multiplier", datatype=int, group="loss", min_max=(1, 40), rounding=1, default=3, fixed=False, info="The amount of priority to give to the eyes.\n\nThe value given here is as a " "multiplier of the main loss score. For example:" "\n\t 1 - The eyes will receive the same priority as the rest of the face. " "\n\t 10 - The eyes will be given a score 10 times higher than the rest of the " "face." "\n\nNB: Penalized Mask Loss must be enable to use this option.") self.add_item( section=section, title="mouth_multiplier", datatype=int, group="loss", min_max=(1, 40), rounding=1, default=2, fixed=False, info="The amount of priority to give to the mouth.\n\nThe value given here is as a " "multiplier of the main loss score. For Example:" "\n\t 1 - The mouth will receive the same priority as the rest of the face. " "\n\t 10 - The mouth will be given a score 10 times higher than the rest of the " "face." "\n\nNB: Penalized Mask Loss must be enable to use this option.") self.add_item( section=section, title="penalized_mask_loss", datatype=bool, default=True, group="loss", info="Image loss function is weighted by mask presence. For areas of " "the image without the facial mask, reconstuction errors will be " "ignored while the masked face area is prioritized. May increase " "overall quality by focusing attention on the core face area.") self.add_item( section=section, title="mask_type", datatype=str, default="extended", choices=PluginLoader.get_available_extractors("mask", add_none=True), group="mask", gui_radio=True, info="The mask to be used for training. If you have selected 'Learn Mask' or " "'Penalized Mask Loss' you must select a value other than 'none'. The required " "mask should have been selected as part of the Extract process. If it does not " "exist in the alignments file then it will be generated prior to training " "commencing." "\n\tnone: Don't use a mask." "\n\tcomponents: Mask designed to provide facial segmentation based on the " "positioning of landmark locations. A convex hull is constructed around the " "exterior of the landmarks to create a mask." "\n\textended: Mask designed to provide facial segmentation based on the " "positioning of landmark locations. A convex hull is constructed around the " "exterior of the landmarks and the mask is extended upwards onto the forehead." "\n\tvgg-clear: Mask designed to provide smart segmentation of mostly frontal " "faces clear of obstructions. Profile faces and obstructions may result in " "sub-par performance." "\n\tvgg-obstructed: Mask designed to provide smart segmentation of mostly " "frontal faces. The mask model has been specifically trained to recognize " "some facial obstructions (hands and eyeglasses). Profile faces may result in " "sub-par performance." "\n\tunet-dfl: Mask designed to provide smart segmentation of mostly frontal " "faces. The mask model has been trained by community members and will need " "testing for further description. Profile faces may result in sub-par " "performance.") self.add_item( section=section, title="mask_blur_kernel", datatype=int, min_max=(0, 9), rounding=1, default=3, group="mask", info="Apply gaussian blur to the mask input. This has the effect of smoothing the " "edges of the mask, which can help with poorly calculated masks and give less " "of a hard edge to the predicted mask. The size is in pixels (calculated from " "a 128px mask). Set to 0 to not apply gaussian blur. This value should be odd, " "if an even number is passed in then it will be rounded to the next odd number.") self.add_item( section=section, title="mask_threshold", datatype=int, default=4, min_max=(0, 50), rounding=1, group="mask", info="Sets pixels that are near white to white and near black to black. Set to 0 for " "off.") self.add_item( section=section, title="learn_mask", datatype=bool, default=False, group="mask", info="Dedicate a portion of the model to learning how to duplicate the input " "mask. Increases VRAM usage in exchange for learning a quick ability to try " "to replicate more complex mask models.")
def get_argument_list(self): argument_list = list() argument_list.append({ "opts": ("-a", "--alignments"), "action": FileFullPaths, "type": str, "group": "data", "required": True, "filetypes": "alignments", "help": "Full path to the alignments file to add the mask to. NB: if the mask already " "exists in the alignments file it will be overwritten."}) argument_list.append({ "opts": ("-i", "--input"), "action": DirOrFileFullPaths, "type": str, "group": "data", "required": True, "help": "Directory containing extracted faces, source frames, or a video file."}) argument_list.append({ "opts": ("-it", "--input-type"), "action": Radio, "type": str.lower, "choices": ("faces", "frames"), "dest": "input_type", "group": "data", "default": "frames", "help": "R|Whether the `input` is a folder of faces or a folder frames/video" "\nL|faces: The input is a folder containing extracted faces." "\nL|frames: The input is a folder containing frames or is a video"}) argument_list.append({ "opts": ("-M", "--masker"), "action": Radio, "type": str.lower, "choices": PluginLoader.get_available_extractors("mask"), "default": "extended", "group": "process", "help": "R|Masker to use." "\nL|components: Mask designed to provide facial segmentation based on the " "positioning of landmark locations. A convex hull is constructed around the " "exterior of the landmarks to create a mask." "\nL|extended: Mask designed to provide facial segmentation based on the " "positioning of landmark locations. A convex hull is constructed around the " "exterior of the landmarks and the mask is extended upwards onto the forehead." "\nL|vgg-clear: Mask designed to provide smart segmentation of mostly frontal " "faces clear of obstructions. Profile faces and obstructions may result in " "sub-par performance." "\nL|vgg-obstructed: Mask designed to provide smart segmentation of mostly " "frontal faces. The mask model has been specifically trained to recognize " "some facial obstructions (hands and eyeglasses). Profile faces may result in " "sub-par performance." "\nL|unet-dfl: Mask designed to provide smart segmentation of mostly frontal " "faces. The mask model has been trained by community members and will need " "testing for further description. Profile faces may result in sub-par " "performance."}) argument_list.append({ "opts": ("-p", "--processing"), "action": Radio, "type": str.lower, "choices": ("all", "missing", "output"), "default": "missing", "group": "process", "help": "R|Whether to update all masks in the alignments files, only those faces " "that do not already have a mask of the given `mask type` or just to output " "the masks to the `output` location." "\nL|all: Update the mask for all faces in the alignments file." "\nL|missing: Create a mask for all faces in the alignments file where a mask " "does not previously exist." "\nL|output: Don't update the masks, just output them for review in the given " "output folder."}) argument_list.append({ "opts": ("-o", "--output-folder"), "action": DirFullPaths, "dest": "output", "type": str, "group": "output", "help": "Optional output location. If provided, a preview of the masks created will " "be output in the given folder."}) argument_list.append({ "opts": ("-b", "--blur_kernel"), "action": Slider, "type": int, "group": "output", "min_max": (0, 9), "default": 3, "rounding": 1, "help": "Apply gaussian blur to the mask output. Has the effect of smoothing the " "edges of the mask giving less of a hard edge. the size is in pixels. This " "value should be odd, if an even number is passed in then it will be rounded " "to the next odd number. NB: Only effects the output preview. Set to 0 for " "off"}) argument_list.append({ "opts": ("-t", "--threshold"), "action": Slider, "type": int, "group": "output", "min_max": (0, 50), "default": 4, "rounding": 1, "help": "Helps reduce 'blotchiness' on some masks by making light shades white " "and dark shades black. Higher values will impact more of the mask. NB: " "Only effects the output preview. Set to 0 for off"}) return argument_list
def set_globals(self): """ Set the global options for training Loss Documentation MAE https://heartbeat.fritz.ai/5-regression-loss-functions-all-machine -learners-should-know-4fb140e9d4b0 MSE https://heartbeat.fritz.ai/5-regression-loss-functions-all-machine -learners-should-know-4fb140e9d4b0 LogCosh https://heartbeat.fritz.ai/5-regression-loss-functions-all-machine -learners-should-know-4fb140e9d4b0 Smooth L1 https://arxiv.org/pdf/1701.03077.pdf L_inf_norm https://medium.com/@montjoile/l0-norm-l1-norm-l2-norm-l-infinity -norm-7a7d18a4f40c SSIM http://www.cns.nyu.edu/pub/eero/wang03-reprint.pdf GMSD https://arxiv.org/ftp/arxiv/papers/1308/1308.3052.pdf """ logger.debug("Setting global config") section = "global" self.add_section(title=section, info="Options that apply to all models" + ADDITIONAL_INFO) self.add_item( section=section, title="coverage", datatype=float, default=68.75, min_max=(62.5, 100.0), rounding=2, fixed=True, group="face", info= "How much of the extracted image to train on. A lower coverage will limit the " "model's scope to a zoomed-in central area while higher amounts can include the " "entire face. A trade-off exists between lower amounts given more detail " "versus higher amounts avoiding noticeable swap transitions. Sensible values to " "use are:" "\n\t62.5%% spans from eyebrow to eyebrow." "\n\t75.0%% spans from temple to temple." "\n\t87.5%% spans from ear to ear." "\n\t100.0%% is a mugshot.") self.add_item( section=section, title="mask_type", datatype=str, default="extended", choices=PluginLoader.get_available_extractors("mask", add_none=True), group="mask", gui_radio=True, info= "The mask to be used for training. If you have selected 'Learn Mask' or " "'Penalized Mask Loss' you must select a value other than 'none'. The required " "mask should have been selected as part of the Extract process. If it does not " "exist in the alignments file then it will be generated prior to training " "commencing." "\n\tnone: Don't use a mask." "\n\tcomponents: Mask designed to provide facial segmentation based on the " "positioning of landmark locations. A convex hull is constructed around the " "exterior of the landmarks to create a mask." "\n\textended: Mask designed to provide facial segmentation based on the " "positioning of landmark locations. A convex hull is constructed around the " "exterior of the landmarks and the mask is extended upwards onto the forehead." "\n\tvgg-clear: Mask designed to provide smart segmentation of mostly frontal " "faces clear of obstructions. Profile faces and obstructions may result in " "sub-par performance." "\n\tvgg-obstructed: Mask designed to provide smart segmentation of mostly " "frontal faces. The mask model has been specifically trained to recognize " "some facial obstructions (hands and eyeglasses). Profile faces may result in " "sub-par performance." "\n\tunet-dfl: Mask designed to provide smart segmentation of mostly frontal " "faces. The mask model has been trained by community members and will need " "testing for further description. Profile faces may result in sub-par " "performance.") self.add_item( section=section, title="mask_blur_kernel", datatype=int, min_max=(0, 9), rounding=1, default=3, group="mask", info= "Apply gaussian blur to the mask input. This has the effect of smoothing the " "edges of the mask, which can help with poorly calculated masks and give less " "of a hard edge to the predicted mask. The size is in pixels (calculated from " "a 128px mask). Set to 0 to not apply gaussian blur. This value should be odd, " "if an even number is passed in then it will be rounded to the next odd number." ) self.add_item( section=section, title="mask_threshold", datatype=int, default=4, min_max=(0, 50), rounding=1, group="mask", info= "Sets pixels that are near white to white and near black to black. Set to 0 for " "off.") self.add_item( section=section, title="learn_mask", datatype=bool, default=False, group="mask", info= "Dedicate a portion of the model to learning how to duplicate the input " "mask. Increases VRAM usage in exchange for learning a quick ability to try " "to replicate more complex mask models.") self.add_item( section=section, title="penalized_mask_loss", datatype=bool, default=True, group="loss", info= "Image loss function is weighted by mask presence. For areas of " "the image without the facial mask, reconstuction errors will be " "ignored while the masked face area is prioritized. May increase " "overall quality by focusing attention on the core face area.") self.add_item( section=section, title="loss_function", datatype=str, group="loss", default="mae", choices=[ "mae", "mse", "logcosh", "smooth_loss", "l_inf_norm", "ssim", "gmsd", "pixel_gradient_diff" ], info="The loss function to use." "\n\t MAE - Mean absolute error will guide reconstructions of each pixel " "towards its median value in the training dataset. Robust to outliers but as " "a median, it can potentially ignore some infrequent image types in the dataset." "\n\t MSE - Mean squared error will guide reconstructions of each pixel " "towards its average value in the training dataset. As an avg, it will be " "suspectible to outliers and typically produces slightly blurrier results." "\n\t LogCosh - log(cosh(x)) acts similiar to MSE for small errors and to " "MAE for large errors. Like MSE, it is very stable and prevents overshoots " "when errors are near zero. Like MAE, it is robust to outliers. NB: Due to a bug " "in PlaidML, this loss does not work on AMD cards." "\n\t Smooth_L1 --- Modification of the MAE loss to correct two of its " "disadvantages. This loss has improved stability and guidance for small errors." "\n\t L_inf_norm --- The L_inf norm will reduce the largest individual pixel " "error in an image. As each largest error is minimized sequentially, the " "overall error is improved. This loss will be extremely focused on outliers." "\n\t SSIM - Structural Similarity Index Metric is a perception-based " "loss that considers changes in texture, luminance, contrast, and local spatial " "statistics of an image. Potentially delivers more realistic looking images." "\n\t GMSD - Gradient Magnitude Similarity Deviation seeks to match " "the global standard deviation of the pixel to pixel differences between two " "images. Similiar in approach to SSIM. NB: This loss does not currently work on " "AMD cards." "\n\t Pixel_Gradient_Difference - Instead of minimizing the difference between " "the absolute value of each pixel in two reference images, compute the pixel to " "pixel spatial difference in each image and then minimize that difference " "between two images. Allows for large color shifts,but maintains the structure " "of the image.\n") self.add_item( section=section, title="icnr_init", datatype=bool, default=False, group="initialization", info= "Use ICNR to tile the default initializer in a repeating pattern. " "This strategy is designed for pairing with sub-pixel / pixel shuffler " "to reduce the 'checkerboard effect' in image reconstruction. " "\n\t https://arxiv.org/ftp/arxiv/papers/1707/1707.02937.pdf") self.add_item( section=section, title="conv_aware_init", datatype=bool, default=False, group="initialization", info= "Use Convolution Aware Initialization for convolutional layers. " "This can help eradicate the vanishing and exploding gradient problem " "as well as lead to higher accuracy, lower loss and faster convergence.\nNB:" "\n\t This can use more VRAM when creating a new model so you may want to " "lower the batch size for the first run. The batch size can be raised " "again when reloading the model. " "\n\t Multi-GPU is not supported for this option, so you should start the model " "on a single GPU. Once training has started, you can stop training, enable " "multi-GPU and resume." "\n\t Building the model will likely take several minutes as the calculations " "for this initialization technique are expensive. This will only impact starting " "a new model.") self.add_item( section=section, title="optimizer", datatype=str, gui_radio=True, group="optimizer", default="adam", choices=["adam", "nadam", "rms-prop"], info="The optimizer to use." "\n\t adam - Adaptive Moment Optimization. A stochastic gradient descent method " "that is based on adaptive estimation of first-order and second-order moments." "\n\t nadam - Adaptive Moment Optimization with Nesterov Momentum. Much like " "Adam but uses a different formula for calculating momentum." "\n\t rms-prop - Root Mean Square Propogation. Maintains a moving (discounted) " "average of the square of the gradients. Divides the gradient by the root of " "this average.") self.add_item( section=section, title="learning_rate", datatype=float, default=5e-5, min_max=(1e-6, 1e-4), rounding=6, fixed=False, group="optimizer", info= "Learning rate - how fast your network will learn (how large are the " "modifications to the model weights after one batch of training). Values that " "are too large might result in model crashes and the inability of the model to " "find the best solution. Values that are too small might be unable to escape " "from dead-ends and find the best global minimum.") self.add_item( section=section, title="reflect_padding", datatype=bool, default=False, group="network", info= "Use reflection padding rather than zero padding with convolutions. " "Each convolution must pad the image boundaries to maintain the proper " "sizing. More complex padding schemes can reduce artifacts at the " "border of the image." "\n\t http://www-cs.engr.ccny.cuny.edu/~wolberg/cs470/hw/hw2_pad.txt" ) self.add_item( section=section, title="allow_growth", datatype=bool, default=False, group="network", fixed=False, info= "[Nvidia Only]. Enable the Tensorflow GPU 'allow_growth' configuration option. " "This option prevents Tensorflow from allocating all of the GPU VRAM at launch " "but can lead to higher VRAM fragmentation and slower performance. Should only " "be enabled if you are receiving errors regarding 'cuDNN fails to initialize' " "when commencing training.") self.add_item( section=section, title="mixed_precision", datatype=bool, default=False, group="network", info= "R|[Nvidia Only], NVIDIA GPUs can run operations in float16 faster than in " "float32. Mixed precision allows you to use a mix of float16 with float32, to " "get the performance benefits from float16 and the numeric stability benefits " "from float32.\nWhile mixed precision will run on most Nvidia models, it will " "only speed up training on more recent GPUs. Those with compute capability 7.0 " "or higher will see the greatest performance benefit from mixed precision " "because they have Tensor Cores. Older GPUs offer no math performance benefit " "for using mixed precision, however memory and bandwidth savings can enable some " "speedups. Generally RTX GPUs and later will offer the most benefit." ) self.add_item( section=section, title="convert_batchsize", datatype=int, default=16, min_max=(1, 32), rounding=1, fixed=False, group="convert", info= "[GPU Only]. The number of faces to feed through the model at once when running " "the Convert process.\n\nNB: Increasing this figure is unlikely to improve " "convert speed, however, if you are getting Out of Memory errors, then you may " "want to reduce the batch size.")
def _set_loss(self) -> None: # pylint:disable=line-too-long """ Set the default loss options. Loss Documentation MAE https://heartbeat.fritz.ai/5-regression-loss-functions-all-machine-learners-should-know-4fb140e9d4b0 MSE https://heartbeat.fritz.ai/5-regression-loss-functions-all-machine-learners-should-know-4fb140e9d4b0 LogCosh https://heartbeat.fritz.ai/5-regression-loss-functions-all-machine-learners-should-know-4fb140e9d4b0 Smooth L1 https://arxiv.org/pdf/1701.03077.pdf L_inf_norm https://medium.com/@montjoile/l0-norm-l1-norm-l2-norm-l-infinity-norm-7a7d18a4f40c SSIM http://www.cns.nyu.edu/pub/eero/wang03-reprint.pdf MSSIM https://www.cns.nyu.edu/pub/eero/wang03b.pdf GMSD https://arxiv.org/ftp/arxiv/papers/1308/1308.3052.pdf """ # noqa # pylint:enable=line-too-long logger.debug("Setting Loss config") section = "global.loss" self.add_section( title=section, info="Loss configuration options\n" "Loss is the mechanism by which a Neural Network judges how well it " "thinks that it is recreating a face." + ADDITIONAL_INFO) self.add_item(section=section, title="loss_function", datatype=str, group="loss", default="ssim", fixed=False, choices=[ x for x in sorted(_LOSS_HELP) if x not in _NON_PRIMARY_LOSS ], info="The loss function to use.\n\n\t" + "\n\t".join(f"{k}: {v}" for k, v in sorted(_LOSS_HELP.items()) if k not in _NON_PRIMARY_LOSS)) self.add_item( section=section, title="loss_function_2", datatype=str, group="loss", default="mse", fixed=False, choices=list(sorted(_LOSS_HELP)), info= "The second loss function to use. If using a structural based loss (such as " "SSIM, MS-SSIM or GMSD) it is common to add an L1 regularization(MAE) or L2 " "regularization (MSE) function. You can adjust the weighting of this loss " "function with the loss_weight_2 option.\n\n\t" + "\n\t".join(f"{k}: {v}" for k, v in sorted(_LOSS_HELP.items()))) self.add_item( section=section, title="loss_weight_2", datatype=int, group="loss", min_max=(0, 400), rounding=1, default=100, fixed=False, info="The amount of weight to apply to the second loss function.\n\n" "\n\nThe value given here is as a percentage denoting how much the selected " "function should contribute to the overall loss cost of the model. For example:" "\n\t 100 - The loss calculated for the second loss function will be applied at " "its full amount towards the overall loss score. " "\n\t 25 - The loss calculated for the second loss function will be reduced by a " "quarter prior to adding to the overall loss score. " "\n\t 400 - The loss calculated for the second loss function will be mulitplied " "4 times prior to adding to the overall loss score. " "\n\t 0 - Disables the second loss function altogether.") self.add_item( section=section, title="loss_function_3", datatype=str, group="loss", default="none", fixed=False, choices=list(sorted(_LOSS_HELP)), info= "The third loss function to use. You can adjust the weighting of this loss " "function with the loss_weight_3 option.\n\n\t" + "\n\t".join(f"{k}: {v}" for k, v in sorted(_LOSS_HELP.items()))) self.add_item( section=section, title="loss_weight_3", datatype=int, group="loss", min_max=(0, 400), rounding=1, default=0, fixed=False, info="The amount of weight to apply to the third loss function.\n\n" "\n\nThe value given here is as a percentage denoting how much the selected " "function should contribute to the overall loss cost of the model. For example:" "\n\t 100 - The loss calculated for the third loss function will be applied at " "its full amount towards the overall loss score. " "\n\t 25 - The loss calculated for the third loss function will be reduced by a " "quarter prior to adding to the overall loss score. " "\n\t 400 - The loss calculated for the third loss function will be mulitplied 4 " "times prior to adding to the overall loss score. " "\n\t 0 - Disables the third loss function altogether.") self.add_item( section=section, title="loss_function_4", datatype=str, group="loss", default="none", fixed=False, choices=list(sorted(_LOSS_HELP)), info= "The fourth loss function to use. You can adjust the weighting of this loss " "function with the loss_weight_3 option.\n\n\t" + "\n\t".join(f"{k}: {v}" for k, v in sorted(_LOSS_HELP.items()))) self.add_item( section=section, title="loss_weight_4", datatype=int, group="loss", min_max=(0, 400), rounding=1, default=0, fixed=False, info="The amount of weight to apply to the fourth loss function.\n\n" "\n\nThe value given here is as a percentage denoting how much the selected " "function should contribute to the overall loss cost of the model. For example:" "\n\t 100 - The loss calculated for the fourth loss function will be applied at " "its full amount towards the overall loss score. " "\n\t 25 - The loss calculated for the fourth loss function will be reduced by a " "quarter prior to adding to the overall loss score. " "\n\t 400 - The loss calculated for the fourth loss function will be mulitplied " "4 times prior to adding to the overall loss score. " "\n\t 0 - Disables the fourth loss function altogether.") self.add_item( section=section, title="mask_loss_function", datatype=str, group="loss", default="mse", fixed=False, choices=["mae", "mse"], info="The loss function to use when learning a mask." "\n\t MAE - Mean absolute error will guide reconstructions of each pixel " "towards its median value in the training dataset. Robust to outliers but as " "a median, it can potentially ignore some infrequent image types in the dataset." "\n\t MSE - Mean squared error will guide reconstructions of each pixel " "towards its average value in the training dataset. As an average, it will be " "susceptible to outliers and typically produces slightly blurrier results." ) self.add_item( section=section, title="eye_multiplier", datatype=int, group="loss", min_max=(1, 40), rounding=1, default=3, fixed=False, info= "The amount of priority to give to the eyes.\n\nThe value given here is as a " "multiplier of the main loss score. For example:" "\n\t 1 - The eyes will receive the same priority as the rest of the face. " "\n\t 10 - The eyes will be given a score 10 times higher than the rest of the " "face." "\n\nNB: Penalized Mask Loss must be enable to use this option.") self.add_item( section=section, title="mouth_multiplier", datatype=int, group="loss", min_max=(1, 40), rounding=1, default=2, fixed=False, info= "The amount of priority to give to the mouth.\n\nThe value given here is as a " "multiplier of the main loss score. For Example:" "\n\t 1 - The mouth will receive the same priority as the rest of the face. " "\n\t 10 - The mouth will be given a score 10 times higher than the rest of the " "face." "\n\nNB: Penalized Mask Loss must be enable to use this option.") self.add_item( section=section, title="penalized_mask_loss", datatype=bool, default=True, group="loss", info= "Image loss function is weighted by mask presence. For areas of " "the image without the facial mask, reconstruction errors will be " "ignored while the masked face area is prioritized. May increase " "overall quality by focusing attention on the core face area.") self.add_item( section=section, title="mask_type", datatype=str, default="extended", choices=PluginLoader.get_available_extractors("mask", add_none=True, extend_plugin=True), group="mask", gui_radio=True, info= "The mask to be used for training. If you have selected 'Learn Mask' or " "'Penalized Mask Loss' you must select a value other than 'none'. The required " "mask should have been selected as part of the Extract process. If it does not " "exist in the alignments file then it will be generated prior to training " "commencing." "\n\tnone: Don't use a mask." "\n\tbisenet-fp-face: Relatively lightweight NN based mask that provides more " "refined control over the area to be masked (configurable in mask settings). " "Use this version of bisenet-fp if your model is trained with 'face' or " "'legacy' centering." "\n\tbisenet-fp-head: Relatively lightweight NN based mask that provides more " "refined control over the area to be masked (configurable in mask settings). " "Use this version of bisenet-fp if your model is trained with 'head' centering." "\n\tcomponents: Mask designed to provide facial segmentation based on the " "positioning of landmark locations. A convex hull is constructed around the " "exterior of the landmarks to create a mask." "\n\textended: Mask designed to provide facial segmentation based on the " "positioning of landmark locations. A convex hull is constructed around the " "exterior of the landmarks and the mask is extended upwards onto the forehead." "\n\tvgg-clear: Mask designed to provide smart segmentation of mostly frontal " "faces clear of obstructions. Profile faces and obstructions may result in " "sub-par performance." "\n\tvgg-obstructed: Mask designed to provide smart segmentation of mostly " "frontal faces. The mask model has been specifically trained to recognize " "some facial obstructions (hands and eyeglasses). Profile faces may result in " "sub-par performance." "\n\tunet-dfl: Mask designed to provide smart segmentation of mostly frontal " "faces. The mask model has been trained by community members and will need " "testing for further description. Profile faces may result in sub-par " "performance.") self.add_item( section=section, title="mask_blur_kernel", datatype=int, min_max=(0, 9), rounding=1, default=3, group="mask", info= "Apply gaussian blur to the mask input. This has the effect of smoothing the " "edges of the mask, which can help with poorly calculated masks and give less " "of a hard edge to the predicted mask. The size is in pixels (calculated from " "a 128px mask). Set to 0 to not apply gaussian blur. This value should be odd, " "if an even number is passed in then it will be rounded to the next odd number." ) self.add_item( section=section, title="mask_threshold", datatype=int, default=4, min_max=(0, 50), rounding=1, group="mask", info= "Sets pixels that are near white to white and near black to black. Set to 0 for " "off.") self.add_item( section=section, title="learn_mask", datatype=bool, default=False, group="mask", info= "Dedicate a portion of the model to learning how to duplicate the input " "mask. Increases VRAM usage in exchange for learning a quick ability to try " "to replicate more complex mask models.")
def get_optional_arguments(): """ Put the arguments in a list so that they are accessible from both argparse and gui """ argument_list = [] argument_list.append({ "opts": ("--serializer", ), "type": str.lower, "dest": "serializer", "default": "json", "choices": ("json", "pickle", "yaml"), "help": "Serializer for alignments file. If " "yaml is chosen and not available, then " "json will be used as the default " "fallback." }) argument_list.append({ "opts": ("-D", "--detector"), "type": str, "choices": PluginLoader.get_available_extractors("detect"), "default": "mtcnn", "help": "R|Detector to use." "\n'dlib-hog': uses least resources, but is the" "\n\tleast reliable." "\n'dlib-cnn': faster than mtcnn but detects" "\n\tfewer faces and fewer false positives." "\n'mtcnn': slower than dlib, but uses fewer" "\n\tresources whilst detecting more faces and" "\n\tmore false positives. Has superior" "\n\talignment to dlib" }) argument_list.append({ "opts": ("-A", "--aligner"), "type": str, "choices": PluginLoader.get_available_extractors("align"), "default": "fan", "help": "R|Aligner to use." "\n'dlib': Dlib Pose Predictor. Faster, less " "\n\tresource intensive, but less accurate." "\n'fan': Face Alignment Network. Best aligner." "\n\tGPU heavy." }) argument_list.append({ "opts": ("-mtms", "--mtcnn-minsize"), "type": int, "dest": "mtcnn_minsize", "default": 20, "help": "The minimum size of a face to be " "accepted. Lower values use " "significantly more VRAM. Minimum " "value is 10. Default is 20 " "(MTCNN detector only)" }) argument_list.append({ "opts": ("-mtth", "--mtcnn-threshold"), "nargs": "+", "type": str, "dest": "mtcnn_threshold", "default": ["0.6", "0.7", "0.7"], "help": "R|Three step threshold for face " "detection. Should be\nthree decimal " "numbers each less than 1. Eg:\n" "'--mtcnn-threshold 0.6 0.7 0.7'.\n" "1st stage: obtains face candidates.\n" "2nd stage: refinement of face " "candidates.\n3rd stage: further " "refinement of face candidates.\n" "Default is 0.6 0.7 0.7 " "(MTCNN detector only)" }) argument_list.append({ "opts": ("-mtsc", "--mtcnn-scalefactor"), "type": float, "dest": "mtcnn_scalefactor", "default": 0.709, "help": "The scale factor for the image " "pyramid. Should be a decimal number " "less than one. Default is 0.709 " "(MTCNN detector only)" }) argument_list.append({ "opts": ("-r", "--rotate-images"), "type": str, "dest": "rotate_images", "default": None, "help": "If a face isn't found, rotate the " "images to try to find a face. Can find " "more faces at the cost of extraction " "speed. Pass in a single number to use " "increments of that size up to 360, or " "pass in a list of numbers to enumerate " "exactly what angles to check" }) argument_list.append({ "opts": ("-bt", "--blur-threshold"), "type": int, "dest": "blur_thresh", "default": None, "help": "Automatically discard images blurrier " "than the specified threshold. " "Discarded images are moved into a " "\"blurry\" sub-folder. Lower values " "allow more blur" }) argument_list.append({ "opts": ("-mp", "--multiprocess"), "action": "store_true", "default": False, "help": "Run extraction in parallel. Offers " "speed up for some extractor/detector " "combinations, less so for others. " "Only has an effect if both the " "aligner and detector use the GPU, " "otherwise this is automatic." }) argument_list.append({ "opts": ("-sz", "--size"), "type": int, "default": 256, "help": "The output size of extracted faces. " "Make sure that the model you intend " "to train supports your required " "size. This will only need to be " "changed for hi-res models." }) argument_list.append({ "opts": ("-s", "--skip-existing"), "action": "store_true", "dest": "skip_existing", "default": False, "help": "Skips frames that have already been " "extracted and exist in the alignments " "file" }) argument_list.append({ "opts": ("-sf", "--skip-existing-faces"), "action": "store_true", "dest": "skip_faces", "default": False, "help": "Skip frames that already have " "detected faces in the alignments " "file" }) argument_list.append({ "opts": ("-dl", "--debug-landmarks"), "action": "store_true", "dest": "debug_landmarks", "default": False, "help": "Draw landmarks on the ouput faces for " "debug" }) argument_list.append({ "opts": ("-ae", "--align-eyes"), "action": "store_true", "dest": "align_eyes", "default": False, "help": "Perform extra alignment to ensure " "left/right eyes are at the same " "height" }) argument_list.append({ "opts": ("-si", "--save-interval"), "dest": "save_interval", "type": int, "default": None, "help": "Automatically save the alignments file " "after a set amount of frames. Will " "only save at the end of extracting by " "default. WARNING: Don't interrupt the " "script when writing the file because " "it might get corrupted." }) return argument_list
def get_argument_list(self): argument_list = list() argument_list.append(dict( opts=("-a", "--alignments"), action=FileFullPaths, type=str, group=_("data"), required=True, filetypes="alignments", help=_("Full path to the alignments file to add the mask to. NB: if the mask already " "exists in the alignments file it will be overwritten."))) argument_list.append(dict( opts=("-i", "--input"), action=DirOrFileFullPaths, type=str, group=_("data"), filetypes="video", required=True, help=_("Directory containing extracted faces, source frames, or a video file."))) argument_list.append(dict( opts=("-it", "--input-type"), action=Radio, type=str.lower, choices=("faces", "frames"), dest="input_type", group=_("data"), default="frames", help=_("R|Whether the `input` is a folder of faces or a folder frames/video" "\nL|faces: The input is a folder containing extracted faces." "\nL|frames: The input is a folder containing frames or is a video"))) argument_list.append(dict( opts=("-M", "--masker"), action=Radio, type=str.lower, choices=PluginLoader.get_available_extractors("mask"), default="extended", group=_("process"), help=_("R|Masker to use." "\nL|bisenet-fp: Relatively lightweight NN based mask that provides more " "refined control over the area to be masked including full head masking " "(configurable in mask settings)." "\nL|components: Mask designed to provide facial segmentation based on the " "positioning of landmark locations. A convex hull is constructed around the " "exterior of the landmarks to create a mask." "\nL|extended: Mask designed to provide facial segmentation based on the " "positioning of landmark locations. A convex hull is constructed around the " "exterior of the landmarks and the mask is extended upwards onto the forehead." "\nL|vgg-clear: Mask designed to provide smart segmentation of mostly frontal " "faces clear of obstructions. Profile faces and obstructions may result in " "sub-par performance." "\nL|vgg-obstructed: Mask designed to provide smart segmentation of mostly " "frontal faces. The mask model has been specifically trained to recognize " "some facial obstructions (hands and eyeglasses). Profile faces may result in " "sub-par performance." "\nL|unet-dfl: Mask designed to provide smart segmentation of mostly frontal " "faces. The mask model has been trained by community members and will need " "testing for further description. Profile faces may result in sub-par " "performance."))) argument_list.append(dict( opts=("-p", "--processing"), action=Radio, type=str.lower, choices=("all", "missing", "output"), default="missing", group=_("process"), help=_("R|Whether to update all masks in the alignments files, only those faces " "that do not already have a mask of the given `mask type` or just to output " "the masks to the `output` location." "\nL|all: Update the mask for all faces in the alignments file." "\nL|missing: Create a mask for all faces in the alignments file where a mask " "does not previously exist." "\nL|output: Don't update the masks, just output them for review in the given " "output folder."))) argument_list.append(dict( opts=("-o", "--output-folder"), action=DirFullPaths, dest="output", type=str, group=_("output"), help=_("Optional output location. If provided, a preview of the masks created will " "be output in the given folder."))) argument_list.append(dict( opts=("-b", "--blur_kernel"), action=Slider, type=int, group=_("output"), min_max=(0, 9), default=3, rounding=1, help=_("Apply gaussian blur to the mask output. Has the effect of smoothing the " "edges of the mask giving less of a hard edge. the size is in pixels. This " "value should be odd, if an even number is passed in then it will be rounded " "to the next odd number. NB: Only effects the output preview. Set to 0 for " "off"))) argument_list.append(dict( opts=("-t", "--threshold"), action=Slider, type=int, group=_("output"), min_max=(0, 50), default=4, rounding=1, help=_("Helps reduce 'blotchiness' on some masks by making light shades white " "and dark shades black. Higher values will impact more of the mask. NB: " "Only effects the output preview. Set to 0 for off"))) argument_list.append(dict( opts=("-ot", "--output-type"), action=Radio, type=str.lower, choices=("combined", "masked", "mask"), default="combined", group=_("output"), help=_("R|How to format the output when processing is set to 'output'." "\nL|combined: The image contains the face/frame, face mask and masked face." "\nL|masked: Output the face/frame as rgba image with the face masked." "\nL|mask: Only output the mask as a single channel image."))) argument_list.append(dict( opts=("-f", "--full-frame"), action="store_true", default=False, group=_("output"), help=_("R|Whether to output the whole frame or only the face box when using " "output processing. Only has an effect when using frames as input."))) return argument_list