def process(self):
""" Call the training process object """
logger.debug("Starting Training Process")
logger.info("Training data directory: %s", self.args.model_dir)
# TODO Move these args to config and remove these deprecation warnings
if hasattr(self.args,
"warp_to_landmarks") and self.args.warp_to_landmarks:
deprecation_warning(
"`-wl`, ``--warp-to-landmarks``",
additional_info="This option will be available within training "
"config settings (/config/train.ini).")
if hasattr(self.args, "no_augment_color") and self.args.no_flip:
deprecation_warning(
"`-nac`, ``--no-augment-color``",
additional_info="This option will be available within training "
"config settings (/config/train.ini).")
set_system_verbosity(self.args.loglevel)
thread = self.start_thread()
# queue_manager.debug_monitor(1)
err = self.monitor(thread)
self.end_thread(thread, err)
logger.debug("Completed Training Process")
def process(self):
""" The entry point for triggering the Training Process.
Should only be called from :class:`lib.cli.ScriptExecutor`
"""
logger.debug("Starting Training Process")
logger.info("Training data directory: %s", self._args.model_dir)
# TODO Move these args to config and remove these deprecation warnings
if hasattr(self._args,
"warp_to_landmarks") and self._args.warp_to_landmarks:
deprecation_warning(
"`-wl`, ``--warp-to-landmarks``",
additional_info="This option will be available within training "
"config settings (/config/train.ini).")
if hasattr(self._args, "no_augment_color") and self._args.no_flip:
deprecation_warning(
"`-nac`, ``--no-augment-color``",
additional_info="This option will be available within training "
"config settings (/config/train.ini).")
thread = self._start_thread()
# from lib.queue_manager import queue_manager; queue_manager.debug_monitor(1)
err = self._monitor(thread)
self._end_thread(thread, err)
logger.debug("Completed Training Process")
def __init__(self,
loglevel,
configfile=None,
git_model_id=None,
model_filename=None,
rotation=None,
min_size=0):
logger.debug(
"Initializing %s: (loglevel: %s, configfile: %s, git_model_id: %s, "
"model_filename: %s, rotation: %s, min_size: %s)",
self.__class__.__name__, loglevel, configfile, git_model_id,
model_filename, rotation, min_size)
self.config = get_config(".".join(self.__module__.split(".")[-2:]),
configfile=configfile)
self.loglevel = loglevel
self.rotation = self.get_rotation_angles(rotation)
self.min_size = min_size
self.parent_is_pool = False
self.init = None
self.error = None
# The input and output queues for the plugin.
# See lib.queue_manager.QueueManager for getting queues
self.queues = {"in": None, "out": None}
# Path to model if required
self.model_path = self.get_model(git_model_id, model_filename)
# Target image size for passing images through the detector
# Set to tuple of dimensions (x, y) or int of pixel count
self.target = None
# Approximate VRAM used for the set target. Used to calculate
# how many parallel processes / batches can be run.
# Be conservative to avoid OOM.
self.vram = None
# Set to true if the plugin supports PlaidML
self.supports_plaidml = False
# For detectors that support batching, this should be set to
# the calculated batch size that the amount of available VRAM
# will support. It is also used for holding the number of threads/
# processes for parallel processing plugins
self.batch_size = 1
if rotation is not None:
deprecation_warning(
"Rotation ('-r', '--rotation')",
additional_info=
"It is not necessary for most detectors and will "
"be moved to plugin config for those detectors "
"that require it.")
logger.debug("Initialized _base %s", self.__class__.__name__)
def run_extraction(self):
""" Run Face Detection """
to_process = self.process_item_count()
size = self.args.size if hasattr(self.args, "size") else 256
align_eyes = self.args.align_eyes if hasattr(self.args,
"align_eyes") else False
if align_eyes:
deprecation_warning(
"Align eyes (-ae --align-eyes)",
additional_info="This functionality will still be available "
"within the alignments tool.")
exception = False
for phase in range(self.extractor.passes):
if exception:
break
is_final = self.extractor.final_pass
detected_faces = dict()
self.extractor.launch()
self.check_thread_error()
for idx, faces in enumerate(
tqdm(self.extractor.detected_faces(),
total=to_process,
file=sys.stdout,
desc="Running pass {} of {}: {}".format(
phase + 1, self.extractor.passes,
self.extractor.phase.title()))):
self.check_thread_error()
exception = faces.get("exception", False)
if exception:
break
filename = faces["filename"]
if self.extractor.final_pass:
self.output_processing(faces, align_eyes, size, filename)
self.output_faces(filename, faces)
if self.save_interval and (idx +
1) % self.save_interval == 0:
self.alignments.save()
else:
del faces["image"]
detected_faces[filename] = faces
if is_final:
logger.debug("Putting EOF to save")
self.save_queue.put("EOF")
else:
logger.debug("Reloading images")
self.threaded_io("reload", detected_faces)
def __init__(self,
model_dir,
gpus=1,
configfile=None,
snapshot_interval=0,
no_logs=False,
warp_to_landmarks=False,
augment_color=True,
no_flip=False,
training_image_size=256,
alignments_paths=None,
preview_scale=100,
input_shape=None,
encoder_dim=None,
trainer="original",
pingpong=False,
memory_saving_gradients=False,
optimizer_savings=False,
predict=False):
logger.debug(
"Initializing ModelBase (%s): (model_dir: '%s', gpus: %s, configfile: %s, "
"snapshot_interval: %s, no_logs: %s, warp_to_landmarks: %s, augment_color: "
"%s, no_flip: %s, training_image_size, %s, alignments_paths: %s, "
"preview_scale: %s, input_shape: %s, encoder_dim: %s, trainer: %s, "
"pingpong: %s, memory_saving_gradients: %s, optimizer_savings: %s, "
"predict: %s)", self.__class__.__name__, model_dir, gpus,
configfile, snapshot_interval, no_logs, warp_to_landmarks,
augment_color, no_flip, training_image_size, alignments_paths,
preview_scale, input_shape, encoder_dim, trainer, pingpong,
memory_saving_gradients, optimizer_savings, predict)
self.predict = predict
self.model_dir = model_dir
self.vram_savings = VRAMSavings(pingpong, optimizer_savings,
memory_saving_gradients)
self.backup = Backup(self.model_dir, self.name)
self.gpus = gpus
self.configfile = configfile
self.input_shape = input_shape
self.encoder_dim = encoder_dim
self.trainer = trainer
self.load_config(
) # Load config if plugin has not already referenced it
self.state = State(self.model_dir, self.name,
self.config_changeable_items, no_logs,
self.vram_savings.pingpong, training_image_size)
self.blocks = NNBlocks(
use_subpixel=self.config["subpixel_upscaling"],
use_icnr_init=self.config["icnr_init"],
use_convaware_init=self.config["conv_aware_init"],
use_reflect_padding=self.config["reflect_padding"],
first_run=self.state.first_run)
self.is_legacy = False
self.rename_legacy()
self.load_state_info()
self.networks = dict() # Networks for the model
self.predictors = dict() # Predictors for model
self.history = dict() # Loss history per save iteration)
# Training information specific to the model should be placed in this
# dict for reference by the trainer.
self.training_opts = {
"alignments":
alignments_paths,
"preview_scaling":
preview_scale / 100,
"warp_to_landmarks":
warp_to_landmarks,
"augment_color":
augment_color,
"no_flip":
no_flip,
"pingpong":
self.vram_savings.pingpong,
"snapshot_interval":
snapshot_interval,
"training_size":
self.state.training_size,
"no_logs":
self.state.current_session["no_logs"],
"coverage_ratio":
self.calculate_coverage_ratio(),
"mask_type":
self.config["mask_type"],
"mask_blur_kernel":
self.config["mask_blur_kernel"],
"mask_threshold":
self.config["mask_threshold"],
"learn_mask": (self.config["learn_mask"]
and self.config["mask_type"] is not None),
"penalized_mask_loss": (self.config["penalized_mask_loss"]
and self.config["mask_type"] is not None)
}
logger.debug("training_opts: %s", self.training_opts)
if self.multiple_models_in_folder:
deprecation_warning(
"Support for multiple model types within the same folder",
additional_info=
"Please split each model into separate folders to "
"avoid issues in future.")
self.build()
logger.debug("Initialized ModelBase (%s)", self.__class__.__name__)
def __init__(self,
model_dir,
gpus,
configfile=None,
snapshot_interval=0,
no_logs=False,
warp_to_landmarks=False,
augment_color=True,
no_flip=False,
training_image_size=256,
alignments_paths=None,
preview_scale=100,
input_shape=None,
encoder_dim=None,
trainer="original",
pingpong=False,
memory_saving_gradients=False,
predict=False):
logger.debug(
"Initializing ModelBase (%s): (model_dir: '%s', gpus: %s, configfile: %s, "
"snapshot_interval: %s, no_logs: %s, warp_to_landmarks: %s, augment_color: "
"%s, no_flip: %s, training_image_size, %s, alignments_paths: %s, "
"preview_scale: %s, input_shape: %s, encoder_dim: %s, trainer: %s, "
"pingpong: %s, memory_saving_gradients: %s, predict: %s)",
self.__class__.__name__, model_dir, gpus, configfile,
snapshot_interval, no_logs, warp_to_landmarks, augment_color,
no_flip, training_image_size, alignments_paths, preview_scale,
input_shape, encoder_dim, trainer, pingpong,
memory_saving_gradients, predict)
self.predict = predict
self.model_dir = model_dir
self.backup = Backup(self.model_dir, self.name)
self.gpus = gpus
self.configfile = configfile
self.blocks = NNBlocks(
use_subpixel=self.config["subpixel_upscaling"],
use_icnr_init=self.config["icnr_init"],
use_reflect_padding=self.config["reflect_padding"])
self.input_shape = input_shape
self.output_shape = None # set after model is compiled
self.encoder_dim = encoder_dim
self.trainer = trainer
self.state = State(self.model_dir, self.name,
self.config_changeable_items, no_logs, pingpong,
training_image_size)
self.is_legacy = False
self.rename_legacy()
self.load_state_info()
self.networks = dict() # Networks for the model
self.predictors = dict() # Predictors for model
self.history = dict() # Loss history per save iteration)
# Training information specific to the model should be placed in this
# dict for reference by the trainer.
self.training_opts = {
"alignments": alignments_paths,
"preview_scaling": preview_scale / 100,
"warp_to_landmarks": warp_to_landmarks,
"augment_color": augment_color,
"no_flip": no_flip,
"pingpong": pingpong,
"snapshot_interval": snapshot_interval
}
self.set_gradient_type(memory_saving_gradients)
if self.multiple_models_in_folder:
deprecation_warning(
"Support for multiple model types within the same folder",
additional_info=
"Please split each model into separate folders to "
"avoid issues in future.")
self.build()
self.set_training_data()
logger.debug("Initialized ModelBase (%s)", self.__class__.__name__)
