def __init__(self, args, **kwargs):
""" Initializes a util class for vaidation. """
super(Validation, self).__init__(**kwargs)
self._tb_log_dir = args["tb_log_dir"]
self._waiting_interval = args["waiting_interval"]
self._maximum_waiting_time = args["maximum_waiting_time"]
self._validator = build_validator(args)
def __init__(self, args, **kwargs):
""" Initializes a util class for training neural models. """
super(Trainer, self).__init__(**kwargs)
self._tb_log_dir = args["tb_log_dir"]
self._train_steps = args["train_steps"]
self._summary_steps = args["summary_steps"]
self._save_checkpoint_steps = args["save_checkpoint_steps"]
self._checkpoints_max_to_keep = args["checkpoints_max_to_keep"]
self._initial_global_step = args["initial_global_step"]
self._pretrain_variable_pattern = args["pretrain_variable_pattern"]
if args["pretrain_model"] and isinstance(args["pretrain_model"][0],
dict):
self._pretrain_v2 = True
self._pretrain_model = args["pretrain_model"]
if self._pretrain_variable_pattern:
logging.info(
"Using pretrain model v2 and ignoring pretrain_variable_pattern: "
f"{self._pretrain_variable_pattern}")
else:
self._pretrain_v2 = False
self._pretrain_model = flatten_string_list(args["pretrain_model"])
if self._pretrain_model and self._pretrain_variable_pattern is None:
self._pretrain_variable_pattern = [None] * len(
self._pretrain_model)
assert (
(self._pretrain_model is None
and self._pretrain_variable_pattern is None)
or len(self._pretrain_model) == len(
self._pretrain_variable_pattern)
or len(self._pretrain_model) == 1
), ("`pretrain_variable_pattern` must match with `pretrain_model`."
)
if self._pretrain_model is not None and self._pretrain_variable_pattern is None:
self._pretrain_variable_pattern = [None] * len(
self._pretrain_model)
self._update_cycle = args["update_cycle"]
self._clip_value = args["clip_value"]
self._clip_norm = args["clip_norm"]
self._hvd_backend = self.strategy if self.strategy in [
"byteps", "horovod"
] else None
with training_utils.get_strategy_scope(self.strategy):
self._criterion = build_criterion(args)
self._criterion.set_model(self.model)
self._lr_schedule_args = args
if compat.IS_PREV_TF_2_4_0:
self._optimizer = build_optimizer(args)
else:
self._optimizer = build_optimizer(args,
clipnorm=self._clip_norm,
clipvalue=self._clip_value)
assert self._optimizer is not None, "optimizer parameters must be provided for training."
self._validator = build_validator(args)
self._experimental_count_batch_num = args[
"experimental_count_batch_num"]
self._freeze_variables = args["freeze_variables"]
def __init__(self, args, **kwargs):
""" Initializes a util class for training neural models. """
super(Trainer, self).__init__(**kwargs)
self._tb_log_dir = args["tb_log_dir"]
self._train_steps = args["train_steps"]
self._summary_steps = args["summary_steps"]
self._save_checkpoint_steps = args["save_checkpoint_steps"]
self._checkpoints_max_to_keep = args["checkpoints_max_to_keep"]
self._initial_global_step = args["initial_global_step"]
self._pretrain_model = flatten_string_list(args["pretrain_model"])
self._pretrain_variable_pattern = args["pretrain_variable_pattern"]
if self._pretrain_model and self._pretrain_variable_pattern is None:
self._pretrain_variable_pattern = [None] * len(
self._pretrain_model)
assert (
(self._pretrain_model is None
and self._pretrain_variable_pattern is None) or len(
self._pretrain_model) == len(self._pretrain_variable_pattern)
or len(self._pretrain_model) == 1
), ("`pretrain_variable_pattern` must match with `pretrain_model`.")
if self._pretrain_model is not None and self._pretrain_variable_pattern is None:
self._pretrain_variable_pattern = [None] * len(
self._pretrain_model)
self._update_cycle = args["update_cycle"]
self._clip_value = args["clip_value"]
self._clip_norm = args["clip_norm"]
self._hvd_backend = self.strategy if self.strategy in [
"byteps", "horovod"
] else None
with training_utils.get_strategy_scope(self.strategy):
self._criterion = build_criterion(args)
self._criterion.set_model(self.model)
self._lr_schedule = build_lr_schedule(args)
optimizer = build_optimizer(args)
assert optimizer is not None, "optimizer parameters must be provided for training."
self._optimizer = _handle_fp16_and_distributed_optimizer(
optimizer, self._lr_schedule, self._hvd_backend)
self._validator = build_validator(args)
self._experimental_count_batch_num = args[
"experimental_count_batch_num"]
def __init__(self, args, **kwargs):
""" Initializes a util class for training neural models. """
super(Trainer, self).__init__(**kwargs)
self._tb_log_dir = args["tb_log_dir"]
self._train_steps = args["train_steps"]
self._summary_steps = args["summary_steps"]
self._save_checkpoint_steps = args["save_checkpoint_steps"]
self._checkpoints_max_to_keep = args["checkpoints_max_to_keep"]
self._initial_global_step = args["initial_global_step"]
self._pretrain_variable_pattern = args["pretrain_variable_pattern"]
if args["pretrain_model"] and isinstance(args["pretrain_model"][0],
dict):
self._pretrain_v2 = True
self._pretrain_model = args["pretrain_model"]
if self._pretrain_variable_pattern:
logging.info(
"Using pretrain model v2 and ignoring pretrain_variable_pattern: "
f"{self._pretrain_variable_pattern}")
else:
self._pretrain_v2 = False
self._pretrain_model = flatten_string_list(args["pretrain_model"])
if args["mask_dir"]:
self.mask_dir = args["mask_dir"][0]
# print(self.mask_dir)
# self.load_mask = np.load(self.mask_dir, allow_pickle=True)
with open(self.mask_dir, 'rb') as f:
self.load_mask = pickle.load(f)
# i = 0
# for weight in self.load_mask:
# if i <= 1000:
# tf.print(weight.name, output_stream='file://./mask.txt')
# if weight.shape.ndims > 0:
# tf.print(weight[:1], output_stream='file://./mask.txt', summarize=-1, name=weight.name)
# else:
# tf.print(weight, output_stream='file://./mask.txt', summarize=-1, name=weight.name)
# else:
# i += 1
else:
self.mask_dir = os.path.join(self.model_dir, "mask.pkl")
self.load_mask = None
if self._pretrain_model:
if self._pretrain_variable_pattern is None:
self._pretrain_variable_pattern = [None] * len(
self._pretrain_model)
elif isinstance(self._pretrain_variable_pattern, str):
self._pretrain_variable_pattern = [
self._pretrain_variable_pattern
]
assert (
(self._pretrain_model is None
and self._pretrain_variable_pattern is None)
or len(self._pretrain_model) == len(
self._pretrain_variable_pattern)
or len(self._pretrain_model) == 1
), ("`pretrain_variable_pattern` must match with `pretrain_model`."
)
if self._pretrain_model is not None and self._pretrain_variable_pattern is None:
self._pretrain_variable_pattern = [None] * len(
self._pretrain_model)
self._update_cycle = args["update_cycle"]
self._clip_value = args["clip_value"]
self._clip_norm = args["clip_norm"]
self._hvd_backend = self.strategy if self.strategy in [
"byteps", "horovod"
] else None
with training_utils.get_strategy_scope(self.strategy):
self._criterion = build_criterion(args)
self._criterion.set_model(self.model)
self._lr_schedule_args = args
if compat.IS_PREV_TF_2_4_0:
self._optimizer = build_optimizer(args)
else:
self._optimizer = build_optimizer(args,
clipnorm=self._clip_norm,
clipvalue=self._clip_value)
assert self._optimizer is not None, "optimizer parameters must be provided for training."
self._validator = build_validator(args)
self._experimental_count_batch_num = args[
"experimental_count_batch_num"]
self._freeze_variables = args["freeze_variables"]
self._pruning_schedule = build_pruning_schedule(args)
self._partial_tuning = args["partial_tuning"]
self._pruning_variable_pattern = args["pruning_variable_pattern"]
self._nopruning_variable_pattern = args["nopruning_variable_pattern"]