def _compute_validation_metrics(self) -> workload.Response:
# Set the behavior of certain layers (e.g., dropout) that are
# different between training and inference.
for model in self.context.models:
model.eval()
for callback in self.callbacks.values():
logging.warning(
"on_validation_step_start is now deprecated, please use on_validation_start instead"
)
callback.on_validation_step_start()
for callback in self.callbacks.values():
callback.on_validation_start()
num_inputs = 0
metrics = {} # type: Optional[Dict[str, Any]]
if self._evaluate_batch_defined():
keys = None
batch_metrics = []
self.validation_loader = cast(torch.utils.data.DataLoader,
self.validation_loader)
check.gt(len(self.validation_loader), 0)
for batch in self.validation_loader:
batch = self.context.to_device(batch)
num_inputs += data_length(batch)
vld_metrics = self.trial.evaluate_batch(batch=batch)
# Verify validation metric names are the same across batches.
if keys is None:
keys = vld_metrics.keys()
else:
check.eq(
keys,
vld_metrics.keys(),
"Validation metric names must match across all batches of data.",
)
check.is_instance(
vld_metrics,
dict,
"validation_metrics() must return a "
"dictionary of string names to Tensor "
"metrics",
)
# TODO: For performance perform -> cpu() only at the end of validation.
batch_metrics.append(
self._convert_metrics_to_numpy(vld_metrics))
metrics = self._reduce_metrics(
batch_metrics=batch_metrics,
keys=keys,
metrics_reducers=self._prepare_metrics_reducers(keys=keys),
)
if self.hvd_config.use:
num_inputs *= hvd.size()
else:
check.true(self._evaluate_full_dataset_defined())
self.validation_loader = cast(torch.utils.data.DataLoader,
self.validation_loader)
if self.is_chief:
metrics = self.trial.evaluate_full_dataset(
data_loader=self.validation_loader)
check.is_instance(
metrics, dict,
f"eval() must return a dictionary, got {type(metrics)}.")
metrics = self._convert_metrics_to_numpy(metrics)
num_inputs = self.context.get_per_slot_batch_size() * len(
self.validation_loader)
if self.hvd_config.use and any(
map(
lambda c: util.is_overridden(c.on_validation_end, _callback
.PyTorchCallback) or util.
is_overridden(c.on_validation_step_end, _callback.
PyTorchCallback),
self.callbacks.values(),
)):
logging.debug(
"Broadcasting metrics to all worker processes to execute a "
"validation step end callback")
metrics = hvd.broadcast_object(metrics, root_rank=0)
for callback in self.callbacks.values():
logging.warning(
"on_validation_step_end is now deprecated, please use on_validation_end instead"
)
callback.on_validation_step_end(cast(Dict[str, Any], metrics))
for callback in self.callbacks.values():
callback.on_validation_end(cast(Dict[str, Any], metrics))
if not self.is_chief:
return workload.Skipped()
return {"num_inputs": num_inputs, "validation_metrics": metrics}
def _compute_validation_metrics(self) -> workload.Response:
# Set the behavior of certain layers (e.g., dropout) that are
# different between training and inference.
self.model.eval()
num_inputs = 0
metrics = {} # type: Optional[Dict[str, Any]]
if self._evaluate_batch_defined():
keys = None
batch_metrics = []
self.validation_loader = cast(torch.utils.data.DataLoader,
self.validation_loader)
check.gt(len(self.validation_loader), 0)
for batch in self.validation_loader:
batch = self._to_device(batch)
num_inputs += data_length(batch)
vld_metrics = self.trial.evaluate_batch(batch=batch,
model=self.model)
# Verify validation metric names are the same across batches.
if keys is None:
keys = vld_metrics.keys()
else:
check.eq(
keys,
vld_metrics.keys(),
"Validation metric names must match across all batches of data.",
)
check.is_instance(
vld_metrics,
dict,
"validation_metrics() must return a "
"dictionary of string names to Tensor "
"metrics",
)
# TODO: For performance perform -> cpu() only at the end of validation.
batch_metrics.append(
self._convert_metrics_to_numpy(vld_metrics))
keys = cast(Any, keys)
metrics = self._reduce_metrics(
batch_metrics=batch_metrics,
keys=keys,
metrics_reducers=self._prepare_metrics_reducers(keys=keys),
)
if self.hvd_config.use:
num_inputs *= hvd.size()
else:
check.true(self._evaluate_full_dataset_defined())
self.validation_loader = cast(torch.utils.data.DataLoader,
self.validation_loader)
if self.is_chief:
metrics = self.trial.evaluate_full_dataset(
data_loader=self.validation_loader, model=self.model)
check.is_instance(
metrics, dict,
f"eval() must return a dictionary, got {type(metrics)}.")
metrics = self._convert_metrics_to_numpy(metrics)
num_inputs = self.context.get_per_slot_batch_size() * len(
self.validation_loader)
if not self.is_chief:
return workload.Skipped()
return {"num_inputs": num_inputs, "validation_metrics": metrics}
def _load(self) -> None:
if not self.load_path:
return
# Backwards compat with older checkpoint formats. List is newest to
# oldest known state_dict locations.
potential_paths = [
["state_dict.pth"],
["determined", "state_dict.pth"],
["pedl", "state_dict.pth"],
["checkpoint.pt"],
]
for ckpt_path in potential_paths:
maybe_ckpt = self.load_path.joinpath(*ckpt_path)
if maybe_ckpt.exists():
checkpoint = torch.load(str(maybe_ckpt),
map_location="cpu") # type: ignore
break
if "model_state_dict" in checkpoint:
# Backward compatible with older checkpoint format.
check.not_in("models_state_dict", checkpoint)
check.eq(len(self.context.models), 1)
self.context.models[0].load_state_dict(
checkpoint["model_state_dict"])
else:
for idx, model in enumerate(self.context.models):
model.load_state_dict(checkpoint["models_state_dict"][idx])
if "optimizer_state_dict" in checkpoint:
# Backward compatible with older checkpoint format.
check.not_in("optimizers_state_dict", checkpoint)
check.eq(len(self.context.optimizers), 1)
self.context.optimizers[0].load_state_dict(
checkpoint["optimizer_state_dict"])
else:
for idx, optimizer in enumerate(self.context.optimizers):
optimizer.load_state_dict(
checkpoint["optimizers_state_dict"][idx])
if "lr_scheduler" in checkpoint:
# Backward compatible with older checkpoint format.
check.not_in("lr_schedulers_state_dict", checkpoint)
check.eq(len(self.context.lr_schedulers), 1)
self.context.lr_schedulers[0].load_state_dict(
checkpoint["lr_scheduler"])
else:
for idx, lr_scheduler in enumerate(self.context.lr_schedulers):
lr_scheduler.load_state_dict(
checkpoint["lr_schedulers_state_dict"][idx])
if "amp_state" in checkpoint:
if self.context._use_amp:
apex.amp.load_state_dict(checkpoint["amp_state"])
else:
logging.warning(
"There exists amp_state in checkpoint but the experiment is not using AMP."
)
else:
if self.context._use_amp:
logging.warning(
"The experiment is using AMP but amp_state does not exist in the checkpoint."
)
if "rng_state" in checkpoint:
rng_state = checkpoint["rng_state"]
np.random.set_state(rng_state["np_rng_state"])
random.setstate(rng_state["random_rng_state"])
torch.random.set_rng_state(rng_state["cpu_rng_state"])
if torch.cuda.device_count():
if "gpu_rng_state" in rng_state:
torch.cuda.set_rng_state(
rng_state["gpu_rng_state"],
device=self.context.distributed.get_local_rank())
else:
logging.warning(
"The system has a gpu but no gpu_rng_state exists in the checkpoint."
)
else:
if "gpu_rng_state" in rng_state:
logging.warning(
"There exists gpu_rng_state in checkpoint but the system has no gpu."
)
else:
logging.warning("The checkpoint has no random state to restore.")
callback_state = checkpoint.get("callbacks", {})
for name in self.callbacks:
if name in callback_state:
self.callbacks[name].load_state_dict(callback_state[name])
elif util.is_overridden(self.callbacks[name].load_state_dict,
pytorch.PyTorchCallback):
logging.warning(
"Callback '{}' implements load_state_dict(), but no callback state "
"was found for that name when restoring from checkpoint. This "
"callback will be initialized from scratch")
def configure_apex_amp(
self,
models: Union[torch.nn.Module, List[torch.nn.Module]],
optimizers: Union[torch.optim.Optimizer,
List[torch.optim.Optimizer]], # type: ignore
enabled: Optional[bool] = True,
opt_level: Optional[str] = "O1",
cast_model_type: Optional[torch.dtype] = None,
patch_torch_functions: Optional[bool] = None,
keep_batchnorm_fp32: Optional[Union[bool, str]] = None,
master_weights: Optional[bool] = None,
loss_scale: Optional[Union[float, str]] = None,
cast_model_outputs: Optional[torch.dtype] = None,
num_losses: Optional[int] = 1,
verbosity: Optional[int] = 1,
min_loss_scale: Optional[float] = None,
max_loss_scale: Optional[float] = 2.0**24,
) -> Tuple:
"""
Configure automatic mixed precision for your models and optimizers. Note that details
for apex.amp are handled automatically within Determined after this call.
This function must be called **after** you have finished constructing your models and
optimizers with :meth:`wrap_model` and :meth:`wrap_optimizer`.
This function has the same arguments as
`apex.amp.initialize <https://nvidia.github.io/apex/amp.html#apex.amp.initialize>`_.
.. warning::
When using distributed training and automatic mixed precision,
we only support ``num_losses=1`` and calling backward on the loss once.
Arguments:
models (``torch.nn.Module`` or list of ``torch.nn.Module`` s): Model(s) to modify/cast.
optimizers (``torch.optim.Optimizer`` or list of ``torch.optim.Optimizer`` s):
Optimizers to modify/cast. REQUIRED for training.
enabled (bool, optional, default=True): If False, renders all Amp calls no-ops,
so your script should run as if Amp were not present.
opt_level (str, optional, default="O1"): Pure or mixed precision optimization level.
Accepted values are "O0", "O1", "O2", and "O3", explained in detail above.
cast_model_type (``torch.dtype``, optional, default=None): Optional property override,
see above.
patch_torch_functions (bool, optional, default=None): Optional property override.
keep_batchnorm_fp32 (bool or str, optional, default=None): Optional property override.
If passed as a string, must be the string "True" or "False".
master_weights (bool, optional, default=None): Optional property override.
loss_scale (float or str, optional, default=None): Optional property override.
If passed as a string, must be a string representing a number, e.g., "128.0",
or the string "dynamic".
cast_model_outputs (torch.dtype, optional, default=None): Option to ensure that
the outputs of your model is always cast to a particular type regardless of
``opt_level``.
num_losses (int, optional, default=1): Option to tell Amp in advance how many
losses/backward passes you plan to use. When used in conjunction with the
``loss_id`` argument to ``amp.scale_loss``, enables Amp to use a different
loss scale per loss/backward pass, which can improve stability.
If ``num_losses`` is left to 1, Amp will still support multiple losses/backward
passes, but use a single global loss scale for all of them.
verbosity (int, default=1): Set to 0 to suppress Amp-related output.
min_loss_scale (float, default=None): Sets a floor for the loss scale values that
can be chosen by dynamic loss scaling. The default value of None means that no
floor is imposed. If dynamic loss scaling is not used, `min_loss_scale` is ignored.
max_loss_scale (float, default=2.**24): Sets a ceiling for the loss scale values
that can be chosen by dynamic loss scaling. If dynamic loss scaling is not used,
`max_loss_scale` is ignored.
Returns:
Model(s) and optimizer(s) modified according to the ``opt_level``.
If ``optimizers`` args were lists, the corresponding return value will
also be a list.
"""
if not self.env.managed_training:
return models, optimizers
check.false(self._use_amp, "Please only call configure_apex_amp once.")
if self.hvd_config.use:
check.eq(
num_losses,
1,
"When using parallel/distributed training, "
"Determined only supports configure_apex_amp with num_losses = 1",
)
self._use_amp = True
if self.hvd_config.use:
check.eq(
self.hvd_config.aggregation_frequency,
1,
"Mixed precision training (AMP) is not supported with "
"aggregation frequency > 1.",
)
check.true(
torch.cuda.is_available(),
"Mixed precision training (AMP) is supported only on GPU slots.",
)
logging.info(
f"Enabling mixed precision training with opt_level: {opt_level}.")
models, optimizers = apex.amp.initialize(
models=models,
optimizers=optimizers,
enabled=enabled,
opt_level=opt_level,
cast_model_type=cast_model_type,
patch_torch_functions=patch_torch_functions,
keep_batchnorm_fp32=keep_batchnorm_fp32,
master_weights=master_weights,
loss_scale=loss_scale,
cast_model_outputs=cast_model_outputs,
num_losses=num_losses,
min_loss_scale=min_loss_scale,
max_loss_scale=max_loss_scale,
verbosity=verbosity if self.distributed.get_rank() == 0
or self.env.experiment_config.debug_enabled() else 0,
)
if not isinstance(models, list):
self.models = [models]
if not isinstance(optimizers, list):
self.optimizers = [optimizers]
return models, optimizers
def select_checkpoint(
self,
latest: bool = False,
best: bool = False,
uuid: Optional[str] = None,
sort_by: Optional[str] = None,
smaller_is_better: Optional[bool] = None,
) -> checkpoint.Checkpoint:
"""
Return the :py:class:`det.experimental.Checkpoint` instance with the best
validation metric as defined by the `sort_by` and `smaller_is_better`
arguments.
Exactly one of the best, latest, or uuid parameters must be set.
Arguments:
latest (bool, optional): return the most recent checkpoint.
best (bool, optional): return the checkpoint with the best validation
metric as defined by the `sort_by` and `smaller_is_better`
arguments. If `sort_by` and `smaller_is_better` are not
specified, the values from the associated experiment
configuration will be used.
uuid (string, optional): return the checkpoint for the specified uuid.
sort_by (string, optional): the name of the validation metric to
order checkpoints by. If this parameter is unset the metric defined
in the related experiment configuration searcher field will be
used.
smaller_is_better (bool, optional): specifies whether to sort the
metric above in ascending or descending order. If sort_by is unset,
this parameter is ignored. By default the smaller_is_better value
in the related experiment configuration is used.
"""
check.eq(
sum([int(latest), int(best), int(uuid is not None)]),
1,
"Exactly one of latest, best, or uuid must be set",
)
check.eq(
sort_by is None,
smaller_is_better is None,
"sort_by and smaller_is_better must be set together",
)
if sort_by is not None and not best:
raise AssertionError(
"sort_by and smaller_is_better parameters can only be used with --best"
)
q = api.GraphQLQuery(self._master)
if sort_by is not None:
checkpoint_gql = q.op.best_checkpoint_by_metric(
args={"tid": self.id, "metric": sort_by, "smaller_is_better": smaller_is_better},
)
else:
where = gql.checkpoints_bool_exp(
state=gql.checkpoint_state_comparison_exp(_eq="COMPLETED"),
trial_id=gql.Int_comparison_exp(_eq=self.id),
)
order_by = [] # type: List[gql.checkpoints_order_by]
if uuid is not None:
where.uuid = gql.uuid_comparison_exp(_eq=uuid)
elif latest:
order_by = [gql.checkpoints_order_by(end_time=gql.order_by.desc)]
elif best:
where.validation = gql.validations_bool_exp(
state=gql.validation_state_comparison_exp(_eq="COMPLETED")
)
order_by = [
gql.checkpoints_order_by(
validation=gql.validations_order_by(
metric_values=gql.validation_metrics_order_by(signed=gql.order_by.asc)
)
)
]
checkpoint_gql = q.op.checkpoints(where=where, order_by=order_by, limit=1)
checkpoint_gql.state()
checkpoint_gql.uuid()
checkpoint_gql.resources()
validation = checkpoint_gql.validation()
validation.metrics()
validation.state()
step = checkpoint_gql.step()
step.id()
step.start_time()
step.end_time()
step.trial.experiment.config()
resp = q.send()
result = resp.best_checkpoint_by_metric if sort_by is not None else resp.checkpoints
if not result:
raise AssertionError("No checkpoint found for trial {}".format(self.id))
ckpt_gql = result[0]
batch_number = ckpt_gql.step.trial.experiment.config["batches_per_step"] * ckpt_gql.step.id
return checkpoint.Checkpoint(
ckpt_gql.uuid,
ckpt_gql.step.trial.experiment.config["checkpoint_storage"],
batch_number,
ckpt_gql.step.start_time,
ckpt_gql.step.end_time,
ckpt_gql.resources,
ckpt_gql.validation,
)