def from_trial(
trial_inst: det.Trial,
context: det.TrialContext,
env: det.EnvContext,
workloads: workload.Stream,
load_path: Optional[pathlib.Path],
rendezvous_info: det.RendezvousInfo,
hvd_config: horovod.HorovodContext,
) -> det.TrialController:
check.is_instance(
context, keras.TFKerasTrialContext,
"TFKerasTrialController needs a TFKerasTrialContext")
context = cast(keras.TFKerasTrialContext, context)
check.is_instance(trial_inst, TFKerasTrial,
"TFKerasTrialController needs a TFKerasTrial")
trial = cast(TFKerasTrial, trial_inst)
session = TFKerasTrialController._configure_session(
env, hvd_config, trial.session_config())
training_data = keras._adapt_data_from_data_loader(
input_data=trial.build_training_data_loader(),
batch_size=context.get_per_slot_batch_size(),
)
validation_data = keras._adapt_data_from_data_loader(
input_data=trial.build_validation_data_loader(),
batch_size=context.get_per_slot_batch_size(),
)
trial.build_model()
check.is_not_none(context.model, "Please call wrap_model(...).")
check.is_not_none(context.compile_args,
"Please call model.compile(...).")
compile_args = cast(inspect.BoundArguments, context.compile_args)
TFKerasTrialController.compile_model(context=context,
compile_args=compile_args,
env=env,
hvd_config=hvd_config)
tf_keras_callbacks = trial.keras_callbacks()
return TFKerasTrialController(
context.model,
session,
keras.TFKerasTrainConfig(training_data, validation_data,
tf_keras_callbacks),
context,
env,
workloads,
load_path,
rendezvous_info,
hvd_config,
)
def from_trial(
cls: Type["TFKerasTrialController"],
trial_inst: det.Trial,
context: det.TrialContext,
env: det.EnvContext,
workloads: Optional[workload.Stream] = None,
) -> det.TrialController:
check.is_instance(
context, keras.TFKerasTrialContext, "TFKerasTrialController needs a TFKerasTrialContext"
)
context = cast(keras.TFKerasTrialContext, context)
check.is_instance(trial_inst, TFKerasTrial, "TFKerasTrialController needs a TFKerasTrial")
trial = cast(TFKerasTrial, trial_inst)
# Keras only supports horovod backend for distributed training
session = cls._configure_session(
env, trial.session_config(), use_horovod=context.distributed.size > 1
)
training_data = keras._adapt_data_from_data_loader(
input_data=trial.build_training_data_loader(),
batch_size=context.get_per_slot_batch_size(),
)
validation_data = keras._adapt_data_from_data_loader(
input_data=trial.build_validation_data_loader(),
batch_size=context.get_per_slot_batch_size(),
)
trial.build_model()
check.is_not_none(context.model, "Please call wrap_model(...).")
check.is_not_none(context.compile_args, "Please call model.compile(...).")
compile_args = cast(inspect.BoundArguments, context.compile_args)
cls.compile_model(context=context, compile_args=compile_args, env=env)
tf_keras_callbacks = trial.keras_callbacks()
return cls(
context.model,
session,
keras.TFKerasTrainConfig(training_data, validation_data, tf_keras_callbacks),
trial,
context,
env,
workloads,
)
def fit_generator(wrapper, *args: Any, **kwargs: Any) -> None:
if not self.compile_args:
raise errors.InvalidExperimentException(
"Must call .compile before calling .fit_generator().")
fit_generator_args = inspect.signature(
model.fit_generator).bind(*args, **kwargs)
fit_generator_args.apply_defaults()
training_data = fit_generator_args.arguments["generator"]
if fit_generator_args.arguments["validation_data"] is None:
raise errors.InvalidExperimentException(
"Determined requires validation_data in the call to fit_generator()."
)
validation_data = keras._adapt_data_from_data_loader(
input_data=fit_generator_args.arguments["validation_data"],
batch_size=self.env.per_slot_batch_size,
)
self.train_config = TFKerasTrainConfig(
training_data=training_data,
validation_data=validation_data,
callbacks=fit_generator_args.arguments["callbacks"],
)
self.configure_fit(
verbose=fit_generator_args.arguments["verbose"],
class_weight=fit_generator_args.arguments["class_weight"],
shuffle=fit_generator_args.arguments["shuffle"],
workers=fit_generator_args.arguments["workers"],
use_multiprocessing=fit_generator_args.
arguments["use_multiprocessing"],
max_queue_size=fit_generator_args.
arguments["max_queue_size"],
)
if train_fn:
train_fn()
def fit(wrapper, *args: Any, **kwargs: Any) -> None:
"""Communicate a model, data, and other training configuration with the harness.
Parameters:
the same as tf.keras.Model.fit except for this function only handles the
following cases of data
x: Input data. It could be:
1) A Numpy array (or array-like), or a list of arrays (in case the model
has multiple inputs).
2) A dict mapping input names to the corresponding array, if the model
has named inputs.
3) A tf.data dataset. Should return a tuple of either (inputs, targets) or
(inputs, targets, sample_weights).
4) A keras.utils.Sequence returning (inputs, targets) or (inputs, targets,
sample weights).
y: Target data. Like the input data x, it could be either Numpy array(s).
If x is a dataset or keras.utils.Sequence instance, y should not be
specified(since targets will be obtained from x).
validation_data: Data on which to evaluate the loss and any model metrics
at the end of each epoch. The model will not be trained on this data.
validation_data will override validation_split. validation_data could be:
1) tuple (x_val, y_val) of Numpy arrays
2) tuple (x_val, y_val, val_sample_weights) of Numpy arrays
3) dataset For the first two cases, batch_size must be provided.
For the last case, validation_steps could be provided.
"""
if not self.compile_args:
raise errors.InvalidExperimentException(
"Must call .compile before calling .fit()."
)
fit_args = inspect.signature(model.fit).bind(*args, **kwargs)
fit_args.apply_defaults()
training_data = keras._adapt_data_from_fit_args(
x=fit_args.arguments["x"],
y=fit_args.arguments["y"],
sample_weight=fit_args.arguments["sample_weight"],
batch_size=self.env.per_slot_batch_size,
)
if fit_args.arguments["validation_data"] is None:
raise errors.InvalidExperimentException(
"Determined requires validation_data in the call to fit()."
)
validation_data = keras._adapt_data_from_data_loader(
input_data=fit_args.arguments["validation_data"],
batch_size=self.env.per_slot_batch_size,
)
self.train_config = TFKerasTrainConfig(
training_data=training_data,
validation_data=validation_data,
callbacks=fit_args.arguments["callbacks"],
)
self.configure_fit(
verbose=fit_args.arguments["verbose"],
shuffle=fit_args.arguments["shuffle"],
class_weight=fit_args.arguments["class_weight"],
workers=fit_args.arguments["workers"],
use_multiprocessing=fit_args.arguments["use_multiprocessing"],
max_queue_size=fit_args.arguments["max_queue_size"],
)
if train_fn:
train_fn()
def test_adapt_invalid_data_type() -> None:
seqs = utils.make_xor_data_sequences()
test = keras._adapt_data_from_data_loader(seqs[1], batch_size=1)
with pytest.raises(det.errors.InvalidDataTypeException) as err:
keras._adapt_data_from_data_loader((None, test), batch_size=1)
assert err is not None
