def make_workloads() -> workload.Stream:
trainer = utils.TrainAndValidate()
yield from trainer.send(steps=10,
validation_freq=1,
batches_per_step=100)
training_metrics, validation_metrics = trainer.result()
# We expect the validation error and training loss to be
# monotonically decreasing.
# TODO(DET-1597): actually use a model and optimizer where the losses
# are monotonically decreasing.
for older, newer in zip(training_metrics[::100],
training_metrics[::100][1:]):
assert newer["loss"] <= older["loss"]
for older, newer in zip(validation_metrics,
validation_metrics[1:]):
assert newer["val_categorical_error"] <= older[
"val_categorical_error"]
assert validation_metrics[-1][
"val_categorical_error"] == pytest.approx(0.0)
yield workload.terminate_workload(
), [], workload.ignore_workload_response
def make_workloads() -> workload.Stream:
trainer = utils.TrainAndValidate()
yield from trainer.send(steps=2,
validation_freq=1,
batches_per_step=1)
yield workload.terminate_workload(
), [], workload.ignore_workload_response
def make_workloads_2() -> workload.Stream:
trainer = utils.TrainAndValidate()
yield from trainer.send(steps=1, validation_freq=1)
yield workload.checkpoint_workload(), [
checkpoint_dir
], workload.ignore_workload_response
yield workload.terminate_workload(
), [], workload.ignore_workload_response
def make_workloads(tag: str) -> workload.Stream:
trainer = utils.TrainAndValidate()
yield from trainer.send(steps=1000, validation_freq=100)
tm, vm = trainer.result()
training_metrics[tag] = tm
validation_metrics[tag] = vm
yield workload.terminate_workload(
), [], workload.ignore_workload_response
def make_workloads() -> workload.Stream:
trainer = utils.TrainAndValidate()
yield from trainer.send(steps=10, validation_freq=10)
training_metrics, validation_metrics = trainer.result()
for metrics in training_metrics:
assert "accuracy" in metrics
yield workload.terminate_workload(
), [], workload.ignore_workload_response
def make_workloads() -> workload.Stream:
trainer = utils.TrainAndValidate()
yield from trainer.send(steps=1000, validation_freq=100)
training_metrics, validation_metrics = trainer.result()
# We expect the validation error and training loss to be
# monotonically decreasing.
for older, newer in zip(training_metrics, training_metrics[1:]):
assert newer["loss"] <= older["loss"]
yield workload.terminate_workload(
), [], workload.ignore_workload_response
def make_workloads_1() -> workload.Stream:
nonlocal old_loss
trainer = utils.TrainAndValidate()
yield from trainer.send(steps=10, validation_freq=10)
training_metrics, validation_metrics = trainer.result()
old_loss = validation_metrics[-1]["val_loss"]
yield workload.checkpoint_workload(), [
checkpoint_dir
], workload.ignore_workload_response
yield workload.terminate_workload(
), [], workload.ignore_workload_response
def make_workloads(checkpoint_dir: str = "") -> workload.Stream:
nonlocal training_metrics
trainer = utils.TrainAndValidate()
yield from trainer.send(steps=10,
validation_freq=10,
batches_per_step=1)
tm, _ = trainer.result()
training_metrics += tm
if checkpoint_dir:
yield workload.checkpoint_workload(), [
checkpoint_dir
], workload.ignore_workload_response
yield workload.terminate_workload(
), [], workload.ignore_workload_response
def make_workloads() -> workload.Stream:
trainer = utils.TrainAndValidate()
yield from trainer.send(steps=10,
validation_freq=5,
batches_per_step=1000)
training_metrics, validation_metrics = trainer.result()
# We expect the training loss to be monotonically decreasing and the
# accuracy to be monotonically increasing.
for older, newer in zip(training_metrics, training_metrics[1:]):
assert newer["loss"] < older["loss"]
for older, newer in zip(validation_metrics,
validation_metrics[1:]):
assert newer["accuracy"] >= older["accuracy"]
# The final accuracy should be 100%.
assert validation_metrics[-1]["accuracy"] == pytest.approx(1.0)
yield workload.terminate_workload(
), [], workload.ignore_workload_response