def make_workloads() -> workload.Stream: trainer = utils.TrainAndValidate() yield from trainer.send(steps=10, validation_freq=1, batches_per_step=100) training_metrics, validation_metrics = trainer.result() # We expect the validation error and training loss to be # monotonically decreasing. # TODO(DET-1597): actually use a model and optimizer where the losses # are monotonically decreasing. for older, newer in zip(training_metrics[::100], training_metrics[::100][1:]): assert newer["loss"] <= older["loss"] for older, newer in zip(validation_metrics, validation_metrics[1:]): assert newer["val_categorical_error"] <= older[ "val_categorical_error"] assert validation_metrics[-1][ "val_categorical_error"] == pytest.approx(0.0) yield workload.terminate_workload( ), [], workload.ignore_workload_response
def make_workloads() -> workload.Stream: trainer = utils.TrainAndValidate() yield from trainer.send(steps=2, validation_freq=1, batches_per_step=1) yield workload.terminate_workload( ), [], workload.ignore_workload_response
def make_workloads_2() -> workload.Stream: trainer = utils.TrainAndValidate() yield from trainer.send(steps=1, validation_freq=1) yield workload.checkpoint_workload(), [ checkpoint_dir ], workload.ignore_workload_response yield workload.terminate_workload( ), [], workload.ignore_workload_response
def make_workloads(tag: str) -> workload.Stream: trainer = utils.TrainAndValidate() yield from trainer.send(steps=1000, validation_freq=100) tm, vm = trainer.result() training_metrics[tag] = tm validation_metrics[tag] = vm yield workload.terminate_workload( ), [], workload.ignore_workload_response
def make_workloads() -> workload.Stream: trainer = utils.TrainAndValidate() yield from trainer.send(steps=10, validation_freq=10) training_metrics, validation_metrics = trainer.result() for metrics in training_metrics: assert "accuracy" in metrics yield workload.terminate_workload( ), [], workload.ignore_workload_response
def make_workloads() -> workload.Stream: trainer = utils.TrainAndValidate() yield from trainer.send(steps=1000, validation_freq=100) training_metrics, validation_metrics = trainer.result() # We expect the validation error and training loss to be # monotonically decreasing. for older, newer in zip(training_metrics, training_metrics[1:]): assert newer["loss"] <= older["loss"] yield workload.terminate_workload( ), [], workload.ignore_workload_response
def make_workloads_1() -> workload.Stream: nonlocal old_loss trainer = utils.TrainAndValidate() yield from trainer.send(steps=10, validation_freq=10) training_metrics, validation_metrics = trainer.result() old_loss = validation_metrics[-1]["val_loss"] yield workload.checkpoint_workload(), [ checkpoint_dir ], workload.ignore_workload_response yield workload.terminate_workload( ), [], workload.ignore_workload_response
def make_workloads(checkpoint_dir: str = "") -> workload.Stream: nonlocal training_metrics trainer = utils.TrainAndValidate() yield from trainer.send(steps=10, validation_freq=10, batches_per_step=1) tm, _ = trainer.result() training_metrics += tm if checkpoint_dir: yield workload.checkpoint_workload(), [ checkpoint_dir ], workload.ignore_workload_response yield workload.terminate_workload( ), [], workload.ignore_workload_response
def make_workloads() -> workload.Stream: trainer = utils.TrainAndValidate() yield from trainer.send(steps=10, validation_freq=5, batches_per_step=1000) training_metrics, validation_metrics = trainer.result() # We expect the training loss to be monotonically decreasing and the # accuracy to be monotonically increasing. for older, newer in zip(training_metrics, training_metrics[1:]): assert newer["loss"] < older["loss"] for older, newer in zip(validation_metrics, validation_metrics[1:]): assert newer["accuracy"] >= older["accuracy"] # The final accuracy should be 100%. assert validation_metrics[-1]["accuracy"] == pytest.approx(1.0) yield workload.terminate_workload( ), [], workload.ignore_workload_response