def test_logged_lr(self):
# Mock LR scheduler
def scheduler_mock(where):
return where
mock_lr_scheduler = mock.Mock(side_effect=scheduler_mock)
mock_lr_scheduler.update_interval = UpdateInterval.STEP
config = get_test_mlp_task_config()
config["num_epochs"] = 3
config["dataset"]["train"]["batchsize_per_replica"] = 5
config["dataset"]["test"]["batchsize_per_replica"] = 5
task = build_task(config)
task.optimizer.lr_scheduler = mock_lr_scheduler
trainer = LocalTrainer()
# 2 LR updates per epoch
# At end of each epoch for train, LR is logged an additional time
lr_order = [
0.0, 1 / 6, 1 / 6, 2 / 6, 3 / 6, 3 / 6, 4 / 6, 5 / 6, 5 / 6
]
lr_list = []
def mock_log_lr(task: ClassyTask, local_variables) -> None:
lr_list.append(task.optimizer.lr)
with mock.patch.object(LossLrMeterLoggingHook,
"_log_lr",
side_effect=mock_log_lr):
hook = LossLrMeterLoggingHook(1)
task.set_hooks([hook])
trainer.train(task)
self.assertEqual(lr_list, lr_order)
def test_logged_lr(self):
# Mock LR scheduler
def scheduler_mock(where):
return where
mock_lr_scheduler = mock.Mock(side_effect=scheduler_mock)
mock_lr_scheduler.update_interval = UpdateInterval.STEP
config = get_test_mlp_task_config()
config["num_epochs"] = 3
config["dataset"]["train"]["batchsize_per_replica"] = 10
config["dataset"]["test"]["batchsize_per_replica"] = 5
task = build_task(config)
task.optimizer.param_schedulers["lr"] = mock_lr_scheduler
trainer = LocalTrainer()
# 2 LR updates per epoch = 6
lr_order = [0.0, 1 / 6, 2 / 6, 3 / 6, 4 / 6, 5 / 6]
lr_list = []
class LRLoggingHook(ClassyHook):
on_end = ClassyHook._noop
on_phase_end = ClassyHook._noop
on_phase_start = ClassyHook._noop
on_start = ClassyHook._noop
def on_step(self, task):
if task.train:
lr_list.append(task.optimizer.parameters.lr)
hook = LRLoggingHook()
task.set_hooks([hook])
trainer.train(task)
self.assertEqual(lr_list, lr_order)
def test_logged_lr(self):
class SchedulerMock(ClassyParamScheduler):
def __call__(self, where):
return where
mock_lr_scheduler = SchedulerMock(UpdateInterval.STEP)
config = get_test_mlp_task_config()
config["num_epochs"] = 3
config["dataset"]["train"]["batchsize_per_replica"] = 10
config["dataset"]["test"]["batchsize_per_replica"] = 5
task = build_task(config)
task.set_optimizer_schedulers({"lr": mock_lr_scheduler})
trainer = LocalTrainer()
# 2 LR updates per epoch = 6
lr_order = [0.0, 1 / 6, 2 / 6, 3 / 6, 4 / 6, 5 / 6]
lr_list = []
class LRLoggingHook(ClassyHook):
on_end = ClassyHook._noop
on_phase_end = ClassyHook._noop
on_phase_start = ClassyHook._noop
on_start = ClassyHook._noop
def on_step(self, task):
if task.train:
lr_list.append(task.optimizer.options_view.lr)
hook = LRLoggingHook()
task.set_hooks([hook])
trainer.train(task)
self.assertEqual(lr_list, lr_order)
def test_train(self):
config = get_test_mlp_task_config()
task = build_task(config)
num_samples = 10
precise_batch_norm_hook = PreciseBatchNormHook(num_samples)
task.set_hooks([precise_batch_norm_hook])
task.prepare()
trainer = ClassyTrainer()
trainer.train(task)
def test_logged_lr(self):
# Mock LR scheduler
class SchedulerMock(ClassyParamScheduler):
def __call__(self, where):
return where
mock_lr_scheduler = SchedulerMock(UpdateInterval.STEP)
# Mock Logging
class DummySummaryWriter(object):
def __init__(self):
self.scalar_logs = {}
def add_scalar(self,
key,
value,
global_step=None,
walltime=None) -> None:
self.scalar_logs[key] = self.scalar_logs.get(key, []) + [value]
def add_histogram(self,
key,
value,
global_step=None,
walltime=None) -> None:
return
def add_text(self, *args, **kwargs):
pass
def flush(self):
return
config = get_test_mlp_task_config()
config["num_epochs"] = 3
config["dataset"]["train"]["batchsize_per_replica"] = 10
config["dataset"]["test"]["batchsize_per_replica"] = 5
task = build_task(config)
writer = DummySummaryWriter()
hook = TensorboardPlotHook(writer)
hook.log_period = 1
task.set_hooks([hook])
task.set_optimizer_schedulers({"lr": mock_lr_scheduler})
trainer = LocalTrainer()
trainer.train(task)
# We have 20 samples, batch size is 10. Each epoch is done in two steps.
self.assertEqual(
writer.scalar_logs["Learning Rate/train"],
[0, 1 / 6, 2 / 6, 3 / 6, 4 / 6, 5 / 6],
)
def setUp(self):
config = get_test_mlp_task_config()
invalid_config = copy.deepcopy(config)
invalid_config["name"] = "invalid_task"
self.config_files = {}
for config_key, config in [
("config", config),
("invalid_config", invalid_config),
]:
with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
json.dump(config, f)
f.flush()
self.config_files[config_key] = f.name
self.path = Path(__file__).parent.absolute()
def test_training(self):
"""Checks we can train a small MLP model."""
config = get_test_mlp_task_config()
task = (ClassificationTask().set_num_epochs(10).set_loss(
build_loss(config["loss"])).set_model(build_model(
config["model"])).set_optimizer(
build_optimizer(config["optimizer"])).set_meters([
AccuracyMeter(topk=[1])
]).set_hooks([LossLrMeterLoggingHook()]))
for split in ["train", "test"]:
dataset = build_dataset(config["dataset"][split])
task.set_dataset(dataset, split)
self.assertTrue(task is not None)
trainer = LocalTrainer()
trainer.train(task)
accuracy = task.meters[0].value["top_1"]
self.assertAlmostEqual(accuracy, 1.0)
def test_logged_lr(self):
# Mock LR scheduler
def scheduler_mock(where):
return where
mock_lr_scheduler = mock.Mock(side_effect=scheduler_mock)
mock_lr_scheduler.update_interval = UpdateInterval.STEP
# Mock Logging
class DummySummaryWriter(object):
def __init__(self):
self.scalar_logs = {}
def add_scalar(self,
key,
value,
global_step=None,
walltime=None) -> None:
self.scalar_logs[key] = self.scalar_logs.get(key, []) + [value]
def flush(self):
return
config = get_test_mlp_task_config()
config["num_epochs"] = 3
config["dataset"]["train"]["batchsize_per_replica"] = 5
config["dataset"]["test"]["batchsize_per_replica"] = 5
task = build_task(config)
writer = DummySummaryWriter()
hook = TensorboardPlotHook(writer)
task.set_hooks([hook])
task.optimizer.param_schedulers["lr"] = mock_lr_scheduler
trainer = LocalTrainer()
trainer.train(task)
# We have 10 samples, batch size is 5. Each epoch is done in two steps.
self.assertEqual(
writer.scalar_logs["train_learning_rate_updates"],
[0, 1 / 6, 2 / 6, 3 / 6, 4 / 6, 5 / 6],
)
def test_bn_stats(self):
base_self = self
class TestHook(ClassyHook):
on_start = ClassyHook._noop
on_phase_start = ClassyHook._noop
on_phase_end = ClassyHook._noop
on_end = ClassyHook._noop
def __init__(self):
self.train_bn_stats = None
self.test_bn_stats = None
def on_step(self, task):
if task.train:
self.train_bn_stats = base_self._get_bn_stats(
task.base_model)
else:
self.test_bn_stats = base_self._get_bn_stats(
task.base_model)
config = get_test_mlp_task_config()
task = build_task(config)
num_samples = 10
precise_batch_norm_hook = PreciseBatchNormHook(num_samples)
test_hook = TestHook()
task.set_hooks([precise_batch_norm_hook, test_hook])
trainer = ClassyTrainer()
trainer.train(task)
updated_bn_stats = self._get_bn_stats(task.base_model)
# the stats should be modified after train steps but not after test steps
self.assertFalse(
self._compare_bn_stats(test_hook.train_bn_stats, updated_bn_stats))
self.assertTrue(
self._compare_bn_stats(test_hook.test_bn_stats, updated_bn_stats))
