def test_defaults():
"""
Test on ConfigExperiment defaults. It's pretty similar to BaseExperiment's
test but the thing is that those two are very different classes and
inherit from different parent classes.
Also very important to check which callbacks are added by default
"""
exp = ConfigExperiment(config=DEFAULT_MINIMAL_CONFIG)
assert exp.initial_seed == 42
assert exp.logdir is None
assert exp.stages == ["train"]
assert exp.distributed_params == {}
assert exp.monitoring_params == {}
assert exp.get_state_params("train") == {
"logdir": None,
}
assert isinstance(exp.get_model("train"), SomeModel)
assert exp.get_criterion("train") is None
assert exp.get_optimizer("train", SomeModel()) is None
assert exp.get_scheduler("train", None) is None
assert exp.get_callbacks("train").keys() == DEFAULT_CALLBACKS.keys()
cbs = zip(exp.get_callbacks("train").values(), DEFAULT_CALLBACKS.values())
for c1, klass in cbs:
assert isinstance(c1, klass)
def test_when_callback_defined():
"""
There should be no default callback of same kind if there is user defined
already.
"""
config = DEFAULT_MINIMAL_CONFIG.copy()
config["stages"]["callbacks_params"] = {
"my_criterion": {"callback": "CriterionCallback"}
}
exp = ConfigExperiment(config=config)
assert "_criterion" not in exp.get_callbacks("train").keys()
assert "my_criterion" in exp.get_callbacks("train").keys()
def test_when_callback_defined():
"""
There should be no default callback of same kind if there is user defined
already.
"""
callbacks = DEFAULT_CALLBACKS.copy()
callbacks["my_criterion"] = CriterionCallback
callbacks["my_optimizer"] = OptimizerCallback
callbacks["my_scheduler"] = SchedulerCallback
config = DEFAULT_MINIMAL_CONFIG.copy()
config["stages"]["criterion_params"] = {"criterion": "BCEWithLogitsLoss"}
config["stages"]["optimizer_params"] = {"optimizer": "SomeOptimizer"}
config["stages"]["scheduler_params"] = {"scheduler": "SomeScheduler"}
config["stages"]["callbacks_params"] = {
"my_criterion": {
"callback": "CriterionCallback"
},
"my_optimizer": {
"callback": "OptimizerCallback"
},
"my_scheduler": {
"callback": "SchedulerCallback"
},
}
exp = ConfigExperiment(config=config)
_test_callbacks(callbacks, exp)
def test_defaults_criterion_optimizer_scheduler():
"""
Test on ConfigExperiment defaults.
when {criterion, optimizer, scheduler}_params are specified
the respective callback should be generated automatically
"""
callbacks = DEFAULT_CALLBACKS.copy()
callbacks["_criterion"] = CriterionCallback
callbacks["_optimizer"] = OptimizerCallback
callbacks["_scheduler"] = SchedulerCallback
config = DEFAULT_MINIMAL_CONFIG.copy()
config["stages"]["criterion_params"] = {"criterion": "BCEWithLogitsLoss"}
config["stages"]["optimizer_params"] = {"optimizer": "SomeOptimizer"}
config["stages"]["scheduler_params"] = {"scheduler": "SomeScheduler"}
exp = ConfigExperiment(config=config)
assert exp.initial_seed == 42
assert exp.logdir == "./logdir"
assert exp.stages == ["train"]
assert exp.distributed_params == {}
assert exp.get_stage_params("train") == {
"logdir": "./logdir",
}
assert isinstance(exp.get_model("train"), SomeModel)
assert exp.get_criterion("train") is not None
assert exp.get_optimizer("train", SomeModel()) is not None
assert exp.get_scheduler("train", None) is not None
_test_callbacks(callbacks, exp)
def test_when_callback_wrapped():
"""
There should be no default callback of same kind of user defined wrapped
callback.
"""
config = DEFAULT_MINIMAL_CONFIG
config["stages"]["callbacks_params"] = {
"my_wrapped_criterion": {
"_wrapper": {
"callback": "PhaseBatchWrapperCallback",
"active_phases": [1]
},
"callback": "CriterionCallback"
}
}
exp = ConfigExperiment(config=config)
assert "_criterion" not in exp.get_callbacks("train").keys()
callback = exp.get_callbacks("train")["my_wrapped_criterion"]
assert isinstance(callback, PhaseWrapperCallback)
def test_not_implemented_datasets():
"""
Test on ``get_datasets`` method, which should be implememnted by user.
Method ``get_loaders`` will call ``get_dataset``.
"""
exp = ConfigExperiment(config=DEFAULT_MINIMAL_CONFIG.copy())
with pytest.raises(NotImplementedError):
exp.get_loaders("train")
with pytest.raises(NotImplementedError):
exp.get_datasets("train")
def test_defaults():
"""
Test on ConfigExperiment defaults.
It's pretty similar to BaseExperiment's test
but the thing is that those two are very different classes and
inherit from different parent classes.
Also very important to check which callbacks are added by default
"""
exp = ConfigExperiment(config=DEFAULT_MINIMAL_CONFIG.copy())
assert exp.initial_seed == 42
assert exp.logdir == "./logdir"
assert exp.stages == ["train"]
assert exp.distributed_params == {}
assert exp.get_stage_params("train") == {
"logdir": "./logdir",
}
assert isinstance(exp.get_model("train"), SomeModel)
assert exp.get_criterion("train") is None
assert exp.get_optimizer("train", SomeModel()) is None
assert exp.get_scheduler("train", None) is None
_test_callbacks(DEFAULT_CALLBACKS, exp)