def test_controll_flow_callback_filter_fn_ignore_epochs(): wraped = ControlFlowCallbackWrapper(DummyCallback(), ignore_epochs=[3, 4, 6, 8]) mask = [ True, True, False, False, True, False, True, False, True, True, ] expected = { "train": mask, "valid": mask, } actual = {loader: [] for loader in expected.keys()} for epoch in range(1, 10 + 1): for loader in expected.keys(): runner = _Runner(loader, epoch) wraped.on_loader_start(runner) actual[loader].append(wraped._is_enabled) assert actual == expected
def test_ignore_foo_with_wrong_args(self): orders = ( CallbackOrder.Internal, CallbackOrder.Metric, CallbackOrder.MetricAggregation, CallbackOrder.Optimizer, CallbackOrder.Scheduler, CallbackOrder.External, ) order = random.choice(orders) callback = RaiserCallback(order, "on_epoch_start") with self.assertRaises(ValueError): ControlFlowCallbackWrapper(callback, filter_fn=12345) with self.assertRaises(ValueError): ControlFlowCallbackWrapper(callback, filter_fn=lambda arg: True) with self.assertRaises(ValueError): ControlFlowCallbackWrapper(callback, filter_fn=lambda *args: True) with self.assertRaises(ValueError): ControlFlowCallbackWrapper( callback, filter_fn=lambda one, two, three, four: True) with self.assertRaises(ValueError): ControlFlowCallbackWrapper(callback, filter_fn=lambda *args, **kwargs: True)
def test_control_flow_callback_filter_fn_loaders(): wraped = ControlFlowCallbackWrapper(DummyCallback(), loaders=["valid"]) expected = { "train": [False] * 5, "valid": [True] * 5, "another_loader": [False] * 5, "like_valid": [False] * 5, } actual = {loader: [] for loader in expected.keys()} for epoch in range(1, 5 + 1): for loader in expected.keys(): runner = _Runner(loader, epoch) wraped.on_loader_start(runner) actual[loader].append(wraped._is_enabled) assert actual == expected
def test_controll_flow_callback_filter_fn_periodical_ignore_epochs(): wraped = ControlFlowCallbackWrapper(DummyCallback(), ignore_epochs=4) mask = [i % 4 != 0 for i in range(1, 10 + 1)] expected = { "train": mask, "valid": mask, "another_loader": mask, "like_valid": mask, } actual = {loader: [] for loader in expected.keys()} for epoch in range(1, 10 + 1): for loader in expected.keys(): runner = _Runner(loader, epoch) wraped.on_loader_start(runner) actual[loader].append(wraped._is_enabled) assert actual == expected
def test_filter_fn_with_err_in_eval(self): orders = ( CallbackOrder.Internal, CallbackOrder.Metric, CallbackOrder.MetricAggregation, CallbackOrder.Optimizer, CallbackOrder.Scheduler, CallbackOrder.External, ) events = ( "on_loader_start", "on_loader_end", "on_experiment_start", "on_experiment_end", "on_epoch_start", "on_epoch_end", "on_batch_start", "on_batch_end", "on_exception", ) for event in events: for order in orders: callback = RaiserCallback(order, event) with self.assertRaises(ValueError): ControlFlowCallbackWrapper(callback, filter_fn="lambda e, l")
def test_ignore_epochs_with_wrong_args(self): orders = ( CallbackOrder.Internal, CallbackOrder.Metric, CallbackOrder.MetricAggregation, CallbackOrder.Optimizer, CallbackOrder.Scheduler, CallbackOrder.External, ) order = random.choice(orders) callback = RaiserCallback(order, "on_epoch_start") with self.assertRaises(ValueError): ControlFlowCallbackWrapper(callback, ignore_epochs=None) with self.assertRaises(ValueError): ControlFlowCallbackWrapper(callback, ignore_epochs="123456")
def test_ignore_loaders_with_wrong_args(self): orders = ( CallbackOrder.Internal, CallbackOrder.Metric, CallbackOrder.MetricAggregation, CallbackOrder.Optimizer, CallbackOrder.Scheduler, CallbackOrder.External, ) order = random.choice(orders) callback = RaiserCallback(order, "on_epoch_start") with self.assertRaises(ValueError): ControlFlowCallbackWrapper(callback, ignore_loaders=1234.56) with self.assertRaises(ValueError): ControlFlowCallbackWrapper(callback, ignore_loaders=1234.56) with self.assertRaises(ValueError): ControlFlowCallbackWrapper( callback, ignore_loaders={"train": ["", "fjdskjfdk", "1234"]})
def test_with_missing_args(self): orders = ( CallbackOrder.Internal, CallbackOrder.Metric, CallbackOrder.MetricAggregation, CallbackOrder.Optimizer, CallbackOrder.Scheduler, CallbackOrder.External, ) for order in orders: callback = RaiserCallback(order, "on_epoch_start") with self.assertRaises(ValueError): ControlFlowCallbackWrapper(callback)
def test_filter_fn_with_eval(self): runner = Mock(loader_key="train", epoch=1) orders = ( CallbackOrder.Internal, CallbackOrder.Metric, CallbackOrder.MetricAggregation, CallbackOrder.Optimizer, CallbackOrder.Scheduler, CallbackOrder.External, ) for order in orders: callback = RaiserCallback(order, "on_loader_start") wrapper = ControlFlowCallbackWrapper( callback, filter_fn="lambda e, l: False") wrapper.on_loader_start(runner) callback = RaiserCallback(order, "on_loader_start") wrapper = ControlFlowCallbackWrapper(callback, filter_fn="lambda e, l: True") with self.assertRaises(Dummy): wrapper.on_loader_start(runner) events = ( "on_loader_end", "on_experiment_start", "on_experiment_end", "on_epoch_start", "on_epoch_end", "on_batch_start", "on_batch_end", "on_exception", ) for event in events: for order in orders: callback = RaiserCallback(order, event) wrapper = ControlFlowCallbackWrapper( callback, filter_fn="lambda e, l: False") wrapper.on_loader_start(runner) wrapper.__getattribute__(event)(runner) callback = RaiserCallback(order, event) wrapper = ControlFlowCallbackWrapper( callback, filter_fn="lambda e, l: True") wrapper.on_loader_start(runner) with self.assertRaises(Dummy): wrapper.__getattribute__(event)(runner)
def test_filter_fn_with_wrong_args(self): runner = Mock(loader_key="train", epoch=1) orders = ( CallbackOrder.Internal, CallbackOrder.Metric, CallbackOrder.MetricAggregation, CallbackOrder.Optimizer, CallbackOrder.Scheduler, CallbackOrder.External, ) def _ignore_foo(epoch: int, loader: str) -> bool: return False def _raise_foo(epoch: int, loader: str) -> bool: return True for order in orders: callback = RaiserCallback(order, "on_loader_start") wrapper = ControlFlowCallbackWrapper(callback, filter_fn=_ignore_foo) wrapper.on_loader_start(runner) callback = RaiserCallback(order, "on_loader_start") wrapper = ControlFlowCallbackWrapper(callback, filter_fn=_raise_foo) with self.assertRaises(Dummy): wrapper.on_loader_start(runner) events = ( "on_loader_end", "on_experiment_start", "on_experiment_end", "on_epoch_start", "on_epoch_end", "on_batch_start", "on_batch_end", "on_exception", ) for event in events: for order in orders: callback = RaiserCallback(order, event) wrapper = ControlFlowCallbackWrapper(callback, filter_fn=_ignore_foo) wrapper.on_loader_start(runner) wrapper.__getattribute__(event)(runner) callback = RaiserCallback(order, event) wrapper = ControlFlowCallbackWrapper(callback, filter_fn=_raise_foo) wrapper.on_loader_start(runner) with self.assertRaises(Dummy): wrapper.__getattribute__(event)(runner)