def test_evaluator_dict(path): device = 'cpu' progress_bar = False model = MyModel() ppe.to(model, device) model_with_loss = MyModelWithLossFn(model) optimizer = torch.optim.SGD(model.parameters(), lr=0.1) data = torch.utils.data.DataLoader([(torch.rand(20, ), torch.rand(10, )) for i in range(10)]) extensions = _make_extensions() evaluator1 = engine.create_evaluator(model_with_loss, device=device, progress_bar=progress_bar) evaluator2 = engine.create_evaluator(model, device=device, progress_bar=progress_bar) trainer = engine.create_trainer( model_with_loss, optimizer, 20, device=device, evaluator={ '1': evaluator1, # called 20 times. '2': (evaluator2, (5, 'iteration')), # called 40 times. }, extensions=extensions, out_dir=path) path = 'pytorch_pfn_extras.training._evaluator.Evaluator.run' with mock.patch(path) as patched: trainer.run(data, data) assert patched.call_count == 20 + 40
def test_trainer_with_code_block(device, progress_bar, path): model = MyModel() model_with_loss = MyModelWithLossDictOutput(model) ppe.to(model_with_loss, device) optimizer = torch.optim.SGD(model.parameters(), lr=0.1) data = torch.utils.data.DataLoader([{ 'x': torch.rand(20, ), 't': torch.rand(10, ) } for i in range(10)]) extensions = _make_extensions() evaluator = engine.create_evaluator(model_with_loss, device=device, progress_bar=progress_bar, logic=ppe.handler.CodeBlockLogic()) trainer = engine.create_trainer(model_with_loss, optimizer, 20, device=device, evaluator=evaluator, extensions=extensions, out_dir=path, logic=ppe.handler.CodeBlockLogic()) trainer.run(data, data)
def test_evaluator_trigger(evaluator_trigger, path): device = 'cpu' progress_bar = False model = MyModel() ppe.to(model, device) model_with_loss = MyModelWithLossFn(model) optimizer = torch.optim.SGD(model.parameters(), lr=0.1) data = torch.utils.data.DataLoader([(torch.rand(20, ), torch.rand(10, )) for i in range(10)]) extensions = _make_extensions() evaluator = engine.create_evaluator(model_with_loss, device=device, progress_bar=progress_bar) trainer = engine.create_trainer(model_with_loss, optimizer, 20, device=device, evaluator=(evaluator, evaluator_trigger[1]), extensions=extensions, out_dir=path) path = 'pytorch_pfn_extras.training._evaluator._Evaluator.run' with mock.patch(path) as patched: trainer.run(data, data) assert patched.call_count == evaluator_trigger[0]
def test_evaluator_async(accuracy): device = 'async-cpu' model = AsyncModel(accuracy) data = torch.utils.data.DataLoader([{ 'x': torch.rand(20), 't': torch.rand(1) } for i in range(1000)], batch_size=10) options = {'eval_report_keys': ['accuracy'], 'async': True} # Register the handler ppe.runtime.runtime_registry.register(device, DeferRuntime) ppe.to(model, device) evaluator = engine.create_evaluator( model, device=device, options=options, metrics=[ppe.training.metrics.AccuracyMetric('t', 'y')]) reporter = ppe.reporting.Reporter() observation = {} with reporter.scope(observation): evaluator.run(data) assert pytest.approx(observation['val/accuracy'], accuracy) assert model._pending_called
def test_trainer_profile(): device = 'cpu' model = MyModel() model_with_loss = MyModelWithLossDictOutput(model) ppe.to(model_with_loss, device) optimizer = torch.optim.SGD(model.parameters(), lr=0.1) data = torch.utils.data.DataLoader([{ 'x': torch.rand(20, ), 't': torch.rand(10, ) } for i in range(10)]) extensions = _make_extensions() evaluator = engine.create_evaluator(model_with_loss, device=device) trace_handler = mock.Mock() warmup = 1 active = len(data) - warmup profile = torch.profiler.profile( activities=[torch.profiler.ProfilerActivity.CPU], on_trace_ready=trace_handler, schedule=torch.profiler.schedule(wait=0, warmup=warmup, active=active), ) trainer = engine.create_trainer( model_with_loss, optimizer, 20, device=device, evaluator=evaluator, extensions=extensions, profile=profile, ) trainer.run(data, data) assert trace_handler.call_count == 20 # n_epochs
def test_trainer_with_code_block_with_multiple_optimizers( device, progress_bar, path): if not torch.cuda.is_available() and device == 'cuda': pytest.skip() model = MyModel() model_with_loss = MyModelWithLossDictOutput(model) ppe.to(model_with_loss, device) optimizer0 = torch.optim.SGD(model.parameters(), lr=0.1) optimizer1 = torch.optim.Adam(model.parameters(), lr=0.1) data = torch.utils.data.DataLoader([{ 'x': torch.rand(20, ), 't': torch.rand(10, ) } for i in range(10)]) extensions = _make_extensions() evaluator = engine.create_evaluator(model_with_loss, device=device, progress_bar=progress_bar, logic=ppe.handler.CodeBlockLogic()) trainer = engine.create_trainer(model_with_loss, { "0": optimizer0, "1": optimizer1 }, 20, device=device, evaluator=evaluator, extensions=extensions, out_dir=path, logic=ppe.handler.CodeBlockLogic()) trainer.run(data, data)
def test_trainer_dict_input(device, progress_bar, path): if not torch.cuda.is_available() and device == 'cuda': pytest.skip() model = MyModel() ppe.to(model, device) model_with_loss = MyModelWithLossDictOutput(model) optimizer = torch.optim.SGD(model.parameters(), lr=0.1) data = torch.utils.data.DataLoader([{ 'x': torch.rand(20, ), 't': torch.rand(10, ) } for i in range(10)]) extensions = _make_extensions() evaluator = engine.create_evaluator(model_with_loss, device=device, progress_bar=progress_bar) trainer = engine.create_trainer(model_with_loss, optimizer, 20, device=device, evaluator=evaluator, extensions=extensions, out_dir=path) trainer.run(data, data)
def test_train_with_evaluator(device, progress_bar, path): if not torch.cuda.is_available() and device == 'cuda': pytest.skip() model = MyModel() ppe.to(model, device) model_with_loss = MyModelWithLossFn(model) optimizer = torch.optim.SGD(model.parameters(), lr=0.1) data = torch.utils.data.DataLoader([(torch.rand(20, ), torch.rand(10, )) for i in range(10)]) extensions = _make_extensions() evaluator = engine.create_evaluator(model_with_loss, device=device, progress_bar=progress_bar) trainer = engine.create_trainer(model_with_loss, optimizer, 20, device=device, evaluator=evaluator, extensions=extensions, out_dir=path) mpath = 'pytorch_pfn_extras.training._evaluator.Evaluator.run' with mock.patch(mpath) as patched: trainer.run(data, data) assert patched.call_count == 20
def test_evaluator_with_metric(device, accuracy): model = MyModel(accuracy) data = torch.utils.data.DataLoader([{ 'x': torch.rand(20), 't': torch.rand(1) } for i in range(10)], batch_size=10) evaluator = engine.create_evaluator( model, device=device, metrics=[ppe.training.metrics.AccuracyMetric('t', 'y')], options={'eval_report_keys': ['accuracy']}) evaluator.handler.eval_setup(evaluator, data) reporter = ppe.reporting.Reporter() observation = {} with reporter.scope(observation): evaluator.run(data) assert pytest.approx(observation['val/accuracy'], accuracy)
def test_trainer_namedtuple_input(device, progress_bar, path): model = MyModel() ppe.to(model, device) model_with_loss = ModelNamedTupleIO(model) optimizer = torch.optim.SGD(model.parameters(), lr=0.1) data = torch.utils.data.DataLoader( [Input(torch.rand(20, ), torch.rand(10, ), str(i)) for i in range(10)]) extensions = _make_extensions() evaluator = engine.create_evaluator(model_with_loss, device=device, progress_bar=progress_bar) trainer = engine.create_trainer(model_with_loss, optimizer, 20, device=device, evaluator=evaluator, extensions=extensions, out_dir=path) trainer.run(data, data)
def add_dump(self, name, dir): with open(f'{dir}/summary') as f: summary = json.loads(f.read()) class DummyModel(torch.nn.Module): def __init__(self): super().__init__() self.param = torch.nn.Parameter(torch.tensor(1.0)) def forward(self, *args, **kwargs): return () model = DummyModel() ppe.to(model, 'cpu') engine = None args = [] if summary['evaluator']: engine = _engine_module.create_evaluator(model) args = [[None] * summary['eval_len']] if summary['trainer']: engine = _engine_module.create_trainer( {'main': model}, {'main': torch.optim.SGD(model.parameters(), lr=0.01)}, summary['max_epochs'], evaluator=engine, ) args = [[None] * summary['train_len']] + args engine.handler = _ComparableHandler( engine.handler, name, self._compare_dump, self._trigger, dir=dir) child_evaluator = getattr(engine, 'evaluator', None) if child_evaluator is not None: # For trainer with evaluator child_evaluator.handler = engine.handler self._engines[name] = engine, args, {}
def test_evaluator_async(accuracy): device = 'cpu' model = AsyncModel(accuracy) data = torch.utils.data.DataLoader([{ 'x': torch.rand(20), 't': torch.rand(1) } for i in range(1000)], batch_size=10) options = {'eval_report_keys': ['accuracy']} ppe.to(model, device) evaluator = engine.create_evaluator( model, device=device, options=options, metrics=[ppe.training.metrics.AccuracyMetric('t', 'y')]) reporter = ppe.reporting.Reporter() observation = {} with reporter.scope(observation): evaluator.run(data) assert pytest.approx(observation['val/accuracy']) == accuracy