def test_mmcv_hook(tmp_dir, mocker): work_dir = tmp_dir / "work_dir" runner = _build_demo_runner(str(work_dir)) log_config = dict( interval=1, hooks=[ dict(type="TextLoggerHook"), dict(type="DvcliveLoggerHook", model_file=tmp_dir / "model.pth"), ], ) runner.register_logger_hooks(log_config) set_step = mocker.spy(dvclive.Live, "set_step") log = mocker.spy(dvclive.Live, "log") loader = torch.utils.data.DataLoader(torch.ones((5, 2))) runner.run([loader, loader], [("train", 1), ("val", 1)]) assert set_step.call_count == 6 assert log.call_count == 12 logs, _ = read_logs(tmp_dir / "dvclive" / Scalar.subfolder) assert "learning_rate" in logs assert "momentum" in logs
def test_xgb_integration(tmp_dir, train_params, iris_data): xgb.train( train_params, iris_data, callbacks=[DvcLiveCallback("eval_data")], num_boost_round=5, evals=[(iris_data, "eval_data")], ) assert os.path.exists("dvclive") logs, _ = read_logs(tmp_dir / "dvclive" / Scalar.subfolder) assert len(logs) == 1 assert len(first(logs.values())) == 5
def test_keras_callback(tmp_dir, xor_model, capture_wrap): model, x, y = xor_model() model.fit( x, y, epochs=1, batch_size=1, callbacks=[DvcLiveCallback()], ) assert os.path.exists("dvclive") logs, _ = read_logs(tmp_dir / "dvclive" / Scalar.subfolder) assert "accuracy" in logs
def test_lgbm_integration(tmp_dir, model_params, iris_data): model = lgbm.LGBMClassifier() model.set_params(**model_params) model.fit( iris_data[0][0], iris_data[0][1], eval_set=(iris_data[1][0], iris_data[1][1]), eval_metric=["multi_logloss"], callbacks=[DvcLiveCallback()], ) assert os.path.exists("dvclive") logs, _ = read_logs("dvclive") assert len(logs) == 1 assert len(first(logs.values())) == 5
def test_lightning_integration(tmp_dir): # init model model = LitMNIST() # init logger dvclive_logger = DvcLiveLogger("test_run", path="logs") trainer = Trainer(logger=dvclive_logger, max_epochs=1, checkpoint_callback=False) trainer.fit(model) assert os.path.exists("logs") assert not os.path.exists("DvcLiveLogger") logs, _ = read_logs(tmp_dir / "logs" / Scalar.subfolder) assert len(logs) == 3 assert "train_loss_step" in logs assert "train_loss_epoch" in logs assert "epoch" in logs
def test_mmcv_hook(tmp_dir, mocker): work_dir = tmp_dir / "work_dir" runner = _build_demo_runner(str(work_dir)) hook = DVCLiveLoggerHook() runner.register_hook(hook, priority="VERY_LOW") next_step = mocker.spy(dvclive.metrics.MetricLogger, "next_step") log = mocker.spy(dvclive.metrics.MetricLogger, "log") loader = torch.utils.data.DataLoader(torch.ones((5, 2))) dvclive.init("logs") runner.run([loader, loader], [("train", 1), ("val", 1)]) assert next_step.call_count == 5 assert log.call_count == 12 logs, _ = read_logs("logs") assert "learning_rate" in logs assert "momentum" in logs
def test_huggingface_integration(tmp_dir, model, args, data, tokenizer): trainer = Trainer( model, args, train_dataset=data[0], eval_dataset=data[1], tokenizer=tokenizer, compute_metrics=compute_metrics, ) trainer.add_callback(DvcLiveCallback()) trainer.train() assert os.path.exists("dvclive") logs, _ = read_logs(tmp_dir / "dvclive" / Scalar.subfolder) assert len(logs) == 10 assert "eval_matthews_correlation" in logs assert "eval_loss" in logs assert len(logs["epoch"]) == 3 assert len(logs["eval_loss"]) == 2