def test_cleanup(tmp_dir): dvclive = Live() img = np.ones((500, 500, 3), np.uint8) dvclive.log_image("image.png", img) assert (tmp_dir / dvclive.dir / LiveImage.subfolder / "image.png").exists() Live() assert not (tmp_dir / dvclive.dir / LiveImage.subfolder).exists()
def test_cleanup(tmp_dir, y_true_y_pred_y_score): live = Live() out = tmp_dir / live.dir / Plot.subfolder y_true, y_pred, _ = y_true_y_pred_y_score live.log_plot("confusion_matrix", y_true, y_pred) assert (out / "confusion_matrix.json").exists() Live() assert not (tmp_dir / live.dir / Plot.subfolder).exists()
def test_get_step_resume(tmp_dir): dvclive = Live() for metric in [0.9, 0.8]: dvclive.log("metric", metric) dvclive.next_step() assert dvclive.get_step() == 2 dvclive = Live(resume=True) assert dvclive.get_step() == 2 dvclive = Live(resume=False) assert dvclive.get_step() == 0
def import_dvclive(self, **kwargs): try: from dvclive import Live except ImportError: raise ImportError( 'Please run "pip install dvclive" to install dvclive') self.dvclive = Live(**kwargs)
def test_init_from_env(tmp_dir, html, monkeypatch): monkeypatch.setenv(env.DVCLIVE_PATH, "logs") monkeypatch.setenv(env.DVCLIVE_HTML, str(int(html))) dvclive = Live() assert dvclive._path == "logs" assert dvclive._report == ("html" if html else None)
def __init__(self, model_file=None, save_weights_only: bool = False, **kwargs): super().__init__() self.model_file = model_file self.save_weights_only = save_weights_only self.dvclive = Live(**kwargs)
def test_invalid_metric_type(tmp_dir, invalid_type): dvclive = Live() with pytest.raises( InvalidDataTypeError, match=f"Data 'm' has not supported type {type(invalid_type)}", ): dvclive.log("m", invalid_type)
def test_init_from_env(tmp_dir, summary, html, monkeypatch): monkeypatch.setenv(env.DVCLIVE_PATH, "logs") monkeypatch.setenv(env.DVCLIVE_SUMMARY, str(int(summary))) monkeypatch.setenv(env.DVCLIVE_HTML, str(int(html))) dvclive = Live() assert dvclive._path == "logs" assert dvclive._summary == summary assert dvclive._html == html
def test_continue(tmp_dir, resume, steps, metrics): dvclive = Live("logs") for metric in [0.9, 0.8]: dvclive.log("metric", metric) dvclive.next_step() assert read_history("logs", "metric") == ([0, 1], [0.9, 0.8]) assert read_latest("logs", "metric") == (1, 0.8) dvclive = Live("logs", resume=resume) for new_metric in [0.7, 0.6]: dvclive.log("metric", new_metric) dvclive.next_step() assert read_history("logs", "metric") == (steps, metrics) assert read_latest("logs", "metric") == (last(steps), last(metrics))
def test_PIL(tmp_dir): dvclive = Live() img = Image.new("RGB", (500, 500), (250, 250, 250)) dvclive.log("image.png", img) assert (tmp_dir / dvclive.dir / "image.png").exists() summary = _parse_json("dvclive.json") assert summary["image.png"] == os.path.join(dvclive.dir, "image.png")
def test_dump_kwargs(tmp_dir, y_true_y_pred_y_score, mocker): live = Live() y_true, _, y_score = y_true_y_pred_y_score spy = mocker.spy(metrics, "roc_curve") live.log_plot("roc", y_true, y_score, drop_intermediate=True) spy.assert_called_once_with(y_true, y_score, drop_intermediate=True)
def test_get_step_custom_steps(tmp_dir): dvclive = Live() steps = [0, 62, 1000] metrics = [0.9, 0.8, 0.7] for step, metric in zip(steps, metrics): dvclive.set_step(step) dvclive.log("x", metric) assert dvclive.get_step() == step
def test_require_step_update(tmp_dir, metric): dvclive = Live("logs") dvclive.log(metric, 1.0) with pytest.raises( DataAlreadyLoggedError, match="has already being logged whith step 'None'", ): dvclive.log(metric, 2.0)
def test_get_step_control_flow(tmp_dir): dvclive = Live() while dvclive.get_step() < 10: dvclive.log("i", dvclive.get_step()) dvclive.next_step() steps, values = read_history("dvclive", "i") assert steps == list(range(10)) assert values == [float(x) for x in range(10)]
def test_step_formatting(tmp_dir): dvclive = Live() img = np.ones((500, 500, 3), np.uint8) for _ in range(3): dvclive.log_image("image.png", img) dvclive.next_step() for step in range(3): assert (tmp_dir / dvclive.dir / LiveImage.subfolder / str(step) / "image.png").exists()
def test_step_exception(tmp_dir, y_true_y_pred_y_score): live = Live() out = tmp_dir / live.dir / Plot.subfolder y_true, y_pred, _ = y_true_y_pred_y_score live.log_plot("confusion_matrix", y_true, y_pred) assert (out / "confusion_matrix.json").exists() with pytest.raises(NotImplementedError): live.next_step()
def test_logging_no_step(tmp_dir): dvclive = Live("logs") dvclive.log("m1", 1) assert not (tmp_dir / "logs" / "m1.tsv").is_file() assert (tmp_dir / dvclive.summary_path).is_file() s = _parse_json(dvclive.summary_path) assert s["m1"] == 1 assert "step" not in s
def test_logging_step(tmp_dir, path): dvclive = Live(path) dvclive.log("m1", 1) dvclive.next_step() assert (tmp_dir / path).is_dir() assert (tmp_dir / path / "m1.tsv").is_file() assert (tmp_dir / dvclive.summary_path).is_file() s = _parse_json(dvclive.summary_path) assert s["m1"] == 1 assert s["step"] == 0
def test_get_step_control_flow(tmp_dir): dvclive = Live() out = tmp_dir / dvclive.dir / Scalar.subfolder while dvclive.get_step() < 10: dvclive.log("i", dvclive.get_step()) dvclive.next_step() steps, values = read_history(out, "i") assert steps == list(range(10)) assert values == [float(x) for x in range(10)]
def test_cleanup(tmp_dir, summary, html): dvclive = Live("logs", summary=summary) dvclive.log("m1", 1) dvclive.next_step() html_path = tmp_dir / dvclive.html_path if html: html_path.touch() (tmp_dir / "logs" / "some_user_file.txt").touch() assert (tmp_dir / "logs" / "m1.tsv").is_file() assert (tmp_dir / dvclive.summary_path).is_file() == summary assert html_path.is_file() == html dvclive = Live("logs", summary=summary) assert (tmp_dir / "logs" / "some_user_file.txt").is_file() assert not (tmp_dir / "logs" / "m1.tsv").is_file() assert (tmp_dir / dvclive.summary_path).is_file() == summary assert not (html_path).is_file()
def test_custom_steps(tmp_dir, mocker): dvclive = Live("logs") steps = [0, 62, 1000] metrics = [0.9, 0.8, 0.7] for step, metric in zip(steps, metrics): dvclive.set_step(step) dvclive.log("m", metric) assert read_history("logs", "m") == (steps, metrics) assert read_latest("logs", "m") == (last(steps), last(metrics))
def test_cleanup(tmp_dir, html): dvclive = Live("logs", report="html" if html else None) dvclive.log("m1", 1) dvclive.next_step() html_path = tmp_dir / dvclive.html_path if html: html_path.touch() (tmp_dir / "logs" / "some_user_file.txt").touch() assert (tmp_dir / dvclive.dir / Scalar.subfolder / "m1.tsv").is_file() assert (tmp_dir / dvclive.summary_path).is_file() assert html_path.is_file() == html dvclive = Live("logs") assert (tmp_dir / "logs" / "some_user_file.txt").is_file() assert not (tmp_dir / dvclive.dir / Scalar.subfolder).exists() assert not (tmp_dir / dvclive.summary_path).is_file() assert not (html_path).is_file()
def test_log_prc_curve(tmp_dir, y_true_y_pred_y_score, mocker): live = Live() out = tmp_dir / live.dir / Plot.subfolder y_true, _, y_score = y_true_y_pred_y_score spy = mocker.spy(metrics, "precision_recall_curve") live.log_plot("precision_recall", y_true, y_score) spy.assert_called_once_with(y_true, y_score) assert (out / "precision_recall.json").exists()
def test_make_report_open(tmp_dir, mocker): mocked_open = mocker.patch("webbrowser.open") live = Live() live.log_plot("confusion_matrix", [0, 0, 1, 1], [1, 0, 0, 1]) live.make_report() live.make_report() mocked_open.assert_called_once() mocked_open = mocker.patch("webbrowser.open") live = Live(auto_open=False) live.log_plot("confusion_matrix", [0, 0, 1, 1], [1, 0, 0, 1]) live.make_report() assert not mocked_open.called mocked_open = mocker.patch("webbrowser.open") live = Live(report=None) live.log("foo", 1) live.next_step() assert not mocked_open.called
def test_log_calibration_curve(tmp_dir, y_true_y_pred_y_score, mocker): live = Live() out = tmp_dir / live.dir / Plot.subfolder y_true, _, y_score = y_true_y_pred_y_score spy = mocker.spy(calibration, "calibration_curve") live.log_plot("calibration", y_true, y_score) spy.assert_called_once_with(y_true, y_score) assert (out / "calibration.json").exists()
def test_log_confusion_matrix(tmp_dir, y_true_y_pred_y_score, mocker): live = Live() out = tmp_dir / live.dir / Plot.subfolder y_true, y_pred, _ = y_true_y_pred_y_score live.log_plot("confusion_matrix", y_true, y_pred) cm = json.loads((out / "confusion_matrix.json").read_text()) assert isinstance(cm, list) assert isinstance(cm[0], dict) assert cm[0]["actual"] == str(y_true[0]) assert cm[0]["predicted"] == str(y_pred[0])
def test_html(tmp_dir, dvc_repo, html, signal_exists, monkeypatch): if dvc_repo: from dvc.repo import Repo Repo.init(no_scm=True) monkeypatch.setenv(env.DVCLIVE_PATH, "logs") monkeypatch.setenv(env.DVCLIVE_HTML, str(int(html))) dvclive = Live() dvclive.log("m1", 1) dvclive.next_step() assert (tmp_dir / ".dvc" / "tmp" / SIGNAL_FILE).is_file() == signal_exists
def test_log_reset_with_set_step(tmp_dir): dvclive = Live() for i in range(3): dvclive.set_step(i) dvclive.log("train_m", 1) for i in range(3): dvclive.set_step(i) dvclive.log("val_m", 1) assert read_history("dvclive", "train_m") == ([0, 1, 2], [1, 1, 1]) assert read_history("dvclive", "val_m") == ([0, 1, 2], [1, 1, 1]) assert read_latest("dvclive", "train_m") == (2, 1) assert read_latest("dvclive", "val_m") == (2, 1)
def test_custom_steps(tmp_dir): dvclive = Live("logs") out = tmp_dir / dvclive.dir / Scalar.subfolder steps = [0, 62, 1000] metrics = [0.9, 0.8, 0.7] for step, metric in zip(steps, metrics): dvclive.set_step(step) dvclive.log("m", metric) assert read_history(out, "m") == (steps, metrics) assert read_latest(out, "m") == (last(steps), last(metrics))
def experiment(self): r""" Actual DVCLive object. To use DVCLive features in your :class:`~LightningModule` do the following. Example:: self.logger.experiment.some_dvclive_function() """ if self._experiment is not None: return self._experiment else: assert (rank_zero_only.rank == 0 ), "tried to init log dirs in non global_rank=0" self._experiment = Live(**self._dvclive_init) return self._experiment