def test_cleanup(tmp_dir):
dvclive = Live()
img = np.ones((500, 500, 3), np.uint8)
dvclive.log_image("image.png", img)
assert (tmp_dir / dvclive.dir / LiveImage.subfolder / "image.png").exists()
Live()
assert not (tmp_dir / dvclive.dir / LiveImage.subfolder).exists()
def test_cleanup(tmp_dir, y_true_y_pred_y_score):
live = Live()
out = tmp_dir / live.dir / Plot.subfolder
y_true, y_pred, _ = y_true_y_pred_y_score
live.log_plot("confusion_matrix", y_true, y_pred)
assert (out / "confusion_matrix.json").exists()
Live()
assert not (tmp_dir / live.dir / Plot.subfolder).exists()
def test_get_step_resume(tmp_dir):
dvclive = Live()
for metric in [0.9, 0.8]:
dvclive.log("metric", metric)
dvclive.next_step()
assert dvclive.get_step() == 2
dvclive = Live(resume=True)
assert dvclive.get_step() == 2
dvclive = Live(resume=False)
assert dvclive.get_step() == 0
def import_dvclive(self, **kwargs):
try:
from dvclive import Live
except ImportError:
raise ImportError(
'Please run "pip install dvclive" to install dvclive')
self.dvclive = Live(**kwargs)
def test_init_from_env(tmp_dir, html, monkeypatch):
monkeypatch.setenv(env.DVCLIVE_PATH, "logs")
monkeypatch.setenv(env.DVCLIVE_HTML, str(int(html)))
dvclive = Live()
assert dvclive._path == "logs"
assert dvclive._report == ("html" if html else None)
def __init__(self,
model_file=None,
save_weights_only: bool = False,
**kwargs):
super().__init__()
self.model_file = model_file
self.save_weights_only = save_weights_only
self.dvclive = Live(**kwargs)
def test_invalid_metric_type(tmp_dir, invalid_type):
dvclive = Live()
with pytest.raises(
InvalidDataTypeError,
match=f"Data 'm' has not supported type {type(invalid_type)}",
):
dvclive.log("m", invalid_type)
def test_init_from_env(tmp_dir, summary, html, monkeypatch):
monkeypatch.setenv(env.DVCLIVE_PATH, "logs")
monkeypatch.setenv(env.DVCLIVE_SUMMARY, str(int(summary)))
monkeypatch.setenv(env.DVCLIVE_HTML, str(int(html)))
dvclive = Live()
assert dvclive._path == "logs"
assert dvclive._summary == summary
assert dvclive._html == html
def test_continue(tmp_dir, resume, steps, metrics):
dvclive = Live("logs")
for metric in [0.9, 0.8]:
dvclive.log("metric", metric)
dvclive.next_step()
assert read_history("logs", "metric") == ([0, 1], [0.9, 0.8])
assert read_latest("logs", "metric") == (1, 0.8)
dvclive = Live("logs", resume=resume)
for new_metric in [0.7, 0.6]:
dvclive.log("metric", new_metric)
dvclive.next_step()
assert read_history("logs", "metric") == (steps, metrics)
assert read_latest("logs", "metric") == (last(steps), last(metrics))
def test_PIL(tmp_dir):
dvclive = Live()
img = Image.new("RGB", (500, 500), (250, 250, 250))
dvclive.log("image.png", img)
assert (tmp_dir / dvclive.dir / "image.png").exists()
summary = _parse_json("dvclive.json")
assert summary["image.png"] == os.path.join(dvclive.dir, "image.png")
def test_dump_kwargs(tmp_dir, y_true_y_pred_y_score, mocker):
live = Live()
y_true, _, y_score = y_true_y_pred_y_score
spy = mocker.spy(metrics, "roc_curve")
live.log_plot("roc", y_true, y_score, drop_intermediate=True)
spy.assert_called_once_with(y_true, y_score, drop_intermediate=True)
def test_get_step_custom_steps(tmp_dir):
dvclive = Live()
steps = [0, 62, 1000]
metrics = [0.9, 0.8, 0.7]
for step, metric in zip(steps, metrics):
dvclive.set_step(step)
dvclive.log("x", metric)
assert dvclive.get_step() == step
def test_require_step_update(tmp_dir, metric):
dvclive = Live("logs")
dvclive.log(metric, 1.0)
with pytest.raises(
DataAlreadyLoggedError,
match="has already being logged whith step 'None'",
):
dvclive.log(metric, 2.0)
def test_get_step_control_flow(tmp_dir):
dvclive = Live()
while dvclive.get_step() < 10:
dvclive.log("i", dvclive.get_step())
dvclive.next_step()
steps, values = read_history("dvclive", "i")
assert steps == list(range(10))
assert values == [float(x) for x in range(10)]
def test_step_formatting(tmp_dir):
dvclive = Live()
img = np.ones((500, 500, 3), np.uint8)
for _ in range(3):
dvclive.log_image("image.png", img)
dvclive.next_step()
for step in range(3):
assert (tmp_dir / dvclive.dir / LiveImage.subfolder / str(step) /
"image.png").exists()
def test_step_exception(tmp_dir, y_true_y_pred_y_score):
live = Live()
out = tmp_dir / live.dir / Plot.subfolder
y_true, y_pred, _ = y_true_y_pred_y_score
live.log_plot("confusion_matrix", y_true, y_pred)
assert (out / "confusion_matrix.json").exists()
with pytest.raises(NotImplementedError):
live.next_step()
def test_logging_no_step(tmp_dir):
dvclive = Live("logs")
dvclive.log("m1", 1)
assert not (tmp_dir / "logs" / "m1.tsv").is_file()
assert (tmp_dir / dvclive.summary_path).is_file()
s = _parse_json(dvclive.summary_path)
assert s["m1"] == 1
assert "step" not in s
def test_logging_step(tmp_dir, path):
dvclive = Live(path)
dvclive.log("m1", 1)
dvclive.next_step()
assert (tmp_dir / path).is_dir()
assert (tmp_dir / path / "m1.tsv").is_file()
assert (tmp_dir / dvclive.summary_path).is_file()
s = _parse_json(dvclive.summary_path)
assert s["m1"] == 1
assert s["step"] == 0
def test_get_step_control_flow(tmp_dir):
dvclive = Live()
out = tmp_dir / dvclive.dir / Scalar.subfolder
while dvclive.get_step() < 10:
dvclive.log("i", dvclive.get_step())
dvclive.next_step()
steps, values = read_history(out, "i")
assert steps == list(range(10))
assert values == [float(x) for x in range(10)]
def test_cleanup(tmp_dir, summary, html):
dvclive = Live("logs", summary=summary)
dvclive.log("m1", 1)
dvclive.next_step()
html_path = tmp_dir / dvclive.html_path
if html:
html_path.touch()
(tmp_dir / "logs" / "some_user_file.txt").touch()
assert (tmp_dir / "logs" / "m1.tsv").is_file()
assert (tmp_dir / dvclive.summary_path).is_file() == summary
assert html_path.is_file() == html
dvclive = Live("logs", summary=summary)
assert (tmp_dir / "logs" / "some_user_file.txt").is_file()
assert not (tmp_dir / "logs" / "m1.tsv").is_file()
assert (tmp_dir / dvclive.summary_path).is_file() == summary
assert not (html_path).is_file()
def test_custom_steps(tmp_dir, mocker):
dvclive = Live("logs")
steps = [0, 62, 1000]
metrics = [0.9, 0.8, 0.7]
for step, metric in zip(steps, metrics):
dvclive.set_step(step)
dvclive.log("m", metric)
assert read_history("logs", "m") == (steps, metrics)
assert read_latest("logs", "m") == (last(steps), last(metrics))
def test_cleanup(tmp_dir, html):
dvclive = Live("logs", report="html" if html else None)
dvclive.log("m1", 1)
dvclive.next_step()
html_path = tmp_dir / dvclive.html_path
if html:
html_path.touch()
(tmp_dir / "logs" / "some_user_file.txt").touch()
assert (tmp_dir / dvclive.dir / Scalar.subfolder / "m1.tsv").is_file()
assert (tmp_dir / dvclive.summary_path).is_file()
assert html_path.is_file() == html
dvclive = Live("logs")
assert (tmp_dir / "logs" / "some_user_file.txt").is_file()
assert not (tmp_dir / dvclive.dir / Scalar.subfolder).exists()
assert not (tmp_dir / dvclive.summary_path).is_file()
assert not (html_path).is_file()
def test_log_prc_curve(tmp_dir, y_true_y_pred_y_score, mocker):
live = Live()
out = tmp_dir / live.dir / Plot.subfolder
y_true, _, y_score = y_true_y_pred_y_score
spy = mocker.spy(metrics, "precision_recall_curve")
live.log_plot("precision_recall", y_true, y_score)
spy.assert_called_once_with(y_true, y_score)
assert (out / "precision_recall.json").exists()
def test_make_report_open(tmp_dir, mocker):
mocked_open = mocker.patch("webbrowser.open")
live = Live()
live.log_plot("confusion_matrix", [0, 0, 1, 1], [1, 0, 0, 1])
live.make_report()
live.make_report()
mocked_open.assert_called_once()
mocked_open = mocker.patch("webbrowser.open")
live = Live(auto_open=False)
live.log_plot("confusion_matrix", [0, 0, 1, 1], [1, 0, 0, 1])
live.make_report()
assert not mocked_open.called
mocked_open = mocker.patch("webbrowser.open")
live = Live(report=None)
live.log("foo", 1)
live.next_step()
assert not mocked_open.called
def test_log_calibration_curve(tmp_dir, y_true_y_pred_y_score, mocker):
live = Live()
out = tmp_dir / live.dir / Plot.subfolder
y_true, _, y_score = y_true_y_pred_y_score
spy = mocker.spy(calibration, "calibration_curve")
live.log_plot("calibration", y_true, y_score)
spy.assert_called_once_with(y_true, y_score)
assert (out / "calibration.json").exists()
def test_log_confusion_matrix(tmp_dir, y_true_y_pred_y_score, mocker):
live = Live()
out = tmp_dir / live.dir / Plot.subfolder
y_true, y_pred, _ = y_true_y_pred_y_score
live.log_plot("confusion_matrix", y_true, y_pred)
cm = json.loads((out / "confusion_matrix.json").read_text())
assert isinstance(cm, list)
assert isinstance(cm[0], dict)
assert cm[0]["actual"] == str(y_true[0])
assert cm[0]["predicted"] == str(y_pred[0])
def test_html(tmp_dir, dvc_repo, html, signal_exists, monkeypatch):
if dvc_repo:
from dvc.repo import Repo
Repo.init(no_scm=True)
monkeypatch.setenv(env.DVCLIVE_PATH, "logs")
monkeypatch.setenv(env.DVCLIVE_HTML, str(int(html)))
dvclive = Live()
dvclive.log("m1", 1)
dvclive.next_step()
assert (tmp_dir / ".dvc" / "tmp" / SIGNAL_FILE).is_file() == signal_exists
def test_log_reset_with_set_step(tmp_dir):
dvclive = Live()
for i in range(3):
dvclive.set_step(i)
dvclive.log("train_m", 1)
for i in range(3):
dvclive.set_step(i)
dvclive.log("val_m", 1)
assert read_history("dvclive", "train_m") == ([0, 1, 2], [1, 1, 1])
assert read_history("dvclive", "val_m") == ([0, 1, 2], [1, 1, 1])
assert read_latest("dvclive", "train_m") == (2, 1)
assert read_latest("dvclive", "val_m") == (2, 1)
def test_custom_steps(tmp_dir):
dvclive = Live("logs")
out = tmp_dir / dvclive.dir / Scalar.subfolder
steps = [0, 62, 1000]
metrics = [0.9, 0.8, 0.7]
for step, metric in zip(steps, metrics):
dvclive.set_step(step)
dvclive.log("m", metric)
assert read_history(out, "m") == (steps, metrics)
assert read_latest(out, "m") == (last(steps), last(metrics))
def experiment(self):
r"""
Actual DVCLive object. To use DVCLive features in your
:class:`~LightningModule` do the following.
Example::
self.logger.experiment.some_dvclive_function()
"""
if self._experiment is not None:
return self._experiment
else:
assert (rank_zero_only.rank == 0
), "tried to init log dirs in non global_rank=0"
self._experiment = Live(**self._dvclive_init)
return self._experiment