def main(
train_dir: str,
train_extra_dir: str,
test_dir: str,
test_extra_dir: str,
param_path: str,
param_parent_key: str,
out_model_path: str,
out_encoder_path: str,
out_metrics_path: str,
out_summaries_path: str,
sigmoid_head: bool,
):
if sigmoid_head:
mlflow.set_experiment("siamese-sigmoid")
else:
mlflow.set_experiment("siamese-distance")
mlflow.tensorflow.autolog(every_n_iter=1)
dvclive.init(out_metrics_path, summary=True, html=True)
with open(param_path, "r") as f:
train_kwargs = yaml.load(f)[param_parent_key]
train(
train_dir=train_dir,
train_extra_dir=train_extra_dir,
test_dir=test_dir,
test_extra_dir=test_extra_dir,
sigmoid_head=sigmoid_head,
out_model_path=out_model_path,
out_encoder_path=out_encoder_path,
out_metrics_path=out_metrics_path,
out_summaries_path=out_summaries_path,
**train_kwargs)
def test_fastai_model_file(tmp_dir, data_loader):
dvclive.init("dvc_logs")
learn = tabular_learner(data_loader, metrics=accuracy)
learn.model_dir = os.path.abspath("./")
learn.fit_one_cycle(2, cbs=[DvcLiveCallback("model")])
assert (tmp_dir / "model.pth").is_file()
def test_init_overrides_env(tmp_dir, summary, monkeypatch):
monkeypatch.setenv(env.DVCLIVE_PATH, "FOO")
monkeypatch.setenv(env.DVCLIVE_SUMMARY, str(int(not summary)))
dvclive.init("logs", summary=summary)
assert dvclive._metric_logger._path == "logs"
assert dvclive._metric_logger._summary == summary
def test_get_step_control_flow(tmp_dir):
dvclive.init("logs")
while dvclive.get_step() < 10:
dvclive.log("i", dvclive.get_step())
dvclive.next_step()
steps, values = read_history("logs", "i")
assert steps == list(range(10))
assert values == [float(x) for x in range(10)]
def test_infer_next_step(tmp_dir, mocker):
dvclive.init("logs")
m = mocker.spy(dvclive.metrics.MetricLogger, "next_step")
dvclive.log("m1", 1.0)
dvclive.log("m1", 2.0)
dvclive.log("m1", 3.0)
assert read_history("logs", "m1") == ([0, 1, 2], [1.0, 2.0, 3.0])
assert m.call_count == 2
def test_custom_steps(tmp_dir):
dvclive.init("logs")
steps = [0, 62, 1000]
metrics = [0.9, 0.8, 0.7]
for step, metric in zip(steps, metrics):
dvclive.log("m", metric, step=step)
assert read_history("logs", "m") == (steps, metrics)
def test_logging(tmp_dir, summary):
dvclive.init("logs", summary=summary)
dvclive.log("m1", 1)
assert (tmp_dir / "logs").is_dir()
assert (tmp_dir / "logs" / "m1.tsv").is_file()
assert not (tmp_dir / "logs.json").is_file()
dvclive.next_step()
assert (tmp_dir / "logs.json").is_file() == summary
def test_html(tmp_dir, dvc_repo, html, signal_exists):
if dvc_repo:
from dvc.repo import Repo
Repo.init(no_scm=True)
dvclive.init("logs", html=html)
dvclive.log("m1", 1)
dvclive.next_step()
assert (tmp_dir / ".dvc" / "tmp" / SIGNAL_FILE).is_file() == signal_exists
def test_xgb_model_file(tmp_dir, train_params, iris_data):
dvclive.init("logs")
model = xgb.train(
train_params,
iris_data,
callbacks=[DvcLiveCallback("eval_data", model_file="model_xgb.json")],
num_boost_round=5,
evals=[(iris_data, "eval_data")],
)
preds = model.predict(iris_data)
model2 = xgb.Booster(model_file="model_xgb.json")
preds2 = model2.predict(iris_data)
assert np.sum(np.abs(preds2 - preds)) == 0
def test_xgb_integration(tmp_dir, train_params, iris_data):
dvclive.init("logs")
xgb.train(
train_params,
iris_data,
callbacks=[DvcLiveCallback("eval_data")],
num_boost_round=5,
evals=[(iris_data, "eval_data")],
)
assert os.path.exists("logs")
logs, _ = read_logs("logs")
assert len(logs) == 1
assert len(first(logs.values())) == 5
def test_fastai_callback(tmp_dir, data_loader):
dvclive.init("dvc_logs")
learn = tabular_learner(data_loader, metrics=accuracy)
learn.model_dir = os.path.abspath("./")
learn.fit_one_cycle(2, cbs=[DvcLiveCallback("model")])
assert os.path.exists("dvc_logs")
train_path = tmp_dir / "dvc_logs/train"
valid_path = tmp_dir / "dvc_logs/valid"
assert train_path.is_dir()
assert valid_path.is_dir()
assert (tmp_dir / "dvc_logs/accuracy.tsv").exists()
def test_keras_callback(tmp_dir, xor_model, capture_wrap):
model, x, y = xor_model()
dvclive.init("logs")
model.fit(
x,
y,
epochs=1,
batch_size=1,
callbacks=[DvcLiveCallback()],
)
assert os.path.exists("logs")
logs, _ = read_logs("logs")
assert "accuracy" in logs
def test_mmcv_model_file(tmp_dir, mocker):
work_dir = tmp_dir / "work_dir"
runner = _build_demo_runner(str(work_dir))
hook = DVCLiveLoggerHook(model_file=tmp_dir / "model.pth")
runner.register_hook(hook, priority="VERY_LOW")
save_checkpoint = mocker.spy(runner, "save_checkpoint")
loader = torch.utils.data.DataLoader(torch.ones((5, 2)))
dvclive.init("logs")
runner.run([loader, loader], [("train", 1), ("val", 1)])
assert save_checkpoint.call_count == 1
assert (tmp_dir / "model.pth").is_file()
def test_lgbm_model_file(tmp_dir, model_params, iris_data):
dvclive.init("logs")
model = lgbm.LGBMClassifier()
model.set_params(**model_params)
model.fit(
iris_data[0][0],
iris_data[0][1],
eval_set=(iris_data[1][0], iris_data[1][1]),
eval_metric=["multi_logloss"],
callbacks=[DvcLiveCallback("lgbm_model")],
)
preds = model.predict(iris_data[1][0])
model2 = lgbm.Booster(model_file="lgbm_model")
preds2 = model2.predict(iris_data[1][0])
preds2 = np.argmax(preds2, axis=1)
assert np.sum(np.abs(preds2 - preds)) == 0
def test_nested_logging(tmp_dir):
dvclive.init("logs", summary=True)
dvclive.log("train/m1", 1)
dvclive.log("val/val_1/m1", 1)
assert (tmp_dir / "logs").is_dir()
assert (tmp_dir / "logs" / "train").is_dir()
assert (tmp_dir / "logs" / "val" / "val_1").is_dir()
assert (tmp_dir / "logs" / "train" / "m1.tsv").is_file()
assert (tmp_dir / "logs" / "val" / "val_1" / "m1.tsv").is_file()
dvclive.next_step()
_, summary = read_logs("logs")
assert summary["train"]["m1"] == 1
assert summary["val"]["val_1"]["m1"] == 1
def test_get_step_custom_steps(tmp_dir):
dvclive.init("logs")
steps = [0, 62, 1000]
metrics = [0.9, 0.8, 0.7]
for step, metric in zip(steps, metrics):
dvclive.log("x", metric, step=step)
assert dvclive.get_step() == step
dvclive.log("y", metric, step=step)
assert dvclive.get_step() == step
dvclive.log("z", metric)
assert dvclive.get_step() == step
for metric in ["x", "y", "z"]:
assert read_history("logs", "x") == (steps, metrics)
def test_lgbm_integration(tmp_dir, model_params, iris_data):
dvclive.init("logs")
model = lgbm.LGBMClassifier()
model.set_params(**model_params)
model.fit(
iris_data[0][0],
iris_data[0][1],
eval_set=(iris_data[1][0], iris_data[1][1]),
eval_metric=["multi_logloss"],
callbacks=[DvcLiveCallback()],
)
assert os.path.exists("logs")
logs, _ = read_logs("logs")
assert len(logs) == 1
assert len(first(logs.values())) == 5
def test_huggingface_model_file(tmp_dir, model, args, data, tokenizer):
dvclive.init("logs")
model_path = tmp_dir / "model_hf"
trainer = Trainer(
model,
args,
train_dataset=data[0],
eval_dataset=data[1],
tokenizer=tokenizer,
compute_metrics=compute_metrics,
)
trainer.add_callback(DvcLiveCallback(model_file=model_path))
trainer.train()
assert model_path.is_dir()
assert (model_path / "pytorch_model.bin").exists()
assert (model_path / "config.json").exists()
def test_continue(tmp_dir, resume, steps, metrics):
dvclive.init("logs")
for metric in [0.9, 0.8]:
dvclive.log("metric", metric)
dvclive.next_step()
assert read_history("logs", "metric") == ([0, 1], [0.9, 0.8])
assert read_latest("logs", "metric") == (1, 0.8)
dvclive.init("logs", resume=resume)
for new_metric in [0.7, 0.6]:
dvclive.log("metric", new_metric)
dvclive.next_step()
assert read_history("logs", "metric") == (steps, metrics)
assert read_latest("logs", "metric") == (last(steps), last(metrics))
def test_cleanup(tmp_dir, summary, html):
dvclive.init("logs", summary=summary)
dvclive.log("m1", 1)
dvclive.next_step()
if html:
(tmp_dir / "logs.html").touch()
(tmp_dir / "logs" / "some_user_file.txt").touch()
assert (tmp_dir / "logs" / "m1.tsv").is_file()
assert (tmp_dir / "logs.json").is_file() == summary
assert (tmp_dir / "logs.html").is_file() == html
dvclive.init("logs")
assert (tmp_dir / "logs" / "some_user_file.txt").is_file()
assert not (tmp_dir / "logs" / "m1.tsv").is_file()
assert not (tmp_dir / "logs.json").is_file()
assert not (tmp_dir / "logs.html").is_file()
def test_keras_model_file(tmp_dir, xor_model, mocker, save_weights_only,
capture_wrap):
model, x, y = xor_model()
save = mocker.spy(model, "save")
save_weights = mocker.spy(model, "save_weights")
dvclive.init("logs")
model.fit(
x,
y,
epochs=1,
batch_size=1,
callbacks=[
DvcLiveCallback(model_file="model.h5",
save_weights_only=save_weights_only)
],
)
assert save.call_count != save_weights_only
assert save_weights.call_count == save_weights_only
def test_mmcv_hook(tmp_dir, mocker):
work_dir = tmp_dir / "work_dir"
runner = _build_demo_runner(str(work_dir))
hook = DVCLiveLoggerHook()
runner.register_hook(hook, priority="VERY_LOW")
next_step = mocker.spy(dvclive.metrics.MetricLogger, "next_step")
log = mocker.spy(dvclive.metrics.MetricLogger, "log")
loader = torch.utils.data.DataLoader(torch.ones((5, 2)))
dvclive.init("logs")
runner.run([loader, loader], [("train", 1), ("val", 1)])
assert next_step.call_count == 5
assert log.call_count == 12
logs, _ = read_logs("logs")
assert "learning_rate" in logs
assert "momentum" in logs
def test_huggingface_integration(tmp_dir, model, args, data, tokenizer):
dvclive.init("logs")
trainer = Trainer(
model,
args,
train_dataset=data[0],
eval_dataset=data[1],
tokenizer=tokenizer,
compute_metrics=compute_metrics,
)
trainer.add_callback(DvcLiveCallback())
trainer.train()
assert os.path.exists("logs")
logs, _ = read_logs("logs")
assert len(logs) == 10
assert "eval_matthews_correlation" in logs
assert "eval_loss" in logs
assert len(logs["epoch"]) == 3
assert len(logs["eval_loss"]) == 2
def main(
train_dir: str,
train_extra_dir: str,
test_dir: str,
test_extra_dir: str,
param_path: str,
param_parent_key: str,
out_model_path: str,
out_encoder_path: str,
out_metrics_path: str,
out_summaries_path: str,
loss: str,
glob_pattern: str,
nway_disabled: bool,
label_func: str,
):
mlflow.tensorflow.autolog(every_n_iter=1)
dvclive.init(out_metrics_path, summary=True, html=True)
with open(param_path, "r") as f:
train_kwargs = yaml.safe_load(f)[param_parent_key]
train(train_dir=train_dir,
train_extra_dir=train_extra_dir,
test_dir=test_dir,
test_extra_dir=test_extra_dir,
loss=loss,
out_model_path=out_model_path,
out_encoder_path=out_encoder_path,
out_metrics_path=out_metrics_path,
out_summaries_path=out_summaries_path,
glob_pattern=glob_pattern,
nway_disabled=nway_disabled,
label_func=label_func,
**train_kwargs)
def test_get_step_resume(tmp_dir):
dvclive.init("logs")
for metric in [0.9, 0.8]:
dvclive.log("metric", metric)
dvclive.next_step()
assert dvclive.get_step() == 2
dvclive.init("logs", resume=True)
assert dvclive.get_step() == 2
dvclive.init("logs", resume=False)
assert dvclive.get_step() == 0
def test_fail_on_conflict(tmp_dir, monkeypatch):
dvclive.init("some_dir")
monkeypatch.setenv(env.DVCLIVE_PATH, "logs")
with pytest.raises(ConfigMismatchError):
dvclive.log("m", 0.1)
def test_create_logs_dir(tmp_dir, path):
dvclive.init(path)
assert (tmp_dir / path).is_dir()
def train(
train_dir,
test_dir,
encoder_model_path,
out_model_path,
out_metric_path,
out_label_encoder_path,
use_imagenet,
is_frozen,
*,
batch_size,
epochs,
verbose,
dropout_rate,
**_ # Ignore other kwargs
):
ds, label_count, label_encoder = setup_ds(train_dir,
batch_size,
decode=random_read_decode)
joblib.dump(label_encoder, out_label_encoder_path)
val_ds, _, _ = setup_ds(test_dir,
batch_size,
label_encoder,
decode=read_decode,
reshuffle=False)
val_ds = val_ds.cache().prefetch(tf.data.AUTOTUNE)
# Setup models
if use_imagenet:
model = build_imagenet_model(freeze=is_frozen)
else:
model = tf.keras.models.load_model(encoder_model_path)
if is_frozen:
for layer in model.layers:
layer.trainable = False
dvclive.init(out_metric_path)
mlflow.tensorflow.autolog(every_n_iter=1)
mlflow.log_artifact(out_label_encoder_path)
head = softmax_model(model.output_shape[1:],
label_count,
dense_nodes=[256],
dropout_rate=dropout_rate)
classifier = tf.keras.Model(inputs=model.inputs,
outputs=head(model.outputs))
classifier.compile(optimizer='adam',
loss=tf.keras.losses.CategoricalCrossentropy(),
metrics=[
'acc',
tf.keras.metrics.Precision(name='precision'),
tf.keras.metrics.Recall(name='recall'),
tf.keras.metrics.AUC(name='auc')
])
classifier.fit(ds,
validation_data=val_ds,
validation_freq=1,
epochs=epochs,
verbose=verbose,
callbacks=[
ReduceLROnPlateau(monitor='loss', patience=10),
MetricsCallback(),
EarlyStopping(monitor='val_loss',
patience=40,
verbose=1,
restore_best_weights=True),
wandb.keras.WandbCallback(),
])
classifier.save(out_model_path, save_format='tf')
mlflow.log_artifact(out_model_path)
