def main( train_dir: str, train_extra_dir: str, test_dir: str, test_extra_dir: str, param_path: str, param_parent_key: str, out_model_path: str, out_encoder_path: str, out_metrics_path: str, out_summaries_path: str, sigmoid_head: bool, ): if sigmoid_head: mlflow.set_experiment("siamese-sigmoid") else: mlflow.set_experiment("siamese-distance") mlflow.tensorflow.autolog(every_n_iter=1) dvclive.init(out_metrics_path, summary=True, html=True) with open(param_path, "r") as f: train_kwargs = yaml.load(f)[param_parent_key] train( train_dir=train_dir, train_extra_dir=train_extra_dir, test_dir=test_dir, test_extra_dir=test_extra_dir, sigmoid_head=sigmoid_head, out_model_path=out_model_path, out_encoder_path=out_encoder_path, out_metrics_path=out_metrics_path, out_summaries_path=out_summaries_path, **train_kwargs)
def test_fastai_model_file(tmp_dir, data_loader): dvclive.init("dvc_logs") learn = tabular_learner(data_loader, metrics=accuracy) learn.model_dir = os.path.abspath("./") learn.fit_one_cycle(2, cbs=[DvcLiveCallback("model")]) assert (tmp_dir / "model.pth").is_file()
def test_init_overrides_env(tmp_dir, summary, monkeypatch): monkeypatch.setenv(env.DVCLIVE_PATH, "FOO") monkeypatch.setenv(env.DVCLIVE_SUMMARY, str(int(not summary))) dvclive.init("logs", summary=summary) assert dvclive._metric_logger._path == "logs" assert dvclive._metric_logger._summary == summary
def test_get_step_control_flow(tmp_dir): dvclive.init("logs") while dvclive.get_step() < 10: dvclive.log("i", dvclive.get_step()) dvclive.next_step() steps, values = read_history("logs", "i") assert steps == list(range(10)) assert values == [float(x) for x in range(10)]
def test_infer_next_step(tmp_dir, mocker): dvclive.init("logs") m = mocker.spy(dvclive.metrics.MetricLogger, "next_step") dvclive.log("m1", 1.0) dvclive.log("m1", 2.0) dvclive.log("m1", 3.0) assert read_history("logs", "m1") == ([0, 1, 2], [1.0, 2.0, 3.0]) assert m.call_count == 2
def test_custom_steps(tmp_dir): dvclive.init("logs") steps = [0, 62, 1000] metrics = [0.9, 0.8, 0.7] for step, metric in zip(steps, metrics): dvclive.log("m", metric, step=step) assert read_history("logs", "m") == (steps, metrics)
def test_logging(tmp_dir, summary): dvclive.init("logs", summary=summary) dvclive.log("m1", 1) assert (tmp_dir / "logs").is_dir() assert (tmp_dir / "logs" / "m1.tsv").is_file() assert not (tmp_dir / "logs.json").is_file() dvclive.next_step() assert (tmp_dir / "logs.json").is_file() == summary
def test_html(tmp_dir, dvc_repo, html, signal_exists): if dvc_repo: from dvc.repo import Repo Repo.init(no_scm=True) dvclive.init("logs", html=html) dvclive.log("m1", 1) dvclive.next_step() assert (tmp_dir / ".dvc" / "tmp" / SIGNAL_FILE).is_file() == signal_exists
def test_xgb_model_file(tmp_dir, train_params, iris_data): dvclive.init("logs") model = xgb.train( train_params, iris_data, callbacks=[DvcLiveCallback("eval_data", model_file="model_xgb.json")], num_boost_round=5, evals=[(iris_data, "eval_data")], ) preds = model.predict(iris_data) model2 = xgb.Booster(model_file="model_xgb.json") preds2 = model2.predict(iris_data) assert np.sum(np.abs(preds2 - preds)) == 0
def test_xgb_integration(tmp_dir, train_params, iris_data): dvclive.init("logs") xgb.train( train_params, iris_data, callbacks=[DvcLiveCallback("eval_data")], num_boost_round=5, evals=[(iris_data, "eval_data")], ) assert os.path.exists("logs") logs, _ = read_logs("logs") assert len(logs) == 1 assert len(first(logs.values())) == 5
def test_fastai_callback(tmp_dir, data_loader): dvclive.init("dvc_logs") learn = tabular_learner(data_loader, metrics=accuracy) learn.model_dir = os.path.abspath("./") learn.fit_one_cycle(2, cbs=[DvcLiveCallback("model")]) assert os.path.exists("dvc_logs") train_path = tmp_dir / "dvc_logs/train" valid_path = tmp_dir / "dvc_logs/valid" assert train_path.is_dir() assert valid_path.is_dir() assert (tmp_dir / "dvc_logs/accuracy.tsv").exists()
def test_keras_callback(tmp_dir, xor_model, capture_wrap): model, x, y = xor_model() dvclive.init("logs") model.fit( x, y, epochs=1, batch_size=1, callbacks=[DvcLiveCallback()], ) assert os.path.exists("logs") logs, _ = read_logs("logs") assert "accuracy" in logs
def test_mmcv_model_file(tmp_dir, mocker): work_dir = tmp_dir / "work_dir" runner = _build_demo_runner(str(work_dir)) hook = DVCLiveLoggerHook(model_file=tmp_dir / "model.pth") runner.register_hook(hook, priority="VERY_LOW") save_checkpoint = mocker.spy(runner, "save_checkpoint") loader = torch.utils.data.DataLoader(torch.ones((5, 2))) dvclive.init("logs") runner.run([loader, loader], [("train", 1), ("val", 1)]) assert save_checkpoint.call_count == 1 assert (tmp_dir / "model.pth").is_file()
def test_lgbm_model_file(tmp_dir, model_params, iris_data): dvclive.init("logs") model = lgbm.LGBMClassifier() model.set_params(**model_params) model.fit( iris_data[0][0], iris_data[0][1], eval_set=(iris_data[1][0], iris_data[1][1]), eval_metric=["multi_logloss"], callbacks=[DvcLiveCallback("lgbm_model")], ) preds = model.predict(iris_data[1][0]) model2 = lgbm.Booster(model_file="lgbm_model") preds2 = model2.predict(iris_data[1][0]) preds2 = np.argmax(preds2, axis=1) assert np.sum(np.abs(preds2 - preds)) == 0
def test_nested_logging(tmp_dir): dvclive.init("logs", summary=True) dvclive.log("train/m1", 1) dvclive.log("val/val_1/m1", 1) assert (tmp_dir / "logs").is_dir() assert (tmp_dir / "logs" / "train").is_dir() assert (tmp_dir / "logs" / "val" / "val_1").is_dir() assert (tmp_dir / "logs" / "train" / "m1.tsv").is_file() assert (tmp_dir / "logs" / "val" / "val_1" / "m1.tsv").is_file() dvclive.next_step() _, summary = read_logs("logs") assert summary["train"]["m1"] == 1 assert summary["val"]["val_1"]["m1"] == 1
def test_get_step_custom_steps(tmp_dir): dvclive.init("logs") steps = [0, 62, 1000] metrics = [0.9, 0.8, 0.7] for step, metric in zip(steps, metrics): dvclive.log("x", metric, step=step) assert dvclive.get_step() == step dvclive.log("y", metric, step=step) assert dvclive.get_step() == step dvclive.log("z", metric) assert dvclive.get_step() == step for metric in ["x", "y", "z"]: assert read_history("logs", "x") == (steps, metrics)
def test_lgbm_integration(tmp_dir, model_params, iris_data): dvclive.init("logs") model = lgbm.LGBMClassifier() model.set_params(**model_params) model.fit( iris_data[0][0], iris_data[0][1], eval_set=(iris_data[1][0], iris_data[1][1]), eval_metric=["multi_logloss"], callbacks=[DvcLiveCallback()], ) assert os.path.exists("logs") logs, _ = read_logs("logs") assert len(logs) == 1 assert len(first(logs.values())) == 5
def test_huggingface_model_file(tmp_dir, model, args, data, tokenizer): dvclive.init("logs") model_path = tmp_dir / "model_hf" trainer = Trainer( model, args, train_dataset=data[0], eval_dataset=data[1], tokenizer=tokenizer, compute_metrics=compute_metrics, ) trainer.add_callback(DvcLiveCallback(model_file=model_path)) trainer.train() assert model_path.is_dir() assert (model_path / "pytorch_model.bin").exists() assert (model_path / "config.json").exists()
def test_continue(tmp_dir, resume, steps, metrics): dvclive.init("logs") for metric in [0.9, 0.8]: dvclive.log("metric", metric) dvclive.next_step() assert read_history("logs", "metric") == ([0, 1], [0.9, 0.8]) assert read_latest("logs", "metric") == (1, 0.8) dvclive.init("logs", resume=resume) for new_metric in [0.7, 0.6]: dvclive.log("metric", new_metric) dvclive.next_step() assert read_history("logs", "metric") == (steps, metrics) assert read_latest("logs", "metric") == (last(steps), last(metrics))
def test_cleanup(tmp_dir, summary, html): dvclive.init("logs", summary=summary) dvclive.log("m1", 1) dvclive.next_step() if html: (tmp_dir / "logs.html").touch() (tmp_dir / "logs" / "some_user_file.txt").touch() assert (tmp_dir / "logs" / "m1.tsv").is_file() assert (tmp_dir / "logs.json").is_file() == summary assert (tmp_dir / "logs.html").is_file() == html dvclive.init("logs") assert (tmp_dir / "logs" / "some_user_file.txt").is_file() assert not (tmp_dir / "logs" / "m1.tsv").is_file() assert not (tmp_dir / "logs.json").is_file() assert not (tmp_dir / "logs.html").is_file()
def test_keras_model_file(tmp_dir, xor_model, mocker, save_weights_only, capture_wrap): model, x, y = xor_model() save = mocker.spy(model, "save") save_weights = mocker.spy(model, "save_weights") dvclive.init("logs") model.fit( x, y, epochs=1, batch_size=1, callbacks=[ DvcLiveCallback(model_file="model.h5", save_weights_only=save_weights_only) ], ) assert save.call_count != save_weights_only assert save_weights.call_count == save_weights_only
def test_mmcv_hook(tmp_dir, mocker): work_dir = tmp_dir / "work_dir" runner = _build_demo_runner(str(work_dir)) hook = DVCLiveLoggerHook() runner.register_hook(hook, priority="VERY_LOW") next_step = mocker.spy(dvclive.metrics.MetricLogger, "next_step") log = mocker.spy(dvclive.metrics.MetricLogger, "log") loader = torch.utils.data.DataLoader(torch.ones((5, 2))) dvclive.init("logs") runner.run([loader, loader], [("train", 1), ("val", 1)]) assert next_step.call_count == 5 assert log.call_count == 12 logs, _ = read_logs("logs") assert "learning_rate" in logs assert "momentum" in logs
def test_huggingface_integration(tmp_dir, model, args, data, tokenizer): dvclive.init("logs") trainer = Trainer( model, args, train_dataset=data[0], eval_dataset=data[1], tokenizer=tokenizer, compute_metrics=compute_metrics, ) trainer.add_callback(DvcLiveCallback()) trainer.train() assert os.path.exists("logs") logs, _ = read_logs("logs") assert len(logs) == 10 assert "eval_matthews_correlation" in logs assert "eval_loss" in logs assert len(logs["epoch"]) == 3 assert len(logs["eval_loss"]) == 2
def main( train_dir: str, train_extra_dir: str, test_dir: str, test_extra_dir: str, param_path: str, param_parent_key: str, out_model_path: str, out_encoder_path: str, out_metrics_path: str, out_summaries_path: str, loss: str, glob_pattern: str, nway_disabled: bool, label_func: str, ): mlflow.tensorflow.autolog(every_n_iter=1) dvclive.init(out_metrics_path, summary=True, html=True) with open(param_path, "r") as f: train_kwargs = yaml.safe_load(f)[param_parent_key] train(train_dir=train_dir, train_extra_dir=train_extra_dir, test_dir=test_dir, test_extra_dir=test_extra_dir, loss=loss, out_model_path=out_model_path, out_encoder_path=out_encoder_path, out_metrics_path=out_metrics_path, out_summaries_path=out_summaries_path, glob_pattern=glob_pattern, nway_disabled=nway_disabled, label_func=label_func, **train_kwargs)
def test_get_step_resume(tmp_dir): dvclive.init("logs") for metric in [0.9, 0.8]: dvclive.log("metric", metric) dvclive.next_step() assert dvclive.get_step() == 2 dvclive.init("logs", resume=True) assert dvclive.get_step() == 2 dvclive.init("logs", resume=False) assert dvclive.get_step() == 0
def test_fail_on_conflict(tmp_dir, monkeypatch): dvclive.init("some_dir") monkeypatch.setenv(env.DVCLIVE_PATH, "logs") with pytest.raises(ConfigMismatchError): dvclive.log("m", 0.1)
def test_create_logs_dir(tmp_dir, path): dvclive.init(path) assert (tmp_dir / path).is_dir()
def train( train_dir, test_dir, encoder_model_path, out_model_path, out_metric_path, out_label_encoder_path, use_imagenet, is_frozen, *, batch_size, epochs, verbose, dropout_rate, **_ # Ignore other kwargs ): ds, label_count, label_encoder = setup_ds(train_dir, batch_size, decode=random_read_decode) joblib.dump(label_encoder, out_label_encoder_path) val_ds, _, _ = setup_ds(test_dir, batch_size, label_encoder, decode=read_decode, reshuffle=False) val_ds = val_ds.cache().prefetch(tf.data.AUTOTUNE) # Setup models if use_imagenet: model = build_imagenet_model(freeze=is_frozen) else: model = tf.keras.models.load_model(encoder_model_path) if is_frozen: for layer in model.layers: layer.trainable = False dvclive.init(out_metric_path) mlflow.tensorflow.autolog(every_n_iter=1) mlflow.log_artifact(out_label_encoder_path) head = softmax_model(model.output_shape[1:], label_count, dense_nodes=[256], dropout_rate=dropout_rate) classifier = tf.keras.Model(inputs=model.inputs, outputs=head(model.outputs)) classifier.compile(optimizer='adam', loss=tf.keras.losses.CategoricalCrossentropy(), metrics=[ 'acc', tf.keras.metrics.Precision(name='precision'), tf.keras.metrics.Recall(name='recall'), tf.keras.metrics.AUC(name='auc') ]) classifier.fit(ds, validation_data=val_ds, validation_freq=1, epochs=epochs, verbose=verbose, callbacks=[ ReduceLROnPlateau(monitor='loss', patience=10), MetricsCallback(), EarlyStopping(monitor='val_loss', patience=40, verbose=1, restore_best_weights=True), wandb.keras.WandbCallback(), ]) classifier.save(out_model_path, save_format='tf') mlflow.log_artifact(out_model_path)