def test_get_experiment_id_with_active_experiment_returns_active_experiment_id(
):
# Create a new experiment and set that as active experiment
with TempDir(chdr=True):
name = "Random experiment %d" % random.randint(1, 1e6)
exp_id = kiwi.create_experiment(name)
assert exp_id is not None
kiwi.set_experiment(name)
assert _get_experiment_id() == exp_id
def test_set_experiment_with_deleted_experiment_name():
name = "dead_exp"
kiwi.set_experiment(name)
with start_run() as run:
exp_id = run.info.experiment_id
tracking.MlflowClient().delete_experiment(exp_id)
with pytest.raises(MlflowException):
kiwi.set_experiment(name)
def test_start_run_exp_id_0():
kiwi.set_experiment("some-experiment")
# Create a run and verify that the current active experiment is the one we just set
with kiwi.start_run() as active_run:
exp_id = active_run.info.experiment_id
assert exp_id != FileStore.DEFAULT_EXPERIMENT_ID
assert MlflowClient().get_experiment(exp_id).name == "some-experiment"
# Set experiment ID to 0 when creating a run, verify that the specified experiment ID is honored
with kiwi.start_run(experiment_id=0) as active_run:
assert active_run.info.experiment_id == FileStore.DEFAULT_EXPERIMENT_ID
def test_set_experiment_with_zero_id(reset_mock):
reset_mock(
MlflowClient, "get_experiment_by_name",
mock.Mock(return_value=attrdict.AttrDict(
experiment_id=0, lifecycle_stage=LifecycleStage.ACTIVE)))
reset_mock(MlflowClient, "create_experiment", mock.Mock())
kiwi.set_experiment("my_exp")
MlflowClient.get_experiment_by_name.assert_called_once()
MlflowClient.create_experiment.assert_not_called()
def test_get_experiment_id_in_databricks_with_active_experiment_returns_active_experiment_id(
):
with TempDir(chdr=True):
exp_name = "random experiment %d" % random.randint(1, 1e6)
exp_id = kiwi.create_experiment(exp_name)
kiwi.set_experiment(exp_name)
notebook_id = str(int(exp_id) + 73)
with mock.patch("mlflow.tracking.fluent.is_in_databricks_notebook") as notebook_detection_mock,\
mock.patch("mlflow.tracking.fluent.get_notebook_id") as notebook_id_mock:
notebook_detection_mock.return_value = True
notebook_id_mock.return_value = notebook_id
assert _get_experiment_id() != notebook_id
assert _get_experiment_id() == exp_id
def test_get_artifact_uri_appends_to_uri_path_component_correctly(
artifact_location, expected_uri_format):
client = MlflowClient()
client.create_experiment("get-artifact-uri-test",
artifact_location=artifact_location)
kiwi.set_experiment("get-artifact-uri-test")
with kiwi.start_run():
run_id = kiwi.active_run().info.run_id
for artifact_path in [
"path/to/artifact", "/artifact/path", "arty.txt"
]:
artifact_uri = kiwi.get_artifact_uri(artifact_path)
assert artifact_uri == tracking.artifact_utils.get_artifact_uri(
run_id, artifact_path)
assert artifact_uri == expected_uri_format.format(
run_id=run_id, path=artifact_path.lstrip("/"))
def test_set_experiment():
with pytest.raises(TypeError):
kiwi.set_experiment() # pylint: disable=no-value-for-parameter
with pytest.raises(Exception):
kiwi.set_experiment(None)
with pytest.raises(Exception):
kiwi.set_experiment("")
name = "random_exp"
exp_id = kiwi.create_experiment(name)
kiwi.set_experiment(name)
with start_run() as run:
assert run.info.experiment_id == exp_id
another_name = "another_experiment"
kiwi.set_experiment(another_name)
exp_id2 = kiwi.tracking.MlflowClient().get_experiment_by_name(another_name)
with start_run() as another_run:
assert another_run.info.experiment_id == exp_id2.experiment_id
# Loop
loop_1_run_id = None
loop_2_run_id = None
with kiwi.start_run(run_name='loop-1') as run_1:
with kiwi.start_run(run_name='loop-2', nested=True) as run_2:
loop_1_run_id = run_1.info.run_id
loop_2_run_id = run_2.info.run_id
client.set_tag(loop_1_run_id, 'mlflow.parentRunId', loop_2_run_id)
# Lot's of children
with kiwi.start_run(run_name='parent-with-lots-of-children'):
for i in range(100):
with kiwi.start_run(run_name='child-{}'.format(i), nested=True):
pass
kiwi.set_experiment("my-empty-experiment")
kiwi.set_experiment("runs-but-no-metrics-params")
for i in range(100):
with kiwi.start_run(run_name="empty-run-{}".format(i)):
pass
if args.large:
kiwi.set_experiment("med-size-experiment")
# Experiment with a mix of nested runs & non-nested runs
for i in range(3):
with kiwi.start_run(run_name='parent-with-children-{}'.format(i)):
params = {rand_str(): rand_str() for _ in range(5)}
metrics = {rand_str(): [rand()] for _ in range(5)}
log_params(params)
log_metrics(metrics)
for j in range(10):
with kiwi.start_run(run_name='child-{}'.format(j),
def test_search_runs():
kiwi.set_experiment("exp-for-search")
# Create a run and verify that the current active experiment is the one we just set
logged_runs = {}
with kiwi.start_run() as active_run:
logged_runs["first"] = active_run.info.run_id
kiwi.log_metric("m1", 0.001)
kiwi.log_metric("m2", 0.002)
kiwi.log_metric("m1", 0.002)
kiwi.log_param("p1", "a")
kiwi.set_tag("t1", "first-tag-val")
with kiwi.start_run() as active_run:
logged_runs["second"] = active_run.info.run_id
kiwi.log_metric("m1", 0.008)
kiwi.log_param("p2", "aa")
kiwi.set_tag("t2", "second-tag-val")
def verify_runs(runs, expected_set):
assert set([r.info.run_id for r in runs
]) == set([logged_runs[r] for r in expected_set])
experiment_id = MlflowClient().get_experiment_by_name(
"exp-for-search").experiment_id
# 2 runs in this experiment
assert len(MlflowClient().list_run_infos(experiment_id,
ViewType.ACTIVE_ONLY)) == 2
# 2 runs that have metric "m1" > 0.001
runs = MlflowClient().search_runs([experiment_id], "metrics.m1 > 0.0001")
verify_runs(runs, ["first", "second"])
# 1 run with has metric "m1" > 0.002
runs = MlflowClient().search_runs([experiment_id], "metrics.m1 > 0.002")
verify_runs(runs, ["second"])
# no runs with metric "m1" > 0.1
runs = MlflowClient().search_runs([experiment_id], "metrics.m1 > 0.1")
verify_runs(runs, [])
# 1 run with metric "m2" > 0
runs = MlflowClient().search_runs([experiment_id], "metrics.m2 > 0")
verify_runs(runs, ["first"])
# 1 run each with param "p1" and "p2"
runs = MlflowClient().search_runs([experiment_id], "params.p1 = 'a'",
ViewType.ALL)
verify_runs(runs, ["first"])
runs = MlflowClient().search_runs([experiment_id], "params.p2 != 'a'",
ViewType.ALL)
verify_runs(runs, ["second"])
runs = MlflowClient().search_runs([experiment_id], "params.p2 = 'aa'",
ViewType.ALL)
verify_runs(runs, ["second"])
# 1 run each with tag "t1" and "t2"
runs = MlflowClient().search_runs([experiment_id],
"tags.t1 = 'first-tag-val'",
ViewType.ALL)
verify_runs(runs, ["first"])
runs = MlflowClient().search_runs([experiment_id], "tags.t2 != 'qwerty'",
ViewType.ALL)
verify_runs(runs, ["second"])
runs = MlflowClient().search_runs([experiment_id],
"tags.t2 = 'second-tag-val'",
ViewType.ALL)
verify_runs(runs, ["second"])
# delete "first" run
MlflowClient().delete_run(logged_runs["first"])
runs = MlflowClient().search_runs([experiment_id], "params.p1 = 'a'",
ViewType.ALL)
verify_runs(runs, ["first"])
runs = MlflowClient().search_runs([experiment_id], "params.p1 = 'a'",
ViewType.DELETED_ONLY)
verify_runs(runs, ["first"])
runs = MlflowClient().search_runs([experiment_id], "params.p1 = 'a'",
ViewType.ACTIVE_ONLY)
verify_runs(runs, [])