def test_run_local_conda_env(tracking_uri_mock): # pylint: disable=unused-argument
with open(os.path.join(TEST_PROJECT_DIR, "conda.yaml"), "r") as handle:
conda_env_contents = handle.read()
expected_env_name = "mlflow-%s" % hashlib.sha1(
conda_env_contents.encode("utf-8")).hexdigest()
try:
process.exec_cmd(
cmd=["conda", "env", "remove", "--name", expected_env_name])
except process.ShellCommandException:
logging_utils.eprint(
"Unable to remove conda environment %s. The environment may not have been present, "
"continuing with running the test." % expected_env_name)
invoke_cli_runner(cli.run, [
TEST_PROJECT_DIR, "-e", "check_conda_env", "-P",
"conda_env_name=%s" % expected_env_name
])
def test_mlflow_run_example(directory, params, tmpdir):
example_dir = os.path.join(EXAMPLES_DIR, directory)
tmp_example_dir = os.path.join(tmpdir.strpath, directory)
shutil.copytree(example_dir, tmp_example_dir)
conda_yml_path = find_conda_yaml(tmp_example_dir)
replace_mlflow_with_dev_version(conda_yml_path)
# remove old conda environments to free disk space
envs = list(filter(is_mlflow_conda_env, get_conda_envs()))
current_env_name = "mlflow-" + hash_conda_env(conda_yml_path)
envs_to_remove = list(filter(lambda e: e != current_env_name, envs))
for env in envs_to_remove:
remove_conda_env(env)
cli_run_list = [tmp_example_dir] + params
invoke_cli_runner(cli.run, cli_run_list)
def test_run_local_params(name):
excitement_arg = 2
invoke_cli_runner(
cli.run,
[
TEST_PROJECT_DIR,
"-e",
"greeter",
"-P",
"greeting=hi",
"-P",
"name=%s" % name,
"-P",
"excitement=%s" % excitement_arg,
],
)
def test_run_git():
with TempDir() as tmp:
with update_temp_env(
{mlflow.tracking._TRACKING_URI_ENV_VAR: tmp.path()}):
res = invoke_cli_runner(
cli.run, [GIT_PROJECT_URI, "--no-conda", "-P", "alpha=0.5"])
assert "python train.py 0.5 0.1" in res.output
def test_run_databricks_cluster_spec(tmpdir):
cluster_spec = {
"spark_version": "5.0.x-scala2.11",
"num_workers": 2,
"node_type_id": "i3.xlarge",
}
cluster_spec_path = str(tmpdir.join("cluster-spec.json"))
with open(cluster_spec_path, "w") as handle:
json.dump(cluster_spec, handle)
with mock.patch("mlflow.projects._run") as run_mock:
for cluster_spec_arg in [json.dumps(cluster_spec), cluster_spec_path]:
invoke_cli_runner(
cli.run,
[
TEST_PROJECT_DIR,
"-b",
"databricks",
"--backend-config",
cluster_spec_arg,
"-e",
"greeter",
"-P",
"name=hi",
],
env={"MLFLOW_TRACKING_URI": "databricks://profile"},
)
assert run_mock.call_count == 1
_, run_kwargs = run_mock.call_args_list[0]
assert run_kwargs["backend_config"] == cluster_spec
run_mock.reset_mock()
res = CliRunner().invoke(
cli.run,
[
TEST_PROJECT_DIR,
"-m",
"databricks",
"--cluster-spec",
json.dumps(cluster_spec) + "JUNK",
"-e",
"greeter",
"-P",
"name=hi",
],
env={"MLFLOW_TRACKING_URI": "databricks://profile"},
)
assert res.exit_code != 0
def test_rename_experiment_cli(mlflow_client, cli_env):
bad_experiment_name = "CLIBadName"
good_experiment_name = "CLIGoodName"
invoke_cli_runner(mlflow.experiments.commands,
['create', bad_experiment_name],
env=cli_env)
experiment_id = mlflow_client.get_experiment_by_name(
bad_experiment_name).experiment_id
assert mlflow_client.get_experiment(
experiment_id).name == bad_experiment_name
invoke_cli_runner(
mlflow.experiments.commands,
['rename', str(experiment_id), good_experiment_name],
env=cli_env)
assert mlflow_client.get_experiment(
experiment_id).name == good_experiment_name
def test_run_local():
with TempDir() as tmp:
with update_temp_env(
{mlflow.tracking._TRACKING_URI_ENV_VAR: tmp.path()}):
excitement_arg = 2
res = invoke_cli_runner(cli.run, [
TEST_PROJECT_DIR, "-e", "greeter", "-P", "greeting=hi", "-P",
"name=friend", "-P",
"excitement=%s" % excitement_arg
])
_assert_succeeded(res.output)
def test_rename_experiment_cli(mlflow_client, cli_env):
bad_experiment_name = "CLIBadName"
good_experiment_name = "CLIGoodName"
invoke_cli_runner(mlflow.experiments.commands,
["create", "-n", bad_experiment_name],
env=cli_env)
experiment_id = mlflow_client.get_experiment_by_name(
bad_experiment_name).experiment_id
assert mlflow_client.get_experiment(
experiment_id).name == bad_experiment_name
invoke_cli_runner(
mlflow.experiments.commands,
[
"rename", "--experiment-id",
str(experiment_id), "--new-name", good_experiment_name
],
env=cli_env,
)
assert mlflow_client.get_experiment(
experiment_id).name == good_experiment_name
def test_run_local_conda_env():
with open(os.path.join(TEST_PROJECT_DIR, "conda.yaml"), "r") as handle:
conda_env_contents = handle.read()
expected_env_name = "mlflow-%s" % hashlib.sha1(
conda_env_contents.encode("utf-8")).hexdigest()
try:
process._exec_cmd(
cmd=["conda", "env", "remove", "--name", expected_env_name])
except process.ShellCommandException:
_logger.error(
"Unable to remove conda environment %s. The environment may not have been present, "
"continuing with running the test.",
expected_env_name,
)
invoke_cli_runner(
cli.run,
[
TEST_PROJECT_DIR, "-e", "check_conda_env", "-P",
"conda_env_name=%s" % expected_env_name
],
)
def test_run_local_experiment_specification(experiment_name,
tracking_uri_mock): # pylint: disable=unused-argument
invoke_cli_runner(
cli.run,
[
TEST_PROJECT_DIR,
"-e", "greeter",
"-P", "name=test",
"--experiment-name", experiment_name,
])
client = MlflowClient()
experiment_id = client.get_experiment_by_name(experiment_name).experiment_id
invoke_cli_runner(
cli.run,
[
TEST_PROJECT_DIR,
"-e", "greeter",
"-P", "name=test",
"--experiment-id", experiment_id,
])
def test_sqlalchemy_store_detects_schema_mismatch(tmpdir, db_url): # pylint: disable=unused-argument
def _assert_invalid_schema(engine):
with pytest.raises(MlflowException,
match="Detected out-of-date database schema."):
_verify_schema(engine)
# Initialize an empty database & verify that we detect a schema mismatch
engine = sqlalchemy.create_engine(db_url)
_assert_invalid_schema(engine)
# Create legacy tables, verify schema is still out of date
InitialBase.metadata.create_all(engine)
_assert_invalid_schema(engine)
# Run each migration. Until the last one, schema should be out of date
config = _get_alembic_config(db_url)
script = ScriptDirectory.from_config(config)
revisions = list(script.walk_revisions())
revisions.reverse()
for rev in revisions[:-1]:
command.upgrade(config, rev.revision)
_assert_invalid_schema(engine)
# Run migrations, schema verification should now pass
invoke_cli_runner(mlflow.db.commands, ["upgrade", db_url])
_verify_schema(engine)
def test_delete_restore_experiment_cli(mlflow_client, cli_env):
experiment_name = "DeleteriousCLI"
invoke_cli_runner(mlflow.experiments.commands, ['create', experiment_name], env=cli_env)
experiment_id = mlflow_client.get_experiment_by_name(experiment_name).experiment_id
assert mlflow_client.get_experiment(experiment_id).lifecycle_stage == 'active'
invoke_cli_runner(mlflow.experiments.commands, ['delete', str(experiment_id)], env=cli_env)
assert mlflow_client.get_experiment(experiment_id).lifecycle_stage == 'deleted'
invoke_cli_runner(mlflow.experiments.commands, ['restore', str(experiment_id)], env=cli_env)
assert mlflow_client.get_experiment(experiment_id).lifecycle_stage == 'active'
def test_run_local_experiment_specification(experiment_name):
invoke_cli_runner(cli.run, [
TEST_PROJECT_DIR,
"-e",
"greeter",
"-P",
"name=test",
"--experiment-name",
experiment_name,
])
client = MlflowClient()
experiment_id = client.get_experiment_by_name(
experiment_name).experiment_id
invoke_cli_runner(cli.run, [
TEST_PROJECT_DIR,
"-e",
"greeter",
"-P",
"name=test",
"--experiment-id",
experiment_id,
])
def test_delete_restore_experiment_cli(mlflow_client, cli_env):
experiment_name = "DeleteriousCLI"
invoke_cli_runner(
mlflow.experiments.commands, ["create", "--experiment-name", experiment_name], env=cli_env
)
experiment_id = mlflow_client.get_experiment_by_name(experiment_name).experiment_id
assert mlflow_client.get_experiment(experiment_id).lifecycle_stage == "active"
invoke_cli_runner(
mlflow.experiments.commands, ["delete", "-x", str(experiment_id)], env=cli_env
)
assert mlflow_client.get_experiment(experiment_id).lifecycle_stage == "deleted"
invoke_cli_runner(
mlflow.experiments.commands, ["restore", "-x", str(experiment_id)], env=cli_env
)
assert mlflow_client.get_experiment(experiment_id).lifecycle_stage == "active"
def test_run_git():
with TempDir() as tmp:
with update_temp_env({mlflow.tracking._TRACKING_URI_ENV_VAR: tmp.path()}):
invoke_cli_runner(cli.run, [GIT_PROJECT_URI, "--no-conda", "-P", "alpha=0.5"])
def test_run_local_with_docker_args(docker_example_base_image): # pylint: disable=unused-argument
# Verify that Docker project execution is successful when Docker flag and string
# commandline arguments are supplied (`tty` and `name`, respectively)
invoke_cli_runner(
cli.run,
[TEST_DOCKER_PROJECT_DIR, "-A", "tty", "-A", "name=mycontainer"])
def test_run_git_https(tracking_uri_mock): # pylint: disable=unused-argument
# Invoke command twice to ensure we set Git state in an isolated manner (e.g. don't attempt to
# create a git repo in the same directory twice, etc)
assert GIT_PROJECT_URI.startswith("https")
invoke_cli_runner(cli.run, [GIT_PROJECT_URI, "-P", "alpha=0.5"])
invoke_cli_runner(cli.run, [GIT_PROJECT_URI, "-P", "alpha=0.5"])
def test_run_local(tracking_uri_mock): # pylint: disable=unused-argument
excitement_arg = 2
name = "friend"
invoke_cli_runner(cli.run, [TEST_PROJECT_DIR, "-e", "greeter", "-P",
"greeting=hi", "-P", "name=%s" % name,
"-P", "excitement=%s" % excitement_arg])
def test_mlflow_run_example(directory, params):
cli_run_list = [os.path.join(EXAMPLES_DIR, directory)] + params
invoke_cli_runner(cli.run, cli_run_list)
def test_run_local_no_spec(tracking_uri_mock): # pylint: disable=unused-argument
# Run an example project that doesn't contain an MLproject file
expected_env_name = "mlflow-%s" % hashlib.sha1("".encode("utf-8")).hexdigest()
invoke_cli_runner(cli.run, [TEST_NO_SPEC_PROJECT_DIR, "-e", "check_conda_env.py", "-P",
"conda-env-name=%s" % expected_env_name])