def test_serialization():
spec = tag_test(base_spec, "test_serialization")
result = run_local(spec)
verify_state(result)
pprint(result.to_dict())
print(result.to_yaml())
pprint(result.to_json())
def test_run_local_handler():
spec = tag_test(base_spec, "test_run_local_handler")
spec.spec.handler = "my_func"
result = run_local(spec,
command=f"{examples_path}/handler.py",
workdir=examples_path)
verify_state(result)
def test_run_local_nb():
spec = tag_test(base_spec, "test_run_local_nb")
spec.spec.handler = "training"
result = run_local(spec,
command=f"{examples_path}/mlrun_jobs.ipynb",
workdir=examples_path)
verify_state(result)
def test_run_local_handler():
spec = tag_test(base_spec, 'test_run_local_handler')
spec.spec.handler = 'my_func'
result = run_local(spec,
command='{}/handler.py'.format(examples_path),
workdir=examples_path)
verify_state(result)
def test_run_local_nb():
spec = tag_test(base_spec, 'test_run_local_handler')
spec.spec.handler = 'training'
result = run_local(spec,
command='{}/mlrun_jobs.ipynb'.format(examples_path),
workdir=examples_path)
verify_state(result)
def test_handler_hyperlist():
run_spec = tag_test(base_spec, "test_handler_hyperlist")
run_spec.spec.param_file = f"{tests_root_directory}/param_file.csv"
result = new_function().run(run_spec, handler=my_func)
print(result)
assert len(result.status.iterations) == 3 + 1, "hyper parameters test failed"
verify_state(result)
def test_handler_project():
spec = tag_test(base_spec, "test_handler_project")
spec.metadata.project = "myproj"
spec.metadata.labels = {"owner": "yaronh"}
result = new_function().run(spec, handler=my_func)
print(result)
assert result.output("accuracy") == 16, "failed to run"
verify_state(result)
def test_handler_project():
spec = tag_test(base_spec, 'test_handler_project')
spec.metadata.project = 'myproj'
spec.metadata.labels = {'owner': 'yaronh'}
result = new_function().run(spec, handler=my_func)
print(result)
assert result.output('accuracy') == 16, 'failed to run'
verify_state(result)
def test_handler_hyperlist():
run_spec = tag_test(base_spec, 'test_handler_hyperlist')
run_spec.spec.param_file = '{}/param_file.csv'.format(here)
result = new_function().run(run_spec, handler=my_func)
print(result)
assert len(
result.status.iterations) == 3 + 1, 'hyper parameters test failed'
verify_state(result)
def test_hyper_random():
grid_params = {"p2": [2, 1, 3], "p3": [10, 20, 30]}
run_spec = tag_test(base_spec, "test_hyper_random")
run_spec.with_hyper_params(grid_params, selector="r1", strategy="random")
run_spec.spec.parameters["MAX_RANDOM_EVALS"] = 5
run = new_function().run(run_spec, handler=hyper_func)
verify_state(run)
assert len(run.status.iterations) == 1 + 5, "wrong number of iterations"
def test_run_local_obj():
spec = tag_test(base_spec, 'test_run_local_handler')
spec.spec.handler = 'training'
nbpath = '{}/mlrun_jobs.ipynb'.format(examples_path)
ymlpath = path.join(out_path, 'nbyaml.yaml')
print('out path:', out_path, ymlpath)
fn = code_to_function(filename=nbpath, kind='job').export(ymlpath)
result = run_local(spec, command=fn, workdir=out_path)
verify_state(result)
def test_run_local_obj():
spec = tag_test(base_spec, "test_run_local_obj")
spec.spec.handler = "training"
nbpath = f"{examples_path}/mlrun_jobs.ipynb"
ymlpath = path.join(out_path, "nbyaml.yaml")
print("out path:", out_path, ymlpath)
fn = code_to_function(filename=nbpath, kind="job").export(ymlpath)
result = run_local(spec, command=fn, workdir=out_path)
verify_state(result)
def test_hyper_custom():
run_spec = tag_test(base_spec, "test_hyper_custom")
run = new_function().run(run_spec, handler=custom_hyper_func)
verify_state(run)
assert len(run.status.iterations) == 1 + 4, "wrong number of iterations"
results = [line[3] for line in run.status.iterations[1:]]
print(results)
assert run.output("best_iteration") == 3, "wrong best iteration"
def test_run_local_from_func():
spec = tag_test(base_spec, "test_run_local_from_func")
spec.spec.handler = "training"
nb_path = f"{examples_path}/mlrun_jobs.ipynb"
nbyml_path = path.join(out_path, "nbyaml.yaml")
print("out path:", out_path, nbyml_path)
fn = code_to_function(filename=nb_path, kind="job").export(nbyml_path)
result = fn.run(spec, workdir=out_path, local=True)
verify_state(result)
def test_vault_end_to_end():
# This requires an MLRun API server to run and work with Vault. This port should
# be configured to allow access to the server.
api_server_port = 57764
_set_vault_mlrun_configuration(api_server_port)
project_name = "abc"
func_name = "vault-function"
aws_key_value = "1234567890"
github_key_value = "proj1Key!!!"
project = new_project(project_name)
# This call will initialize Vault infrastructure and add the given secrets
# It executes on the API server
project.create_vault_secrets({
"aws_key": aws_key_value,
"github_key": github_key_value
})
# This API executes on the client side
project_secrets = project.get_vault_secret_keys()
assert project_secrets == ["aws_key", "github_key"], "secrets not created"
# Create function and set container configuration
function = code_to_function(
name=func_name,
filename="{}/vault_function.py".format(examples_path),
handler="vault_func",
project=project_name,
kind="job",
)
function.spec.image = "saarcoiguazio/mlrun:unstable"
# Create context for the execution
spec = new_task(
project=project_name,
name="vault_test_run",
handler="vault_func",
out_path=out_path,
params={"secrets": ["password", "path", "github_key", "aws_key"]},
)
spec.with_secrets("vault", [])
result = function.run(spec)
verify_state(result)
db = get_run_db().connect()
state, log = db.get_log(result.metadata.uid, project=project_name)
log = str(log)
print(state)
assert (log.find("value: {}".format(aws_key_value)) !=
-1), "secret value not detected in function output"
assert (log.find("value: {}".format(github_key_value)) !=
-1), "secret value not detected in function output"
def test_handler_hyper():
run_spec = tag_test(base_spec, 'test_handler_hyper')
run_spec.with_hyper_params({'p1': [1, 5, 3]}, selector='max.accuracy')
result = new_function().run(run_spec, handler=my_func)
print(result)
assert len(
result.status.iterations) == 3 + 1, 'hyper parameters test failed'
assert (result.status.results['best_iteration'] == 2
), 'failed to select best iteration'
verify_state(result)
def test_handler_hyper():
run_spec = tag_test(base_spec, "test_handler_hyper")
run_spec.with_hyper_params({"p1": [1, 5, 3]}, selector="max.accuracy")
result = new_function().run(run_spec, handler=my_func)
print(result)
assert len(
result.status.iterations) == 3 + 1, "hyper parameters test failed"
assert (result.status.results["best_iteration"] == 2
), "failed to select best iteration"
verify_state(result)
def test_local_no_context():
spec = tag_test(base_spec, 'test_local_no_context')
spec.spec.parameters = {'xyz': '789'}
result = new_function(command='{}/no_ctx.py'.format(here),
mode='noctx').run(spec)
verify_state(result)
db = get_run_db().connect()
state, log = db.get_log(result.metadata.uid)
log = str(log)
print(state)
print(log)
assert log.find(", '--xyz', '789']") != -1, 'params not detected in noctx'
def test_local_no_context():
spec = tag_test(base_spec, "test_local_no_context")
spec.spec.parameters = {"xyz": "789"}
result = new_function(command="{}/no_ctx.py".format(tests_root_directory),
mode="noctx").run(spec)
verify_state(result)
db = get_run_db()
state, log = db.get_log(result.metadata.uid)
log = str(log)
print(state)
print(log)
assert log.find(", '--xyz', '789']") != -1, "params not detected in noctx"
def test_hyper_grid_parallel():
grid_params = '{"p2": [2,1,3], "p3": [10,20]}'
mlrun.datastore.set_in_memory_item("params.json", grid_params)
run_spec = tag_test(base_spec, "test_hyper_grid")
run_spec.with_param_file(
"memory://params.json", selector="r1", strategy="grid", parallel_runs=2
)
run = new_function().run(run_spec, handler=hyper_func)
verify_state(run)
# 3 x p2, 2 x p3 = 6 iterations + 1 header line
assert len(run.status.iterations) == 1 + 2 * 3, "wrong number of iterations"
def test_hyper_list():
list_params = '{"p2": [2,3,1], "p3": [10,30,20]}'
mlrun.datastore.set_in_memory_item("params.json", list_params)
run_spec = tag_test(base_spec, "test_hyper_list")
run_spec.with_param_file("memory://params.json", selector="r1", strategy="list")
run = new_function().run(run_spec, handler=hyper_func)
verify_state(run)
assert len(run.status.iterations) == 1 + 3, "wrong number of iterations"
results = [line[5] for line in run.status.iterations[1:]]
assert results == [20, 90, 20], "unexpected results"
assert run.output("best_iteration") == 2, "wrong best iteration"
def test_hyper_grid():
grid_params = '{"p2": [2,1,3], "p3": [10,20]}'
mlrun.datastore.set_in_memory_item("params.json", grid_params)
run_spec = tag_test(base_spec, "test_hyper_grid")
run_spec.with_param_file("memory://params.json", selector="r1", strategy="grid")
run = new_function().run(run_spec, handler=hyper_func)
verify_state(run)
# 3 x p2, 2 x p3 = 6 iterations + 1 header line
assert len(run.status.iterations) == 1 + 2 * 3, "wrong number of iterations"
results = [line[5] for line in run.status.iterations[1:]]
assert results == [20, 10, 30, 40, 20, 60], "unexpected results"
assert run.output("best_iteration") == 6, "wrong best iteration"
def test_hyper_list_with_stop():
list_params = '{"p2": [2,3,7,4,5], "p3": [10,10,10,10,10]}'
mlrun.datastore.set_in_memory_item("params.json", list_params)
run_spec = tag_test(base_spec, "test_hyper_list_with_stop")
run_spec.with_param_file(
"memory://params.json",
selector="max.r1",
strategy="list",
stop_condition="r1>=70",
)
run = new_function().run(run_spec, handler=hyper_func)
verify_state(run)
# result: r1 = p2 * p3, r1 >= 70 lead to stop on third run
assert len(run.status.iterations) == 1 + 3, "wrong number of iterations"
assert run.output("best_iteration") == 3, "wrong best iteration"
def test_run_local_with_uid_does_not_exist(monkeypatch):
"""
Mocking a scenario that happened in field in which getuser raised the same error as the mock
The problem was basically that the code was
environ.get("V3IO_USERNAME", getpass.getuser())
instead of
environ.get("V3IO_USERNAME") or getpass.getuser()
"""
def mock_getpwuid_raise(*args, **kwargs):
raise KeyError("getpwuid(): uid not found: 400")
environ["V3IO_USERNAME"] = "******"
monkeypatch.setattr(getpass, "getuser", mock_getpwuid_raise)
spec = tag_test(base_spec, "test_run_local")
result = run_local(spec,
command=f"{examples_path}/training.py",
workdir=examples_path)
verify_state(result)
def test_hyper_parallel_with_stop():
list_params = '{"p2": [2,3,7,4,5], "p3": [10,10,10,10,10]}'
mlrun.datastore.set_in_memory_item("params.json", list_params)
run_spec = mlrun.new_task(params={"p1": 1})
run_spec.with_hyper_params(
{
"p2": [2, 3, 7, 4, 5],
"p3": [10, 10, 10, 10, 10]
},
parallel_runs=2,
selector="max.r1",
strategy="list",
stop_condition="r1>=70",
)
run = new_function().run(run_spec, handler=hyper_func)
verify_state(run)
# result: r1 = p2 * p3, r1 >= 70 lead to stop on third run
# may have one extra iterations in flight so checking both 4 or 5
assert len(run.status.iterations) in [4, 5], "wrong number of iterations"
assert run.output("best_iteration") == 3, "wrong best iteration"
def test_azure_vault_end_to_end():
mlconf.dbpath = f"http://localhost:{api_db_port}"
project_name = "proj1"
# Create function and set container configuration
function = code_to_function(
name="azure_vault_func",
filename="vault_function.py",
handler="vault_func",
project=project_name,
kind="job",
)
function.spec.image = "mlrun/mlrun:unstable"
# Create context for the execution
spec = new_task(
project=project_name,
name="azure_vault_test_run",
handler="vault_func",
out_path=out_path,
params={"secrets": ["demo-key-1", "demo-key-2"]},
)
spec.with_secrets(
"azure_vault",
{
"name": "saar-key-vault",
"k8s_secret": azure_key_vault_k8s_secret,
"secrets": [],
},
)
result = function.run(spec)
verify_state(result)
db = get_run_db().connect()
db.get_log(result.metadata.uid, project=project_name)
def test_run_local():
spec = tag_test(base_spec, "test_run_local")
result = run_local(spec,
command=f"{examples_path}/training.py",
workdir=examples_path)
verify_state(result)
def test_local_handler():
spec = tag_test(base_spec, "test_local_runtime")
result = new_function(command="{}/handler.py".format(examples_path)).run(
spec, handler="my_func")
verify_state(result)
def test_run_local():
spec = tag_test(base_spec, 'test_run_local')
result = run_local(spec,
command='{}/training.py'.format(examples_path),
workdir=examples_path)
verify_state(result)
def test_dask_local():
spec = tag_test(NewTask(params={'p1': 3, 'p2': 'vv'}), 'test_dask_local')
run = new_function(kind='dask').run(spec, handler=my_func)
verify_state(run)