def test_serialization(): spec = tag_test(base_spec, "test_serialization") result = run_local(spec) verify_state(result) pprint(result.to_dict()) print(result.to_yaml()) pprint(result.to_json())
def test_run_local_handler(): spec = tag_test(base_spec, "test_run_local_handler") spec.spec.handler = "my_func" result = run_local(spec, command=f"{examples_path}/handler.py", workdir=examples_path) verify_state(result)
def test_run_local_nb(): spec = tag_test(base_spec, "test_run_local_nb") spec.spec.handler = "training" result = run_local(spec, command=f"{examples_path}/mlrun_jobs.ipynb", workdir=examples_path) verify_state(result)
def test_run_local_handler(): spec = tag_test(base_spec, 'test_run_local_handler') spec.spec.handler = 'my_func' result = run_local(spec, command='{}/handler.py'.format(examples_path), workdir=examples_path) verify_state(result)
def test_run_local_nb(): spec = tag_test(base_spec, 'test_run_local_handler') spec.spec.handler = 'training' result = run_local(spec, command='{}/mlrun_jobs.ipynb'.format(examples_path), workdir=examples_path) verify_state(result)
def test_handler_hyperlist(): run_spec = tag_test(base_spec, "test_handler_hyperlist") run_spec.spec.param_file = f"{tests_root_directory}/param_file.csv" result = new_function().run(run_spec, handler=my_func) print(result) assert len(result.status.iterations) == 3 + 1, "hyper parameters test failed" verify_state(result)
def test_handler_project(): spec = tag_test(base_spec, "test_handler_project") spec.metadata.project = "myproj" spec.metadata.labels = {"owner": "yaronh"} result = new_function().run(spec, handler=my_func) print(result) assert result.output("accuracy") == 16, "failed to run" verify_state(result)
def test_handler_project(): spec = tag_test(base_spec, 'test_handler_project') spec.metadata.project = 'myproj' spec.metadata.labels = {'owner': 'yaronh'} result = new_function().run(spec, handler=my_func) print(result) assert result.output('accuracy') == 16, 'failed to run' verify_state(result)
def test_handler_hyperlist(): run_spec = tag_test(base_spec, 'test_handler_hyperlist') run_spec.spec.param_file = '{}/param_file.csv'.format(here) result = new_function().run(run_spec, handler=my_func) print(result) assert len( result.status.iterations) == 3 + 1, 'hyper parameters test failed' verify_state(result)
def test_hyper_random(): grid_params = {"p2": [2, 1, 3], "p3": [10, 20, 30]} run_spec = tag_test(base_spec, "test_hyper_random") run_spec.with_hyper_params(grid_params, selector="r1", strategy="random") run_spec.spec.parameters["MAX_RANDOM_EVALS"] = 5 run = new_function().run(run_spec, handler=hyper_func) verify_state(run) assert len(run.status.iterations) == 1 + 5, "wrong number of iterations"
def test_run_local_obj(): spec = tag_test(base_spec, 'test_run_local_handler') spec.spec.handler = 'training' nbpath = '{}/mlrun_jobs.ipynb'.format(examples_path) ymlpath = path.join(out_path, 'nbyaml.yaml') print('out path:', out_path, ymlpath) fn = code_to_function(filename=nbpath, kind='job').export(ymlpath) result = run_local(spec, command=fn, workdir=out_path) verify_state(result)
def test_run_local_obj(): spec = tag_test(base_spec, "test_run_local_obj") spec.spec.handler = "training" nbpath = f"{examples_path}/mlrun_jobs.ipynb" ymlpath = path.join(out_path, "nbyaml.yaml") print("out path:", out_path, ymlpath) fn = code_to_function(filename=nbpath, kind="job").export(ymlpath) result = run_local(spec, command=fn, workdir=out_path) verify_state(result)
def test_hyper_custom(): run_spec = tag_test(base_spec, "test_hyper_custom") run = new_function().run(run_spec, handler=custom_hyper_func) verify_state(run) assert len(run.status.iterations) == 1 + 4, "wrong number of iterations" results = [line[3] for line in run.status.iterations[1:]] print(results) assert run.output("best_iteration") == 3, "wrong best iteration"
def test_run_local_from_func(): spec = tag_test(base_spec, "test_run_local_from_func") spec.spec.handler = "training" nb_path = f"{examples_path}/mlrun_jobs.ipynb" nbyml_path = path.join(out_path, "nbyaml.yaml") print("out path:", out_path, nbyml_path) fn = code_to_function(filename=nb_path, kind="job").export(nbyml_path) result = fn.run(spec, workdir=out_path, local=True) verify_state(result)
def test_vault_end_to_end(): # This requires an MLRun API server to run and work with Vault. This port should # be configured to allow access to the server. api_server_port = 57764 _set_vault_mlrun_configuration(api_server_port) project_name = "abc" func_name = "vault-function" aws_key_value = "1234567890" github_key_value = "proj1Key!!!" project = new_project(project_name) # This call will initialize Vault infrastructure and add the given secrets # It executes on the API server project.create_vault_secrets({ "aws_key": aws_key_value, "github_key": github_key_value }) # This API executes on the client side project_secrets = project.get_vault_secret_keys() assert project_secrets == ["aws_key", "github_key"], "secrets not created" # Create function and set container configuration function = code_to_function( name=func_name, filename="{}/vault_function.py".format(examples_path), handler="vault_func", project=project_name, kind="job", ) function.spec.image = "saarcoiguazio/mlrun:unstable" # Create context for the execution spec = new_task( project=project_name, name="vault_test_run", handler="vault_func", out_path=out_path, params={"secrets": ["password", "path", "github_key", "aws_key"]}, ) spec.with_secrets("vault", []) result = function.run(spec) verify_state(result) db = get_run_db().connect() state, log = db.get_log(result.metadata.uid, project=project_name) log = str(log) print(state) assert (log.find("value: {}".format(aws_key_value)) != -1), "secret value not detected in function output" assert (log.find("value: {}".format(github_key_value)) != -1), "secret value not detected in function output"
def test_handler_hyper(): run_spec = tag_test(base_spec, 'test_handler_hyper') run_spec.with_hyper_params({'p1': [1, 5, 3]}, selector='max.accuracy') result = new_function().run(run_spec, handler=my_func) print(result) assert len( result.status.iterations) == 3 + 1, 'hyper parameters test failed' assert (result.status.results['best_iteration'] == 2 ), 'failed to select best iteration' verify_state(result)
def test_handler_hyper(): run_spec = tag_test(base_spec, "test_handler_hyper") run_spec.with_hyper_params({"p1": [1, 5, 3]}, selector="max.accuracy") result = new_function().run(run_spec, handler=my_func) print(result) assert len( result.status.iterations) == 3 + 1, "hyper parameters test failed" assert (result.status.results["best_iteration"] == 2 ), "failed to select best iteration" verify_state(result)
def test_local_no_context(): spec = tag_test(base_spec, 'test_local_no_context') spec.spec.parameters = {'xyz': '789'} result = new_function(command='{}/no_ctx.py'.format(here), mode='noctx').run(spec) verify_state(result) db = get_run_db().connect() state, log = db.get_log(result.metadata.uid) log = str(log) print(state) print(log) assert log.find(", '--xyz', '789']") != -1, 'params not detected in noctx'
def test_local_no_context(): spec = tag_test(base_spec, "test_local_no_context") spec.spec.parameters = {"xyz": "789"} result = new_function(command="{}/no_ctx.py".format(tests_root_directory), mode="noctx").run(spec) verify_state(result) db = get_run_db() state, log = db.get_log(result.metadata.uid) log = str(log) print(state) print(log) assert log.find(", '--xyz', '789']") != -1, "params not detected in noctx"
def test_hyper_grid_parallel(): grid_params = '{"p2": [2,1,3], "p3": [10,20]}' mlrun.datastore.set_in_memory_item("params.json", grid_params) run_spec = tag_test(base_spec, "test_hyper_grid") run_spec.with_param_file( "memory://params.json", selector="r1", strategy="grid", parallel_runs=2 ) run = new_function().run(run_spec, handler=hyper_func) verify_state(run) # 3 x p2, 2 x p3 = 6 iterations + 1 header line assert len(run.status.iterations) == 1 + 2 * 3, "wrong number of iterations"
def test_hyper_list(): list_params = '{"p2": [2,3,1], "p3": [10,30,20]}' mlrun.datastore.set_in_memory_item("params.json", list_params) run_spec = tag_test(base_spec, "test_hyper_list") run_spec.with_param_file("memory://params.json", selector="r1", strategy="list") run = new_function().run(run_spec, handler=hyper_func) verify_state(run) assert len(run.status.iterations) == 1 + 3, "wrong number of iterations" results = [line[5] for line in run.status.iterations[1:]] assert results == [20, 90, 20], "unexpected results" assert run.output("best_iteration") == 2, "wrong best iteration"
def test_hyper_grid(): grid_params = '{"p2": [2,1,3], "p3": [10,20]}' mlrun.datastore.set_in_memory_item("params.json", grid_params) run_spec = tag_test(base_spec, "test_hyper_grid") run_spec.with_param_file("memory://params.json", selector="r1", strategy="grid") run = new_function().run(run_spec, handler=hyper_func) verify_state(run) # 3 x p2, 2 x p3 = 6 iterations + 1 header line assert len(run.status.iterations) == 1 + 2 * 3, "wrong number of iterations" results = [line[5] for line in run.status.iterations[1:]] assert results == [20, 10, 30, 40, 20, 60], "unexpected results" assert run.output("best_iteration") == 6, "wrong best iteration"
def test_hyper_list_with_stop(): list_params = '{"p2": [2,3,7,4,5], "p3": [10,10,10,10,10]}' mlrun.datastore.set_in_memory_item("params.json", list_params) run_spec = tag_test(base_spec, "test_hyper_list_with_stop") run_spec.with_param_file( "memory://params.json", selector="max.r1", strategy="list", stop_condition="r1>=70", ) run = new_function().run(run_spec, handler=hyper_func) verify_state(run) # result: r1 = p2 * p3, r1 >= 70 lead to stop on third run assert len(run.status.iterations) == 1 + 3, "wrong number of iterations" assert run.output("best_iteration") == 3, "wrong best iteration"
def test_run_local_with_uid_does_not_exist(monkeypatch): """ Mocking a scenario that happened in field in which getuser raised the same error as the mock The problem was basically that the code was environ.get("V3IO_USERNAME", getpass.getuser()) instead of environ.get("V3IO_USERNAME") or getpass.getuser() """ def mock_getpwuid_raise(*args, **kwargs): raise KeyError("getpwuid(): uid not found: 400") environ["V3IO_USERNAME"] = "******" monkeypatch.setattr(getpass, "getuser", mock_getpwuid_raise) spec = tag_test(base_spec, "test_run_local") result = run_local(spec, command=f"{examples_path}/training.py", workdir=examples_path) verify_state(result)
def test_hyper_parallel_with_stop(): list_params = '{"p2": [2,3,7,4,5], "p3": [10,10,10,10,10]}' mlrun.datastore.set_in_memory_item("params.json", list_params) run_spec = mlrun.new_task(params={"p1": 1}) run_spec.with_hyper_params( { "p2": [2, 3, 7, 4, 5], "p3": [10, 10, 10, 10, 10] }, parallel_runs=2, selector="max.r1", strategy="list", stop_condition="r1>=70", ) run = new_function().run(run_spec, handler=hyper_func) verify_state(run) # result: r1 = p2 * p3, r1 >= 70 lead to stop on third run # may have one extra iterations in flight so checking both 4 or 5 assert len(run.status.iterations) in [4, 5], "wrong number of iterations" assert run.output("best_iteration") == 3, "wrong best iteration"
def test_azure_vault_end_to_end(): mlconf.dbpath = f"http://localhost:{api_db_port}" project_name = "proj1" # Create function and set container configuration function = code_to_function( name="azure_vault_func", filename="vault_function.py", handler="vault_func", project=project_name, kind="job", ) function.spec.image = "mlrun/mlrun:unstable" # Create context for the execution spec = new_task( project=project_name, name="azure_vault_test_run", handler="vault_func", out_path=out_path, params={"secrets": ["demo-key-1", "demo-key-2"]}, ) spec.with_secrets( "azure_vault", { "name": "saar-key-vault", "k8s_secret": azure_key_vault_k8s_secret, "secrets": [], }, ) result = function.run(spec) verify_state(result) db = get_run_db().connect() db.get_log(result.metadata.uid, project=project_name)
def test_run_local(): spec = tag_test(base_spec, "test_run_local") result = run_local(spec, command=f"{examples_path}/training.py", workdir=examples_path) verify_state(result)
def test_local_handler(): spec = tag_test(base_spec, "test_local_runtime") result = new_function(command="{}/handler.py".format(examples_path)).run( spec, handler="my_func") verify_state(result)
def test_run_local(): spec = tag_test(base_spec, 'test_run_local') result = run_local(spec, command='{}/training.py'.format(examples_path), workdir=examples_path) verify_state(result)
def test_dask_local(): spec = tag_test(NewTask(params={'p1': 3, 'p2': 'vv'}), 'test_dask_local') run = new_function(kind='dask').run(spec, handler=my_func) verify_state(run)