class TestProject():
def setup_class(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.cli = Helper()
self.init = ProjectCommand(self.temp_dir, self.cli)
def teardown_class(self):
pass
def test_datmo_init(self):
self.init.parse(
["init", "--name", "foobar", "--description", "test model"])
self.init.execute()
# test for desired side effects
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
def test_datmo_init_invalid_arg(self):
exception_thrown = False
try:
self.init.parse(["init", "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def __set_variables(self):
init = ProjectCommand(self.temp_dir, self.cli_helper)
init.parse(["init", "--name", "foobar", "--description", "test model"])
init.execute()
self.task = TaskCommand(self.temp_dir, self.cli_helper)
# Create environment_driver definition
env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
def setup_class(self):
# provide mountable tmp directory for docker
tempfile.tempdir = '/tmp'
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.cli_helper = Helper()
self.session_command = SessionCommand(self.temp_dir, self.cli_helper)
init = ProjectCommand(self.temp_dir, self.cli_helper)
init.parse(["init", "--name", "foobar", "--description", "test model"])
init.execute()
class TestSessionCommand():
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.cli_helper = Helper()
def teardown_method(self):
pass
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.session_command = SessionCommand(self.cli_helper)
def test_session_no_subcommand(self):
self.__set_variables()
self.session_command.parse(["session"])
assert self.session_command.execute()
def test_session_create(self):
self.__set_variables()
self.session_command.parse(["session", "create", "--name", "pizza"])
result = self.session_command.execute()
assert result
return result
def test_session_select(self):
session_obj = self.test_session_create()
# Test successful select name
self.session_command.parse(["session", "select", "pizza"])
assert self.session_command.execute()
current = 0
for s in self.session_command.session_controller.list():
print("%s - %s" % (s.name, s.current))
if s.current == True:
current = current + 1
assert current == 1
# Reset to default
self.session_command.parse(["session", "select", "default"])
# Test successful select id
self.session_command.parse(["session", "select", session_obj.id])
assert self.session_command.execute()
current = 0
for s in self.session_command.session_controller.list():
print("%s - %s" % (s.name, s.current))
if s.current == True:
current = current + 1
assert current == 1
def test_session_ls(self):
created_session_obj = self.test_session_create()
# Test success (defaults)
self.session_command.parse(["session", "ls"])
session_objs = self.session_command.execute()
assert created_session_obj in session_objs
# Test failure (format)
failed = False
try:
self.session_command.parse(["session", "ls", "--format"])
except ArgumentError:
failed = True
assert failed
# Test success format csv
self.session_command.parse(["session", "ls", "--format", "csv"])
session_objs = self.session_command.execute()
assert created_session_obj in session_objs
# Test success format csv, download default
self.session_command.parse(
["session", "ls", "--format", "csv", "--download"])
session_objs = self.session_command.execute()
assert created_session_obj in session_objs
test_wildcard = os.path.join(
self.session_command.session_controller.home, "session_ls_*")
paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
assert paths
assert open(paths[0], "r").read()
os.remove(paths[0])
# Test success format csv, download exact path
test_path = os.path.join(self.temp_dir, "my_output")
self.session_command.parse([
"session", "ls", "--format", "csv", "--download",
"--download-path", test_path
])
session_objs = self.session_command.execute()
assert created_session_obj in session_objs
assert os.path.isfile(test_path)
assert open(test_path, "r").read()
os.remove(test_path)
# Test success format table
self.session_command.parse(["session", "ls"])
session_objs = self.session_command.execute()
assert created_session_obj in session_objs
# Test success format table, download default
self.session_command.parse(["session", "ls", "--download"])
session_objs = self.session_command.execute()
assert created_session_obj in session_objs
test_wildcard = os.path.join(
self.session_command.session_controller.home, "session_ls_*")
paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
assert paths
assert open(paths[0], "r").read()
os.remove(paths[0])
# Test success format table, download exact path
test_path = os.path.join(self.temp_dir, "my_output")
self.session_command.parse(
["session", "ls", "--download", "--download-path", test_path])
session_objs = self.session_command.execute()
assert created_session_obj in session_objs
assert os.path.isfile(test_path)
assert open(test_path, "r").read()
os.remove(test_path)
def test_session_update(self):
created_session_obj = self.test_session_create()
# Test successful update
updated_name = "new"
self.session_command.parse([
"session", "update", created_session_obj.id, "--name", updated_name
])
assert self.session_command.execute()
# Test failure update
self.session_command.parse(
["session", "update", "random_id", "--name", updated_name])
assert not self.session_command.execute()
# Test success nothing given
self.session_command.parse(
["session", "update", created_session_obj.id])
assert self.session_command.execute()
def test_session_delete(self):
self.test_session_create()
# Test successful delete name
self.session_command.parse(["session", "delete", "pizza"])
assert self.session_command.execute()
# Test failure delete default name
self.session_command.parse(["session", "delete", "default"])
assert not self.session_command.execute()
# Test failure delete random name
self.session_command.parse(["session", "delete", "random_name"])
assert not self.session_command.execute()
session_obj = self.test_session_create()
# Test successful delete id
self.session_command.parse(["session", "delete", session_obj.id])
assert self.session_command.execute()
# Test failure delete default id
self.session_command.parse(["session", "ls"])
session_objs = self.session_command.execute()
self.session_command.parse(["session", "delete", session_objs[0].id])
assert not self.session_command.execute()
# Test failure delete random id
self.session_command.parse(["session", "delete", "random_id"])
assert not self.session_command.execute()
class TestRunCommand():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.cli_helper = Helper()
self.snapshot_dict = {
"id": "test",
"model_id": "my_model",
"session_id": "my_session",
"message": "my message",
"code_id": "my_code_id",
"environment_id": "my_environment_id",
"file_collection_id": "my file collection",
"config": {
"test": 0.56
},
"stats": {
"test": 0.34
}
}
self.task_dict = {
"id": "test",
"model_id": "my_model",
"session_id": "my_session",
"command": "python test.py"
}
def teardown_method(self):
pass
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.run_command = RunCommand(self.cli_helper)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
def test_run_object_instantiate(self):
task_obj = CoreTask(self.task_dict)
result = RunObject(task_obj)
assert result
assert isinstance(result, RunObject)
def test_run_should_fail1(self):
self.__set_variables()
# Test failure case
self.run_command.parse(["run"])
result = self.run_command.execute()
assert not result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_run(self):
# TODO: Adding test with `--interactive` argument and terminate inside container
self.__set_variables()
# Test failure command execute
test_command = ["yo", "yo"]
self.run_command.parse(["run", test_command])
result = self.run_command.execute()
assert result
# Test success case
test_command = ["sh", "-c", "echo accuracy:0.45"]
test_ports = ["8888:8888", "9999:9999"]
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
test_mem_limit = "4g"
# test for single set of ports
self.run_command.parse([
"run", "--ports", test_ports[0], "--environment-paths",
test_dockerfile, "--mem-limit", test_mem_limit, test_command
])
# test for desired side effects
assert self.run_command.args.cmd == test_command
assert self.run_command.args.ports == [test_ports[0]]
assert self.run_command.args.environment_paths == [test_dockerfile]
assert self.run_command.args.mem_limit == test_mem_limit
self.run_command.parse([
"run", "-p", test_ports[0], "-p", test_ports[1],
"--environment-paths", test_dockerfile, "--mem-limit",
test_mem_limit, test_command
])
# test for desired side effects
assert self.run_command.args.cmd == test_command
assert self.run_command.args.ports == test_ports
assert self.run_command.args.environment_paths == [test_dockerfile]
assert self.run_command.args.mem_limit == test_mem_limit
# test proper execution of run command
result = self.run_command.execute()
time.sleep(1)
assert result
assert isinstance(result, RunObject)
assert result.logs
assert "accuracy" in result.logs
assert result.results
assert result.results == {"accuracy": "0.45"}
assert result.status == "SUCCESS"
assert result.start_time
assert result.end_time
assert result.duration
assert result.core_snapshot_id
assert result.core_snapshot_id == result.after_snapshot_id
assert result.environment_id
# teardown
self.run_command.parse(["stop", "--all"])
# test when all is passed to stop all
self.run_command.execute()
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_run_data_dir(self):
# TODO: Adding test with `--interactive` argument and terminate inside container
self.__set_variables()
# Test success case
test_filename = "script.py"
test_command = ["python", test_filename]
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
test_mem_limit = "4g"
# Test success for run with directory being passed
test_data_dir_1 = os.path.join(tempfile.mkdtemp(dir=test_datmo_dir),
"data1")
os.mkdir(test_data_dir_1)
test_data_dir_2 = os.path.join(tempfile.mkdtemp(dir=test_datmo_dir),
"data2")
os.mkdir(test_data_dir_2)
with open(os.path.join(test_data_dir_1, "file.txt"), "wb") as f:
f.write(to_bytes('my initial line in 1\n'))
with open(os.path.join(test_data_dir_2, "file.txt"), "wb") as f:
f.write(to_bytes('my initial line in 2\n'))
test_filename = "script.py"
test_filepath = os.path.join(self.temp_dir, test_filename)
with open(test_filepath, "wb") as f:
f.write(to_bytes("import os\n"))
f.write(to_bytes("print('hello')\n"))
f.write(to_bytes("import shutil\n"))
f.write(
to_bytes(
"with open(os.path.join('/data', 'data1', 'file.txt'), 'a') as f:\n"
))
f.write(to_bytes(" f.write('my test file in 1')\n"))
f.write(
to_bytes(
"with open(os.path.join('/data', 'data2', 'file.txt'), 'a') as f:\n"
))
f.write(to_bytes(" f.write('my test file in 2')\n"))
self.run_command.parse([
"run", "--environment-paths", test_dockerfile, "--data",
test_data_dir_1, "--data", test_data_dir_2, "--mem-limit",
test_mem_limit, test_command
])
# test proper execution of run command
result = self.run_command.execute()
time.sleep(1)
assert result
assert isinstance(result, RunObject)
assert result.logs
assert result.status == "SUCCESS"
assert result.start_time
assert result.end_time
assert result.duration
assert result.core_snapshot_id
assert result.core_snapshot_id == result.after_snapshot_id
assert result.environment_id
assert "my initial line in 1" in open(
os.path.join(test_data_dir_1, "file.txt"), "r").read()
assert "my test file in 1" in open(
os.path.join(test_data_dir_1, "file.txt"), "r").read()
assert "my initial line in 2" in open(
os.path.join(test_data_dir_2, "file.txt"), "r").read()
assert "my test file in 2" in open(
os.path.join(test_data_dir_2, "file.txt"), "r").read()
# teardown
self.run_command.parse(["stop", "--all"])
# test when all is passed to stop all
self.run_command.execute()
# test failure
test_data_dir_dne = os.path.join(tempfile.mkdtemp(dir=test_datmo_dir),
"data_dne")
self.run_command.parse([
"run", "--environment-paths", test_dockerfile, "--data",
test_data_dir_dne
])
result = self.run_command.execute()
assert not result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_run_string_command(self):
# TODO: Adding test with `--interactive` argument and terminate inside container
self.__set_variables()
# Test success case
test_command = "sh -c 'echo accuracy:0.45'"
test_ports = ["8888:8888", "9999:9999"]
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
test_mem_limit = "4g"
self.run_command.parse([
"run", "--ports", test_ports[0], "--ports", test_ports[1],
"--environment-paths", test_dockerfile, "--mem-limit",
test_mem_limit, test_command
])
# test for desired side effects
assert self.run_command.args.cmd == test_command
assert self.run_command.args.ports == test_ports
assert self.run_command.args.environment_paths == [test_dockerfile]
assert self.run_command.args.mem_limit == test_mem_limit
# test proper execution of run command
result = self.run_command.execute()
assert result
assert isinstance(result, RunObject)
assert result.logs
assert "accuracy" in result.logs
assert result.results
assert result.results == {"accuracy": "0.45"}
assert result.status == "SUCCESS"
assert result.start_time
assert result.end_time
assert result.duration
assert result.core_snapshot_id
assert result.core_snapshot_id == result.after_snapshot_id
assert result.environment_id
assert result
# teardown
self.run_command.parse(["stop", "--all"])
# test when all is passed to stop all
self.run_command.execute()
# def test_multiple_concurrent_run_command(self):
# test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
# test_command = ["sh", "-c", "echo accuracy:0.45"]
# manager = Manager()
# return_dict = manager.dict()
#
# def run_exec_func(procnum, return_dict):
# print("Creating Task object")
# run = RunCommand(self.temp_dir, self.cli_helper)
# print("Parsing command")
# run.parse(
# ["run", "--environment-paths", test_dockerfile, test_command])
# print("Executing command")
# result = run.execute()
# return_dict[procnum] = result
#
# self.__set_variables()
# test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
#
# # Run all three runs in parallel
# jobs = []
# number_runs = 3
# for i in range(number_runs):
# p = Process(target=run_exec_func, args=(i, return_dict))
# jobs.append(p)
# p.start()
#
# # Join
# for proc in jobs:
# proc.join()
#
# results = return_dict.values()
# assert len(results) == number_runs
# for result in results:
# assert result
# assert isinstance(result, CoreTask)
# assert result.logs
# assert "accuracy" in result.logs
# assert result.results
# assert result.results == {"accuracy": "0.45"}
# assert result.status == "SUCCESS"
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_run_notebook(self):
self.__set_variables()
# Update the default Dockerfile to test with
with open(self.env_def_path, "wb") as f:
f.write(to_bytes("FROM nbgallery/jupyter-alpine:latest"))
# Test success case
test_command = ["jupyter", "notebook", "list"]
test_ports = ["8888:8888", "9999:9999"]
test_mem_limit = "4g"
# test single ports option before command
self.run_command.parse([
"run", "--ports", test_ports[0], "--mem-limit", test_mem_limit,
test_command
])
# test for desired side effects
assert self.run_command.args.cmd == test_command
assert self.run_command.args.ports == [test_ports[0]]
assert self.run_command.args.mem_limit == test_mem_limit
# test multiple ports option before command
self.run_command.parse([
"run", "--ports", test_ports[0], "--ports", test_ports[1],
"--mem-limit", test_mem_limit, "--environment-paths",
self.env_def_path, test_command
])
# test for desired side effects
assert self.run_command.args.cmd == test_command
assert self.run_command.args.ports == test_ports
assert self.run_command.args.mem_limit == test_mem_limit
# test proper execution of run command
result = self.run_command.execute()
assert result
assert isinstance(result, RunObject)
assert result.logs
assert "Currently running servers" in result.logs
assert result.status == "SUCCESS"
# teardown
self.run_command.parse(["stop", "--all"])
# test when all is passed to stop all
self.run_command.execute()
def test_run_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.run_command.parse(["run" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_run_ls(self):
self.__set_variables()
# Running a task
test_command = ["sh", "-c", "echo accuracy:0.45"]
test_ports = ["8888:8888", "9999:9999"]
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
test_mem_limit = "4g"
# test for single set of ports
self.run_command.parse([
"run", "--ports", test_ports[0], "--environment-paths",
test_dockerfile, "--mem-limit", test_mem_limit, test_command
])
# test proper execution of run command
self.run_command.execute()
# Test defaults
self.run_command.parse(["ls"])
run_objs = self.run_command.execute()
assert run_objs
assert run_objs[0].status == "SUCCESS"
test_session_id = 'test_session_id'
self.run_command.parse(["ls", "--session-id", test_session_id])
# test for desired side effects
assert self.run_command.args.session_id == test_session_id
# Test failure no session
failed = False
try:
_ = self.run_command.execute()
except SessionDoesNotExist:
failed = True
assert failed
# Test failure (format)
failed = False
try:
self.run_command.parse(["ls", "--format"])
except ArgumentError:
failed = True
assert failed
# Test success format csv
self.run_command.parse(["ls", "--format", "csv"])
run_objs = self.run_command.execute()
assert run_objs
assert run_objs[0].status == "SUCCESS"
# Test success format csv, download default
self.run_command.parse(["ls", "--format", "csv", "--download"])
run_objs = self.run_command.execute()
assert run_objs
assert run_objs[0].status == "SUCCESS"
test_wildcard = os.path.join(os.getcwd(), "run_ls_*")
paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
assert paths
assert open(paths[0], "r").read()
os.remove(paths[0])
# Test success format csv, download exact path
test_path = os.path.join(self.temp_dir, "my_output")
self.run_command.parse([
"ls", "--format", "csv", "--download", "--download-path", test_path
])
run_objs = self.run_command.execute()
assert run_objs
assert run_objs[0].status == "SUCCESS"
assert os.path.isfile(test_path)
assert open(test_path, "r").read()
os.remove(test_path)
# Test success format table
self.run_command.parse(["ls"])
run_objs = self.run_command.execute()
assert run_objs
assert run_objs[0].status == "SUCCESS"
# Test success format table, download default
self.run_command.parse(["ls", "--download"])
run_objs = self.run_command.execute()
assert run_objs
assert run_objs[0].status == "SUCCESS"
test_wildcard = os.path.join(os.getcwd(), "run_ls_*")
paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
assert paths
assert open(paths[0], "r").read()
os.remove(paths[0])
# Test success format table, download exact path
test_path = os.path.join(self.temp_dir, "my_output")
self.run_command.parse(
["ls", "--download", "--download-path", test_path])
run_objs = self.run_command.execute()
assert run_objs
assert run_objs[0].status == "SUCCESS"
assert os.path.isfile(test_path)
assert open(test_path, "r").read()
os.remove(test_path)
def test_run_ls_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.run_command.parse(["ls" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_run_stop_success(self):
# 1) Test stop with task_id
# 2) Test stop with all
self.__set_variables()
test_command = ["sh", "-c", "echo yo"]
test_ports = "8888:8888"
test_mem_limit = "4g"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.run_command.parse([
"run", "--ports", test_ports, "--environment-paths",
test_dockerfile, "--mem-limit", test_mem_limit, test_command
])
test_run_obj = self.run_command.execute()
# 1) Test option 1
self.run_command.parse(["stop", "--id", test_run_obj.id])
# test for desired side effects
assert self.run_command.args.id == test_run_obj.id
# test when task id is passed to stop it
run_stop_command = self.run_command.execute()
assert run_stop_command == True
# 2) Test option 2
self.run_command.parse(["stop", "--all"])
# test when all is passed to stop all
run_stop_command = self.run_command.execute()
assert run_stop_command == True
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_run_stop_failure_required_args(self):
self.__set_variables()
# Passing wrong task id
self.run_command.parse(["stop"])
failed = False
try:
_ = self.run_command.execute()
except RequiredArgumentMissing:
failed = True
assert failed
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_run_stop_failure_mutually_exclusive_vars(self):
self.__set_variables()
# Passing wrong task id
self.run_command.parse(["stop", "--id", "invalid-task-id", "--all"])
failed = False
try:
_ = self.run_command.execute()
except MutuallyExclusiveArguments:
failed = True
assert failed
def test_run_stop_failure_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.run_command.parse(["stop" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_run_stop_invalid_task_id(self):
self.__set_variables()
# Passing wrong task id
self.run_command.parse(["stop", "--id", "invalid-task-id"])
# test when wrong task id is passed to stop it
result = self.run_command.execute()
assert not result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_run_delete_success(self):
# 1) Test delete with run_id
self.__set_variables()
test_command = ["sh", "-c", "echo yo"]
test_ports = "8888:8888"
test_mem_limit = "4g"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.run_command.parse([
"run", "--ports", test_ports, "--environment-paths",
test_dockerfile, "--mem-limit", test_mem_limit, test_command
])
test_run_obj = self.run_command.execute()
# 1) Test option 1
self.run_command.parse(["delete", test_run_obj.id])
# test for desired side effects
assert self.run_command.args.id == test_run_obj.id
# test when task id is passed to delete it
run_delete_command = self.run_command.execute()
assert run_delete_command == True
def test_run_delete_invalid_task_id(self):
self.__set_variables()
# Passing wrong task id
self.run_command.parse(["delete", "invalid-task-id"])
# test when wrong task id is passed to stop it
result = self.run_command.execute()
assert not result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_rerun(self):
self.__set_variables()
# Test success case
test_command = ["sh", "-c", "echo accuracy:0.45"]
test_ports = ["8888:8888", "9999:9999"]
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
test_mem_limit = "4g"
# test for single set of ports
self.run_command.parse([
"run", "-p", test_ports[0], "--environment-paths", test_dockerfile,
"--mem-limit", test_mem_limit, test_command
])
# test proper execution of run command
run_obj = self.run_command.execute()
run_id = run_obj.id
# 1. Test success rerun
self.run_command.parse(["rerun", run_id])
result_run_obj = self.run_command.execute()
assert result_run_obj
assert isinstance(result_run_obj, RunObject)
assert result_run_obj.command == run_obj.command
assert result_run_obj.status == "SUCCESS"
assert result_run_obj.logs
assert result_run_obj.before_snapshot_id == run_obj.before_snapshot_id
# teardown
self.run_command.parse(["stop", "--all"])
# test when all is passed to stop all
self.run_command.execute()
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_rerun_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.run_command.parse(["rerun", "foobar"])
self.run_command.execute()
except DoesNotExist:
exception_thrown = True
assert exception_thrown
class TestFlow():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.cli_helper = Helper()
def teardown_method(self):
pass
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.environment_command = EnvironmentCommand(self.cli_helper)
self.run_command = RunCommand(self.cli_helper)
self.snapshot_command = SnapshotCommand(self.cli_helper)
# Create test file
self.filepath = os.path.join(self.snapshot_command.home, "file.txt")
with open(self.filepath, "wb") as f:
f.write(to_bytes(str("test")))
def __environment_setup(self):
self.environment_command.parse(["environment", "setup"])
@self.environment_command.cli_helper.input("\n\n\n")
def dummy(self):
return self.environment_command.execute()
environment_setup_result = dummy(self)
return environment_setup_result
def __run(self):
test_command = ["sh", "-c", "echo accuracy:0.45"]
test_ports = ["8888:8888"]
self.run_command.parse(["run", "-p", test_ports[0], test_command])
run_result = self.run_command.execute()
return run_result
def __run_ls(self):
self.run_command.parse(["ls"])
run_ls_result = self.run_command.execute()
return run_ls_result
def __snapshot_create(self):
test_message = "creating a snapshot"
self.snapshot_command.parse(["snapshot", "create", "-m", test_message])
snapshot_create_result = self.snapshot_command.execute()
return snapshot_create_result
def __snapshot_ls(self):
self.snapshot_command.parse(["snapshot", "ls"])
snapshot_ls_result = self.snapshot_command.execute()
return snapshot_ls_result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_flow_1(self):
# Flow
# Step 1: environment setup
# Step 2: run
# Step 3: ls
# Step 4: snapshot create
# Step 5: snapshot ls
self.__set_variables()
# Step 1: environment setup
environment_setup_result = self.__environment_setup()
assert environment_setup_result
# Step 2: run command
run_result = self.__run()
assert run_result
# Step 3: ls
run_ls_result = self.__run_ls()
assert run_ls_result
# Step 4: snapshot create
snapshot_create_result = self.__snapshot_create()
assert snapshot_create_result
# Step 5: snapshot ls
snapshot_ls_result = self.__snapshot_ls()
assert snapshot_ls_result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_flow_2(self):
# Flow interruption in environment
# Step 1: interrupted environment setup
# Step 2: environment setup
# Step 3: run
# Step 4: ls
# Step 5: snapshot create
# Step 6: snapshot ls
self.__set_variables()
# TODO: TEST more fundamental functions first
# Step 1: interrupted environment setup
# @timeout_decorator.timeout(0.0001, use_signals=False)
# def timed_command_with_interuption():
# result = self.__environment_setup()
# return result
#
# failed = False
# try:
# timed_command_with_interuption()
# except timeout_decorator.timeout_decorator.TimeoutError:
# failed = True
# assert failed
# Step 2: environment setup
environment_setup_result = self.__environment_setup()
assert environment_setup_result
# Step 3: run
run_result = self.__run()
assert run_result
# Step 4: ls
run_ls_result = self.__run_ls()
assert run_ls_result
# Step 5: snapshot create
snapshot_create_result = self.__snapshot_create()
assert snapshot_create_result
# Step 6: snapshot ls
snapshot_ls_result = self.__snapshot_ls()
assert snapshot_ls_result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_flow_3(self):
# Flow interruption in run
# Step 1: environment setup
# Step 2: interrupted run
# Step 3: run
# Step 4: ls
# Step 5: snapshot create
# Step 6: snapshot ls
self.__set_variables()
# Step 1: environment setup
environment_setup_result = self.__environment_setup()
assert environment_setup_result
# TODO: Test more fundamental functions first
# Step 2: interrupted run
# @timeout_decorator.timeout(0.0001, use_signals=False)
# def timed_command_with_interuption():
# result = self.__run()
# return result
#
# failed = False
# try:
# timed_command_with_interuption()
# except timeout_decorator.timeout_decorator.TimeoutError:
# failed = True
# assert failed
# Step 3: run
run_result = self.__run()
assert run_result
# Step 4: task ls
run_ls_result = self.__run_ls()
assert run_ls_result
# Step 5: snapshot create
snapshot_create_result = self.__snapshot_create()
assert snapshot_create_result
# Step 6: snapshot ls
snapshot_ls_result = self.__snapshot_ls()
assert snapshot_ls_result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_flow_4(self):
# Flow interruption in snapshot create
# Step 1: environment setup
# Step 2: run
# Step 3: ls
# Step 4: interrupted snapshot create
# Step 5: snapshot create
# Step 6: snapshot ls
self.__set_variables()
# Step 1: environment setup
environment_setup_result = self.__environment_setup()
assert environment_setup_result
# Step 2: run
run_result = self.__run()
assert run_result
# Step 3: ls
run_ls_result = self.__run_ls()
assert run_ls_result
# TODO: Test more fundamental functions first
# Step 4: interrupted snapshot create
# @timeout_decorator.timeout(0.0001, use_signals=False)
# def timed_command_with_interuption():
# result = self.__snapshot_create()
# return result
#
# failed = False
# try:
# timed_command_with_interuption()
# except timeout_decorator.timeout_decorator.TimeoutError:
# failed = True
# assert failed
# snapshot_ls_result = self.__snapshot_ls()
# assert not snapshot_ls_result
# Step 5: snapshot create
snapshot_create_result = self.__snapshot_create()
assert snapshot_create_result
# Step 6: snapshot ls
snapshot_ls_result = self.__snapshot_ls()
assert snapshot_ls_result
class TestProject():
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.cli_helper = Helper()
self.project = ProjectCommand(self.temp_dir, self.cli_helper)
def teardown_method(self):
pass
def test_init_create_success(self):
test_name = "foobar"
test_description = "test model"
self.project.parse(
["init", "--name", test_name, "--description", test_description])
result = self.project.execute()
# test for desired side effects
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result.name == test_name
assert result.description == test_description
def test_init_create_failure(self):
self.project.parse(["init", "--name", "", "--description", ""])
# test if prompt opens
@self.project.cli_helper.input("\n\n")
def dummy(self):
return self.project.execute()
result = dummy(self)
assert not result
def test_init_update_success(self):
test_name = "foobar"
test_description = "test model"
self.project.parse(
["init", "--name", test_name, "--description", test_description])
result_1 = self.project.execute()
updated_name = "foobar2"
updated_description = "test model 2"
self.project.parse([
"init", "--name", updated_name, "--description",
updated_description
])
result_2 = self.project.execute()
# test for desired side effects
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result_2.id == result_1.id
assert result_2.name == updated_name
assert result_2.description == updated_description
def test_init_update_success_only_name(self):
test_name = "foobar"
test_description = "test model"
self.project.parse(
["init", "--name", test_name, "--description", test_description])
result_1 = self.project.execute()
updated_name = "foobar2"
self.project.parse(
["init", "--name", updated_name, "--description", ""])
@self.project.cli_helper.input("\n")
def dummy(self):
return self.project.execute()
result_2 = dummy(self)
# test for desired side effects
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result_2.id == result_1.id
assert result_2.name == updated_name
assert result_2.description == result_1.description
def test_init_update_success_only_description(self):
test_name = "foobar"
test_description = "test model"
self.project.parse(
["init", "--name", test_name, "--description", test_description])
result_1 = self.project.execute()
updated_description = "test model 2"
self.project.parse(
["init", "--name", "", "--description", updated_description])
@self.project.cli_helper.input("\n")
def dummy(self):
return self.project.execute()
result_2 = dummy(self)
# test for desired side effects
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result_2.id == result_1.id
assert result_2.name == result_1.name
assert result_2.description == updated_description
def test_init_invalid_arg(self):
exception_thrown = False
try:
self.project.parse(["init", "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_version(self):
self.project.parse(["version"])
result = self.project.execute()
# test for desired side effects
assert __version__ in result
def test_version_invalid_arg(self):
exception_thrown = False
try:
self.project.parse(["version", "--foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_status(self):
test_name = "foobar"
test_description = "test model"
self.project.parse(
["init", "--name", test_name, "--description", test_description])
_ = self.project.execute()
self.project.parse(["status"])
result = self.project.execute()
assert isinstance(result[0], dict)
assert not result[1]
assert not result[2]
def test_status_invalid_arg(self):
exception_thrown = False
try:
self.project.parse(["status", "--foobar"])
except UnrecognizedCLIArgument:
exception_thrown = True
assert exception_thrown
def test_cleanup(self):
test_name = "foobar"
test_description = "test model"
self.project.parse(
["init", "--name", test_name, "--description", test_description])
_ = self.project.execute()
self.project.parse(["cleanup"])
@self.project.cli_helper.input("y\n")
def dummy(self):
return self.project.execute()
result = dummy(self)
assert not os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert isinstance(result, bool)
assert result
def test_cleanup_invalid_arg(self):
exception_thrown = False
try:
self.project.parse(["cleanup", "--foobar"])
except UnrecognizedCLIArgument:
exception_thrown = True
assert exception_thrown
class TestWorkspace():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.cli_helper = Helper()
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.workspace_command = WorkspaceCommand(self.cli_helper)
self.task_command = TaskCommand(self.cli_helper)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "wb") as f:
f.write(to_bytes(str("FROM datmo/xgboost:cpu\n")))
def teardown_method(self):
pass
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_notebook(self):
self.__set_variables()
test_mem_limit = "4g"
# test single ports option before command
self.workspace_command.parse([
"notebook",
"--gpu",
"--environment-paths",
self.env_def_path,
"--mem-limit",
test_mem_limit,
])
# test for desired side effects
assert self.workspace_command.args.gpu == True
assert self.workspace_command.args.environment_paths == [
self.env_def_path
]
assert self.workspace_command.args.mem_limit == test_mem_limit
# test multiple ports option before command
self.workspace_command.parse(["notebook"])
assert self.workspace_command.args.gpu == False
@timeout_decorator.timeout(10, use_signals=False)
def timed_run(timed_run_result):
if self.workspace_command.execute():
return timed_run_result
timed_run_result = False
try:
timed_run_result = timed_run(timed_run_result)
except timeout_decorator.timeout_decorator.TimeoutError:
timed_run_result = True
assert timed_run_result
# Stop all running datmo task
self.task_command.parse(["task", "stop", "--all"])
self.task_command.execute()
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_rstudio(self):
self.__set_variables()
# Update environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "wb") as f:
f.write(to_bytes(str("FROM datmo/rstudio:latest\n")))
test_mem_limit = "4g"
# test single ports option before command
self.workspace_command.parse([
"rstudio",
"--environment-paths",
self.env_def_path,
"--mem-limit",
test_mem_limit,
])
# test for desired side effects
assert self.workspace_command.args.environment_paths == [
self.env_def_path
]
assert self.workspace_command.args.mem_limit == test_mem_limit
# test multiple ports option before command
self.workspace_command.parse(["rstudio"])
@timeout_decorator.timeout(10, use_signals=False)
def timed_run(timed_run_result):
if self.workspace_command.execute():
return timed_run_result
timed_run_result = False
try:
timed_run_result = timed_run(timed_run_result)
except timeout_decorator.timeout_decorator.TimeoutError:
timed_run_result = True
assert timed_run_result
# Stop all running datmo task
self.task_command.parse(["task", "stop", "--all"])
self.task_command.execute()
class TestEnvironmentCommand():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.cli_helper = Helper()
Config().set_home(self.temp_dir)
def teardown_method(self):
pass
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
time.sleep(1)
@self.project_command.cli_helper.input("y\n\n\n\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.environment_command = EnvironmentCommand(self.cli_helper)
def test_environment_setup_parameter(self):
self.__set_variables()
# Setup the environement by passing name
definition_filepath = os.path.join(
self.project_command.project_controller.environment_driver.
environment_directory_path, "Dockerfile")
# Test pass with correct input
test_framework = "data-analytics"
test_type = "cpu"
test_language = "py27"
self.environment_command.parse([
"environment", "setup", "--framework", test_framework, "--type",
test_type, "--language", test_language
])
result = self.environment_command.execute()
assert result
assert os.path.isfile(definition_filepath)
assert "FROM datmo/data-analytics:cpu-py27" in open(
definition_filepath, "r").read()
# Test fail with wrong framework input
test_framework = "random"
self.environment_command.parse([
"environment", "setup", "--type", test_type, "--framework",
test_framework, "--language", test_language
])
@self.environment_command.cli_helper.input("\n\n")
def dummy(self):
return self.environment_command.execute()
result = dummy(self)
assert result
assert os.path.isfile(definition_filepath)
assert "FROM datmo/python-base:cpu-py27" in open(
definition_filepath, "r").read()
# Test fail with wrong framework input and wrong language input
test_framework = "wrong_name"
test_type = "cpu"
test_language = "wrong_language"
self.environment_command.parse([
"environment", "setup", "--framework", test_framework, "--type",
test_type, "--language", test_language
])
failed = False
try:
self.environment_command.execute()
except ValueError:
failed = True
assert failed
def test_environment_setup_prompt(self):
self.__set_variables()
# Setup the environement by passing name
definition_filepath = os.path.join(
self.project_command.project_controller.environment_driver.
environment_directory_path, "Dockerfile")
# Test success with correct prompt input using numbers
self.environment_command.parse(["environment", "setup"])
@self.environment_command.cli_helper.input(
"cpu\ndata-analytics\npy27\n")
def dummy(self):
return self.environment_command.execute()
result = dummy(self)
assert result
assert os.path.isfile(definition_filepath)
assert "FROM datmo/data-analytics:cpu-py27" in open(
definition_filepath, "r").read()
# Test success with correct prompt input using numbers
self.environment_command.parse(["environment", "setup"])
@self.environment_command.cli_helper.input("cpu\ncaffe2\n")
def dummy(self):
return self.environment_command.execute()
result = dummy(self)
assert result
assert os.path.isfile(definition_filepath)
assert "FROM datmo/caffe2:cpu" in open(definition_filepath, "r").read()
# Test success with correct prompt input using string
self.environment_command.parse(["environment", "setup"])
@self.environment_command.cli_helper.input("1\n1\n1\n")
def dummy(self):
return self.environment_command.execute()
result = dummy(self)
assert result
assert os.path.isfile(definition_filepath)
assert "FROM datmo" in open(definition_filepath, "r").read()
# Test with prompt input number out of range
self.environment_command.parse(["environment", "setup"])
@self.environment_command.cli_helper.input("-1\n\n\n")
def dummy(self):
return self.environment_command.execute()
result = dummy(self)
assert result
assert os.path.isfile(definition_filepath)
assert "FROM datmo/python-base:cpu-py27" in open(
definition_filepath, "r").read()
# Test with prompt input string incorrect
self.environment_command.parse(["environment", "setup"])
# quit at environment type
@self.environment_command.cli_helper.input("quit\n")
def dummy(self):
return self.environment_command.execute()
with pytest.raises(SystemExit) as pytest_wrapped_e:
dummy(self)
assert pytest_wrapped_e.type == SystemExit
assert pytest_wrapped_e.value.code == 0
# quit at environment framework
@self.environment_command.cli_helper.input("\nquit\n")
def dummy(self):
return self.environment_command.execute()
with pytest.raises(SystemExit) as pytest_wrapped_e:
dummy(self)
assert pytest_wrapped_e.type == SystemExit
assert pytest_wrapped_e.value.code == 0
# quit at environment language
@self.environment_command.cli_helper.input("\n\nquit\n")
def dummy(self):
return self.environment_command.execute()
with pytest.raises(SystemExit) as pytest_wrapped_e:
dummy(self)
assert pytest_wrapped_e.type == SystemExit
assert pytest_wrapped_e.value.code == 0
def test_environment_create(self):
# 1) Environment definition file in project environment directory (with name / description)
# 2) Environment definition file passed as an option
# 3) Environment definition file in root project folder
# 4) Environment definition file in root project folder (should return the same environment)
# 5) No environment definition file present
# 6) No environment definition file present (should return the same environment)
self.__set_variables()
# Test option 1
# Create environment definition in project environment directory
definition_filepath = os.path.join(
self.project_command.project_controller.environment_driver.
environment_directory_path, "Dockerfile")
random_text = str(uuid.uuid1())
with open(definition_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine" + "\n"))
f.write(to_bytes(str("RUN echo " + random_text)))
self.environment_command.parse([
"environment", "create", "--name", "test", "--description",
"test description"
])
result = self.environment_command.execute()
assert result
assert result.name == "test"
assert result.description == "test description"
# remove environment directory path
shutil.rmtree(self.project_command.project_controller.
environment_driver.environment_directory_path)
# Test option 2
self.__set_variables()
random_dir = os.path.join(self.temp_dir, "random_datmo_dir")
os.makedirs(random_dir)
definition_filepath = os.path.join(random_dir, "Dockerfile")
random_text = str(uuid.uuid1())
with open(definition_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine" + "\n"))
f.write(to_bytes(str("RUN echo " + random_text)))
self.environment_command.parse(
["environment", "create", "--paths", definition_filepath])
result = self.environment_command.execute()
assert result
# remove directory with Dockerfile
shutil.rmtree(random_dir)
# Test option 3
definition_filepath = os.path.join(self.temp_dir, "Dockerfile")
random_text = str(uuid.uuid1())
with open(definition_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine" + "\n"))
f.write(to_bytes(str("RUN echo " + random_text)))
self.environment_command.parse(["environment", "create"])
result = self.environment_command.execute()
assert result
# Test option 4
self.environment_command.parse(["environment", "create"])
result_2 = self.environment_command.execute()
assert result == result_2
os.remove(definition_filepath)
# Test option 5
self.environment_command.parse(["environment", "create"])
result = self.environment_command.execute()
assert result
# Test option 6
self.environment_command.parse(["environment", "create"])
result_2 = self.environment_command.execute()
assert result == result_2
def test_environment_update(self):
self.__set_variables()
self.environment_command.parse(["environment", "create"])
environment_obj = self.environment_command.execute()
# Test successful update (none given)
self.environment_command.parse(
["environment", "update", environment_obj.id])
result = self.environment_command.execute()
assert result
assert result.name
assert result.description
# Test successful update (name and description given)
new_name = "test name"
new_description = "test description"
self.environment_command.parse([
"environment", "update", environment_obj.id, "--name", new_name,
"--description", new_description
])
result = self.environment_command.execute()
assert result
assert result.name == new_name
assert result.description == new_description
# Test failed update (passing up error from controller)
failed = False
try:
self.environment_command.parse(
["environment", "update", "random_id"])
self.environment_command.execute()
except EnvironmentDoesNotExist:
failed = True
assert failed
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_environment_delete(self):
self.__set_variables()
self.environment_command.parse(["environment", "create"])
environment_obj = self.environment_command.execute()
self.environment_command.parse(
["environment", "delete", environment_obj.id])
result = self.environment_command.execute()
assert result
def test_environment_ls(self):
self.__set_variables()
self.environment_command.parse(["environment", "create"])
created_environment_obj = self.environment_command.execute()
# Test success (defaults)
self.environment_command.parse(["environment", "ls"])
environment_objs = self.environment_command.execute()
assert created_environment_obj in environment_objs
# Test failure (format)
failed = False
try:
self.environment_command.parse(["environment", "ls", "--format"])
except ArgumentError:
failed = True
assert failed
# Test success format csv
self.environment_command.parse(
["environment", "ls", "--format", "csv"])
environment_objs = self.environment_command.execute()
assert created_environment_obj in environment_objs
# Test success format csv, download default
self.environment_command.parse(
["environment", "ls", "--format", "csv", "--download"])
environment_objs = self.environment_command.execute()
assert created_environment_obj in environment_objs
test_wildcard = os.path.join(
self.project_command.project_controller.home, "environment_ls_*")
paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
assert paths
assert open(paths[0], "r").read()
os.remove(paths[0])
# Test success format csv, download exact path
test_path = os.path.join(self.temp_dir, "my_output")
self.environment_command.parse([
"environment", "ls", "--format", "csv", "--download",
"--download-path", test_path
])
environment_objs = self.environment_command.execute()
assert created_environment_obj in environment_objs
assert os.path.isfile(test_path)
assert open(test_path, "r").read()
os.remove(test_path)
# Test success format table
self.environment_command.parse(["environment", "ls"])
environment_objs = self.environment_command.execute()
assert created_environment_obj in environment_objs
# Test success format table, download default
self.environment_command.parse(["environment", "ls", "--download"])
environment_objs = self.environment_command.execute()
assert created_environment_obj in environment_objs
test_wildcard = os.path.join(
self.project_command.project_controller.home, "environment_ls_*")
paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
assert paths
assert open(paths[0], "r").read()
os.remove(paths[0])
# Test success format table, download exact path
test_path = os.path.join(self.temp_dir, "my_output")
self.environment_command.parse(
["environment", "ls", "--download", "--download-path", test_path])
environment_objs = self.environment_command.execute()
assert created_environment_obj in environment_objs
assert os.path.isfile(test_path)
assert open(test_path, "r").read()
os.remove(test_path)
class TestSnapshot():
def setup_class(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system() == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.cli_helper = Helper()
def teardown_class(self):
pass
def __set_variables(self):
self.init = ProjectCommand(self.temp_dir, self.cli_helper)
self.init.parse([
"init",
"--name", "foobar",
"--description", "test model"])
self.init.execute()
self.snapshot = SnapshotCommand(self.temp_dir, self.cli_helper)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir,
"Dockerfile")
with open(self.env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
# Create config
self.config_filepath = os.path.join(self.snapshot.home,
"config.json")
with open(self.config_filepath, "w") as f:
f.write(to_unicode(str("{}")))
# Create stats
self.stats_filepath = os.path.join(self.snapshot.home,
"stats.json")
with open(self.stats_filepath, "w") as f:
f.write(to_unicode(str("{}")))
# Create test file
self.filepath = os.path.join(self.snapshot.home,
"file.txt")
with open(self.filepath, "w") as f:
f.write(to_unicode(str("test")))
def test_snapshot_project_not_init(self):
failed = False
try:
self.snapshot = SnapshotCommand(self.temp_dir, self.cli_helper)
except ProjectNotInitializedException:
failed = True
assert failed
def test_datmo_snapshot_create(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_session_id = "test_session_id"
test_task_id = "test_task_id"
test_code_id = "test_code_id"
test_environment_def_path = self.env_def_path
test_config_filepath = self.config_filepath
test_stats_filepath = self.config_filepath
test_filepaths = [self.filepath]
self.snapshot.parse([
"snapshot",
"create",
"--message", test_message,
"--label", test_label,
"--session-id", test_session_id,
"--task-id", test_task_id,
"--code-id", test_code_id,
"--environment-def-path", test_environment_def_path,
"--config-filepath", test_config_filepath,
"--stats-filepath", test_stats_filepath,
"--filepaths", test_filepaths[0],
])
# test for desired side effects
assert self.snapshot.args.message == test_message
assert self.snapshot.args.label == test_label
assert self.snapshot.args.session_id == test_session_id
assert self.snapshot.args.task_id == test_task_id
assert self.snapshot.args.code_id == test_code_id
assert self.snapshot.args.environment_def_path == test_environment_def_path
assert self.snapshot.args.config_filepath == test_config_filepath
assert self.snapshot.args.stats_filepath == test_stats_filepath
assert self.snapshot.args.filepaths == test_filepaths
snapshot_id_1 = self.snapshot.execute()
assert snapshot_id_1
def test_datmo_snapshot_create_fail_mutually_exclusive_args(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_session_id = "test_session_id"
test_task_id = "test_task_id"
test_code_id = "test_code_id"
test_commit_id = "test_commit_id"
test_environment_id = "test_environment_id"
test_environment_def_path = self.env_def_path
test_file_collection_id = "test_file_collection_id"
test_filepaths = [self.filepath]
test_config_filename = "config.json"
test_config_filepath = self.config_filepath
test_stats_filename = "stats.json"
test_stats_filepath = self.config_filepath
# Code exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message", test_message,
"--label", test_label,
"--session-id", test_session_id,
"--task-id", test_task_id,
"--code-id", test_code_id,
"--commit-id", test_commit_id
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Environment exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message", test_message,
"--label", test_label,
"--session-id", test_session_id,
"--task-id", test_task_id,
"--environment-id", test_environment_id,
"--environment-def-path", test_environment_def_path,
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# File exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message", test_message,
"--label", test_label,
"--session-id", test_session_id,
"--task-id", test_task_id,
"--file-collection-id", test_file_collection_id,
"--filepaths", test_filepaths[0],
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Config exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message", test_message,
"--label", test_label,
"--session-id", test_session_id,
"--task-id", test_task_id,
"--config-filename", test_config_filename,
"--config-filepath", test_config_filepath,
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Stats exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message", test_message,
"--label", test_label,
"--session-id", test_session_id,
"--task-id", test_task_id,
"--stats-filename", test_stats_filename,
"--stats-filepath", test_stats_filepath,
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_create_default(self):
self.__set_variables()
self.snapshot.parse([
"snapshot",
"create",
"-m", "my test snapshot"
])
snapshot_id_2 = self.snapshot.execute()
assert snapshot_id_2
def test_datmo_snapshot_create_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create"
"--foobar","foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_delete(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot.parse([
"snapshot",
"create",
"-m", "my test snapshot"
])
snapshot_id = self.snapshot.execute()
# Test when optional parameters are not given
self.snapshot.parse([
"snapshot",
"delete",
"--id", snapshot_id
])
result = self.snapshot.execute()
assert result
def test_datmo_snapshot_delete_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"delete"
"--foobar","foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_ls(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot.parse([
"snapshot",
"create",
"-m", "my test snapshot"
])
self.snapshot.execute()
# Test when optional parameters are not given
self.snapshot.parse([
"snapshot",
"ls"
])
result = self.snapshot.execute()
assert result
# Test when optional parameters are not given
self.snapshot.parse([
"snapshot",
"ls", "-a"
])
result = self.snapshot.execute()
assert result
def test_datmo_snapshot_checkout_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"checkout"
"--foobar","foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_checkout(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot.parse([
"snapshot",
"create",
"-m", "my test snapshot"
])
snapshot_id = self.snapshot.execute()
# Test when optional parameters are not given
self.snapshot.parse([
"snapshot",
"checkout",
"--id", snapshot_id
])
result = self.snapshot.execute()
assert result
class TestProjectCommand():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.cli_helper = Helper()
self.project_command = ProjectCommand(self.cli_helper)
def teardown_method(self):
pass
def test_init_create_success_default_name_no_description_no_environment(
self):
self.project_command.parse(["init"])
# Test when environment is created
@self.project_command.cli_helper.input("\n\ny\n\n\n\n")
def dummy(self):
return self.project_command.execute()
result = dummy(self)
# Ensure that the name and description are current
_, default_name = os.path.split(
self.project_command.project_controller.home)
assert result
assert result.name == default_name
assert result.description == None
# Ensure environment is correct
definition_filepath = os.path.join(
self.temp_dir, self.project_command.project_controller.file_driver.
environment_directory, "Dockerfile")
assert os.path.isfile(definition_filepath)
assert "FROM datmo/python-base:cpu-py27" in open(
definition_filepath, "r").read()
def test_init_create_success_no_environment(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
# Test when environment is not created
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
result = dummy(self)
definition_filepath = os.path.join(
self.temp_dir, self.project_command.project_controller.file_driver.
environment_directory, "Dockerfile")
assert result
assert not os.path.isfile(definition_filepath)
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result.name == test_name
assert result.description == test_description
def test_init_create_success_environment(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
# Test when environment is created
@self.project_command.cli_helper.input("y\n\n\n\n")
def dummy(self):
return self.project_command.execute()
result = dummy(self)
definition_filepath = os.path.join(
self.temp_dir, self.project_command.project_controller.file_driver.
environment_directory, "Dockerfile")
assert result
assert os.path.isfile(definition_filepath)
assert "FROM datmo/python-base:cpu-py27" in open(
definition_filepath, "r").read()
# test for desired side effects
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result.name == test_name
assert result.description == test_description
def test_init_create_success_blank(self):
self.project_command.parse(["init", "--name", "", "--description", ""])
# test if prompt opens
@self.project_command.cli_helper.input("\n\n\n")
def dummy(self):
return self.project_command.execute()
result = dummy(self)
assert result
assert result.name
assert not result.description
def test_init_update_success(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
result_1 = dummy(self)
updated_name = "foobar2"
updated_description = "test model 2"
self.project_command.parse([
"init", "--name", updated_name, "--description",
updated_description
])
result_2 = dummy(self)
# test for desired side effects
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result_2.id == result_1.id
assert result_2.name == updated_name
assert result_2.description == updated_description
def test_init_update_success_only_name(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
result_1 = dummy(self)
updated_name = "foobar2"
self.project_command.parse(
["init", "--name", updated_name, "--description", ""])
@self.project_command.cli_helper.input("\n\n")
def dummy(self):
return self.project_command.execute()
result_2 = dummy(self)
# test for desired side effects
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result_2.id == result_1.id
assert result_2.name == updated_name
assert result_2.description == result_1.description
def test_init_update_success_only_description(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
result_1 = dummy(self)
updated_description = "test model 2"
self.project_command.parse(
["init", "--name", "", "--description", updated_description])
@self.project_command.cli_helper.input("\n\n")
def dummy(self):
return self.project_command.execute()
result_2 = dummy(self)
# test for desired side effects
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result_2.id == result_1.id
assert result_2.name == result_1.name
assert result_2.description == updated_description
def test_init_invalid_arg(self):
exception_thrown = False
try:
self.project_command.parse(["init", "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_version(self):
self.project_command.parse(["version"])
result = self.project_command.execute()
# test for desired side effects
assert __version__ in result
def test_version_invalid_arg(self):
exception_thrown = False
try:
self.project_command.parse(["version", "--foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_status_basic(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
_ = dummy(self)
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = result
assert isinstance(status_dict, dict)
assert not current_snapshot
assert not latest_snapshot_user_generated
assert not latest_snapshot_auto_generated
assert unstaged_code
assert not unstaged_environment
assert not unstaged_files
def test_status_user_generated_snapshot(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
_ = dummy(self)
# Create a snapshot
self.snapshot_command = SnapshotCommand(self.cli_helper)
with open(os.path.join(self.project_command.home, "test.py"),
"wb") as f:
f.write(to_bytes(str("import xgboost")))
self.snapshot_command.parse(
["snapshot", "create", "--message", "test"])
snapshot_obj = self.snapshot_command.execute()
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = result
assert isinstance(status_dict, dict)
assert isinstance(current_snapshot, Snapshot)
assert isinstance(latest_snapshot_user_generated, Snapshot)
assert snapshot_obj == latest_snapshot_user_generated
assert current_snapshot == latest_snapshot_user_generated
assert not latest_snapshot_auto_generated
assert not unstaged_code
assert not unstaged_environment
assert not unstaged_files
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_status_autogenerated_snapshot(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
_ = dummy(self)
self.run_command = RunCommand(self.cli_helper)
self.task_controller = TaskController()
# Create and run a task and test if task is shown
test_command = ["sh", "-c", "echo accuracy:0.45"]
self.run_command.parse(["run", test_command])
updated_first_task = self.run_command.execute()
before_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_first_task.before_snapshot_id)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_first_task.after_snapshot_id)
before_environment_obj = self.task_controller.dal.environment.get_by_id(
before_snapshot_obj.environment_id)
after_environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
assert before_environment_obj == after_environment_obj
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
result
assert status_dict
assert isinstance(status_dict, dict)
assert status_dict['name'] == test_name
assert status_dict['description'] == test_description
assert isinstance(status_dict['config'], dict)
assert isinstance(current_snapshot, Snapshot)
assert not latest_snapshot_user_generated
assert isinstance(latest_snapshot_auto_generated, Snapshot)
# current snapshot is the before snapshot for the run
assert current_snapshot == before_snapshot_obj
assert current_snapshot != latest_snapshot_auto_generated
# latest autogenerated snapshot is the after snapshot id
assert latest_snapshot_auto_generated == after_snapshot_obj
# autogenerated snapshot is after the user generated snapshot
assert not unstaged_code
assert not unstaged_environment
assert not unstaged_files
# Create a snapshot
self.snapshot_command = SnapshotCommand(self.cli_helper)
with open(os.path.join(self.project_command.home, "test.py"),
"wb") as f:
f.write(to_bytes(str("import xgboost")))
self.snapshot_command.parse(
["snapshot", "create", "--message", "test"])
first_snapshot = self.snapshot_command.execute()
# Create and run a task and test if task is shown
test_command = ["sh", "-c", "echo accuracy:0.45"]
self.run_command.parse(["run", test_command])
updated_second_task = self.run_command.execute()
before_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_second_task.before_snapshot_id)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_second_task.after_snapshot_id)
before_environment_obj = self.task_controller.dal.environment.get_by_id(
before_snapshot_obj.environment_id)
after_environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
assert before_environment_obj == after_environment_obj
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
result
assert status_dict
assert isinstance(status_dict, dict)
assert status_dict['name'] == test_name
assert status_dict['description'] == test_description
assert isinstance(status_dict['config'], dict)
assert isinstance(current_snapshot, Snapshot)
assert isinstance(latest_snapshot_user_generated, Snapshot)
assert latest_snapshot_user_generated == first_snapshot
assert isinstance(latest_snapshot_auto_generated, Snapshot)
# current snapshot is the before snapshot for the run
assert current_snapshot == before_snapshot_obj
assert current_snapshot == latest_snapshot_user_generated
assert current_snapshot != latest_snapshot_auto_generated
# latest autogenerated snapshot is the after snapshot id
assert latest_snapshot_auto_generated == after_snapshot_obj
assert latest_snapshot_auto_generated != latest_snapshot_user_generated
# user generated snapshot is not associated with any before or after snapshot
assert latest_snapshot_user_generated == before_snapshot_obj
assert latest_snapshot_user_generated != after_snapshot_obj
# autogenerated snapshot is after the user generated snapshot
assert latest_snapshot_auto_generated.created_at > latest_snapshot_user_generated.created_at
assert not unstaged_code
assert not unstaged_environment
assert not unstaged_files
# Check if the same snapshot is given back when created
self.snapshot_command.parse(
["snapshot", "create", "--message", "test"])
new_snapshot = self.snapshot_command.execute()
assert current_snapshot == new_snapshot
# Create a user generated snapshot after some changes
with open(os.path.join(self.project_command.home, "new.py"),
"wb") as f:
f.write(to_bytes(str("import xgboost")))
self.snapshot_command.parse(
["snapshot", "create", "--message", "test"])
new_snapshot = self.snapshot_command.execute()
# ensure we created a new snapshot
assert current_snapshot != new_snapshot
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
result
assert status_dict
assert isinstance(status_dict, dict)
assert status_dict['name'] == test_name
assert status_dict['description'] == test_description
assert isinstance(status_dict['config'], dict)
assert isinstance(current_snapshot, Snapshot)
assert isinstance(latest_snapshot_user_generated, Snapshot)
# current snapshot is the latest user generated one
assert current_snapshot == latest_snapshot_user_generated
# latest autogenerated snapshot is not the user generated one
assert latest_snapshot_auto_generated != latest_snapshot_user_generated
# autogenerated snapshot is after the user generated snapshot
assert latest_snapshot_user_generated.created_at > latest_snapshot_auto_generated.created_at
assert not unstaged_code
assert not unstaged_environment
assert not unstaged_files
# Create a snapshot from the auto generated one and ensure they are the same
self.run_command = RunCommand(self.cli_helper)
self.task_controller = TaskController()
test_command = ["sh", "-c", "echo accuracy:0.45"]
self.run_command.parse(["run", test_command])
updated_third_task = self.run_command.execute()
before_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_third_task.before_snapshot_id)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_third_task.after_snapshot_id)
self.snapshot_command.parse([
"snapshot", "create", "--run-id", updated_third_task.id,
"--message", "test"
])
converted_snapshot = self.snapshot_command.execute()
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
result
assert status_dict
assert isinstance(status_dict, dict)
assert status_dict['name'] == test_name
assert status_dict['description'] == test_description
assert isinstance(status_dict['config'], dict)
assert isinstance(current_snapshot, Snapshot)
assert isinstance(latest_snapshot_user_generated, Snapshot)
assert isinstance(latest_snapshot_auto_generated, Snapshot)
# current snapshot is the before snapshot for the run
assert current_snapshot == before_snapshot_obj
assert current_snapshot != latest_snapshot_user_generated
# latest user generated snapshot is the same as that created by the run
assert converted_snapshot == latest_snapshot_user_generated
assert latest_snapshot_user_generated == after_snapshot_obj
# latest user generated converted the latest task so latest generate is from the last task
assert latest_snapshot_user_generated.created_at > latest_snapshot_auto_generated.created_at
assert not unstaged_code
assert not unstaged_environment
assert not unstaged_files
def test_status_invalid_arg(self):
exception_thrown = False
try:
self.project_command.parse(["status", "--foobar"])
except UnrecognizedCLIArgument:
exception_thrown = True
assert exception_thrown
def test_cleanup(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
_ = dummy(self)
self.project_command.parse(["cleanup"])
@self.project_command.cli_helper.input("y\n\n")
def dummy(self):
return self.project_command.execute()
result = dummy(self)
assert not os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert isinstance(result, bool)
assert result
def test_cleanup_invalid_arg(self):
exception_thrown = False
try:
self.project_command.parse(["cleanup", "--foobar"])
except UnrecognizedCLIArgument:
exception_thrown = True
assert exception_thrown
class TestProjectCommand():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.cli_helper = Helper()
self.project_command = ProjectCommand(self.cli_helper)
def teardown_method(self):
pass
def test_init_create_success_default_name_no_description_no_environment(
self):
self.project_command.parse(["init"])
# Test when environment is created
@self.project_command.cli_helper.input("\n\ny\n\n\n\n")
def dummy(self):
return self.project_command.execute()
result = dummy(self)
# Ensure that the name and description are current
_, default_name = os.path.split(
self.project_command.project_controller.home)
assert result
assert result.name == default_name
assert result.description == None
# Ensure environment is correct
definition_filepath = os.path.join(
self.temp_dir, self.project_command.project_controller.file_driver.
environment_directory, "Dockerfile")
assert os.path.isfile(definition_filepath)
assert "FROM datmo/python-base:cpu-py27" in open(
definition_filepath, "r").read()
def test_init_create_success_no_environment(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
# Test when environment is not created
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
result = dummy(self)
definition_filepath = os.path.join(
self.temp_dir, self.project_command.project_controller.file_driver.
environment_directory, "Dockerfile")
assert result
assert not os.path.isfile(definition_filepath)
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result.name == test_name
assert result.description == test_description
def test_init_create_success_environment(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
# Test when environment is created
@self.project_command.cli_helper.input("y\n\n\n\n")
def dummy(self):
return self.project_command.execute()
result = dummy(self)
definition_filepath = os.path.join(
self.temp_dir, self.project_command.project_controller.file_driver.
environment_directory, "Dockerfile")
assert result
assert os.path.isfile(definition_filepath)
assert "FROM datmo/python-base:cpu-py27" in open(
definition_filepath, "r").read()
# test for desired side effects
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result.name == test_name
assert result.description == test_description
def test_init_create_success_blank(self):
self.project_command.parse(["init", "--name", "", "--description", ""])
# test if prompt opens
@self.project_command.cli_helper.input("\n\n\n")
def dummy(self):
return self.project_command.execute()
result = dummy(self)
assert result
assert result.name
assert not result.description
def test_init_update_success(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
result_1 = dummy(self)
updated_name = "foobar2"
updated_description = "test model 2"
self.project_command.parse([
"init", "--name", updated_name, "--description",
updated_description
])
result_2 = dummy(self)
# test for desired side effects
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result_2.id == result_1.id
assert result_2.name == updated_name
assert result_2.description == updated_description
def test_init_update_success_only_name(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
result_1 = dummy(self)
updated_name = "foobar2"
self.project_command.parse(
["init", "--name", updated_name, "--description", ""])
@self.project_command.cli_helper.input("\n\n")
def dummy(self):
return self.project_command.execute()
result_2 = dummy(self)
# test for desired side effects
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result_2.id == result_1.id
assert result_2.name == updated_name
assert result_2.description == result_1.description
def test_init_update_success_only_description(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
result_1 = dummy(self)
updated_description = "test model 2"
self.project_command.parse(
["init", "--name", "", "--description", updated_description])
@self.project_command.cli_helper.input("\n\n")
def dummy(self):
return self.project_command.execute()
result_2 = dummy(self)
# test for desired side effects
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result_2.id == result_1.id
assert result_2.name == result_1.name
assert result_2.description == updated_description
def test_init_invalid_arg(self):
exception_thrown = False
try:
self.project_command.parse(["init", "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_version(self):
self.project_command.parse(["version"])
result = self.project_command.execute()
# test for desired side effects
assert __version__ in result
def test_version_invalid_arg(self):
exception_thrown = False
try:
self.project_command.parse(["version", "--foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_status(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
_ = dummy(self)
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = result
assert isinstance(status_dict, dict)
assert not latest_snapshot_user_generated
assert not latest_snapshot_auto_generated
assert unstaged_code
assert not unstaged_environment
assert not unstaged_files
def test_status_invalid_arg(self):
exception_thrown = False
try:
self.project_command.parse(["status", "--foobar"])
except UnrecognizedCLIArgument:
exception_thrown = True
assert exception_thrown
def test_cleanup(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
_ = dummy(self)
self.project_command.parse(["cleanup"])
@self.project_command.cli_helper.input("y\n\n")
def dummy(self):
return self.project_command.execute()
result = dummy(self)
assert not os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert isinstance(result, bool)
assert result
def test_cleanup_invalid_arg(self):
exception_thrown = False
try:
self.project_command.parse(["cleanup", "--foobar"])
except UnrecognizedCLIArgument:
exception_thrown = True
assert exception_thrown
class TestTaskCommand():
def setup_class(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.cli_helper = Helper()
def teardown_class(self):
pass
def __set_variables(self):
self.project = ProjectCommand(self.temp_dir, self.cli_helper)
self.project.parse(
["init", "--name", "foobar", "--description", "test model"])
self.project.execute()
self.task = TaskCommand(self.temp_dir, self.cli_helper)
# Create environment_driver definition
env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
def test_task_project_not_init(self):
failed = False
try:
self.task = TaskCommand(self.temp_dir, self.cli_helper)
except ProjectNotInitializedException:
failed = True
assert failed
def test_snapshot_command(self):
self.__set_variables()
self.task.parse(["task"])
assert self.task.execute()
def test_task_run_should_fail1(self):
self.__set_variables()
# Test failure case
self.task.parse(["task", "run"])
result = self.task.execute()
assert not result
def test_task_run(self):
# TODO: Adding test with `--interactive` argument and terminate inside container
self.__set_variables()
# Test failure command execute
test_command = ["yo", "yo"]
self.task.parse(["task", "run", test_command])
result = self.task.execute()
assert result
# Test success case
test_command = ["sh", "-c", "echo accuracy:0.45"]
test_ports = ["8888:8888", "9999:9999"]
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
# test for single set of ports
self.task.parse([
"task", "run", "--ports", test_ports[0], "--environment-def",
test_dockerfile, test_command
])
# test for desired side effects
assert self.task.args.cmd == test_command
assert self.task.args.ports == [test_ports[0]]
assert self.task.args.environment_definition_filepath == test_dockerfile
self.task.parse([
"task", "run", "-p", test_ports[0], "-p", test_ports[1],
"--environment-def", test_dockerfile, test_command
])
# test for desired side effects
assert self.task.args.cmd == test_command
assert self.task.args.ports == test_ports
assert self.task.args.environment_definition_filepath == test_dockerfile
# test proper execution of task run command
result = self.task.execute()
time.sleep(1)
assert result
assert isinstance(result, CoreTask)
assert result.logs
assert "accuracy" in result.logs
assert result.results
assert result.results == {"accuracy": "0.45"}
assert result.status == "SUCCESS"
def test_task_run_string_command(self):
# TODO: Adding test with `--interactive` argument and terminate inside container
self.__set_variables()
# Test success case
test_command = "sh -c 'echo accuracy:0.45'"
test_ports = ["8888:8888", "9999:9999"]
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.task.parse([
"task", "run", "--ports", test_ports[0], "--ports", test_ports[1],
"--environment-def", test_dockerfile, test_command
])
# test for desired side effects
assert self.task.args.cmd == test_command
assert self.task.args.ports == test_ports
assert self.task.args.environment_definition_filepath == test_dockerfile
# test proper execution of task run command
result = self.task.execute()
assert result
assert isinstance(result, CoreTask)
assert result.logs
assert "accuracy" in result.logs
assert result.results
assert result.results == {"accuracy": "0.45"}
assert result.status == "SUCCESS"
# def test_multiple_concurrent_task_run_command(self):
# test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
# test_command = ["sh", "-c", "echo accuracy:0.45"]
# manager = Manager()
# return_dict = manager.dict()
#
# def task_exec_func(procnum, return_dict):
# print("Creating Task object")
# task = TaskCommand(self.temp_dir, self.cli_helper)
# print("Parsing command")
# task.parse(
# ["task", "run", "--environment-def", test_dockerfile, test_command])
# print("Executing command")
# result = task.execute()
# return_dict[procnum] = result
#
# self.__set_variables()
# test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
#
# # Run all three tasks in parallel
# jobs = []
# number_tasks = 3
# for i in range(number_tasks):
# p = Process(target=task_exec_func, args=(i, return_dict))
# jobs.append(p)
# p.start()
#
# # Join
# for proc in jobs:
# proc.join()
#
# results = return_dict.values()
# assert len(results) == number_tasks
# for result in results:
# assert result
# assert isinstance(result, CoreTask)
# assert result.logs
# assert "accuracy" in result.logs
# assert result.results
# assert result.results == {"accuracy": "0.45"}
# assert result.status == "SUCCESS"
def test_task_run_notebook(self):
self.__set_variables()
# Test success case
test_command = ["jupyter", "notebook", "list"]
test_ports = ["8888:8888", "9999:9999"]
# test single ports option before command
self.task.parse(
["task", "run", "--ports", test_ports[0], test_command])
# test for desired side effects
assert self.task.args.cmd == test_command
assert self.task.args.ports == [test_ports[0]]
# test multiple ports option before command
self.task.parse([
"task", "run", "--ports", test_ports[0], "--ports", test_ports[1],
test_command
])
# test for desired side effects
assert self.task.args.cmd == test_command
assert self.task.args.ports == test_ports
# test proper execution of task run command
result = self.task.execute()
assert result
assert isinstance(result, CoreTask)
assert result.logs
assert "Currently running servers" in result.logs
assert result.status == "SUCCESS"
def test_task_run_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.task.parse(["task", "run" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_task_ls(self):
self.__set_variables()
self.task.parse(["task", "ls"])
task_ls_command = self.task.execute()
assert task_ls_command == True
test_session_id = 'test_session_id'
self.task.parse(["task", "ls", "--session-id", test_session_id])
# test for desired side effects
assert self.task.args.session_id == test_session_id
task_ls_command = self.task.execute()
assert task_ls_command == True
def test_task_ls_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.task.parse(["task", "ls" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_task_stop_success(self):
# 1) Test stop with task_id
# 2) Test stop with all
self.__set_variables()
test_command = ["sh", "-c", "echo yo"]
test_ports = "8888:8888"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.task.parse([
"task", "run", "--ports", test_ports, "--environment-def",
test_dockerfile, test_command
])
test_task_obj = self.task.execute()
# 1) Test option 1
self.task.parse(["task", "stop", "--id", test_task_obj.id])
# test for desired side effects
assert self.task.args.id == test_task_obj.id
# test when task id is passed to stop it
task_stop_command = self.task.execute()
assert task_stop_command == True
# 2) Test option 2
self.task.parse(["task", "stop", "--all"])
# test when all is passed to stop all
task_stop_command = self.task.execute()
assert task_stop_command == True
def test_task_stop_failure_required_args(self):
self.__set_variables()
# Passing wrong task id
self.task.parse(["task", "stop"])
failed = False
try:
_ = self.task.execute()
except RequiredArgumentMissing:
failed = True
assert failed
def test_task_stop_failure_mutually_exclusive_vars(self):
self.__set_variables()
# Passing wrong task id
self.task.parse(["task", "stop", "--id", "invalid-task-id", "--all"])
failed = False
try:
_ = self.task.execute()
except MutuallyExclusiveArguments:
failed = True
assert failed
def test_task_stop_failure_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.task.parse(["task", "stop" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_task_stop_invalid_task_id(self):
self.__set_variables()
# Passing wrong task id
self.task.parse(["task", "stop", "--id", "invalid-task-id"])
# test when wrong task id is passed to stop it
result = self.task.execute()
assert not result
class TestSession():
def setup_class(self):
# provide mountable tmp directory for docker
tempfile.tempdir = '/tmp'
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.cli_helper = Helper()
def teardown_class(self):
pass
def __set_variables(self):
self.project = ProjectCommand(self.temp_dir, self.cli_helper)
self.project.parse(
["init", "--name", "foobar", "--description", "test model"])
self.project.execute()
self.session_command = SessionCommand(self.temp_dir, self.cli_helper)
def test_session_project_not_init(self):
failed = False
try:
self.snapshot = SessionCommand(self.temp_dir, self.cli_helper)
except ProjectNotInitializedException:
failed = True
assert failed
def test_session_no_subcommand(self):
self.__set_variables()
self.session_command.parse(["session"])
assert self.session_command.execute()
def test_session_create(self):
self.__set_variables()
self.session_command.parse(["session", "create", "--name", "pizza"])
assert self.session_command.execute()
def test_session_select(self):
self.test_session_create()
self.session_command.parse(["session", "select", "--name", "pizza"])
assert self.session_command.execute()
current = 0
for s in self.session_command.session_controller.list():
print("%s - %s" % (s.name, s.current))
if s.current == True:
current = current + 1
assert current == 1
def test_session_ls(self):
self.test_session_create()
self.session_command.parse(["session", "ls"])
assert self.session_command.execute()
def test_session_delete(self):
self.test_session_create()
self.session_command.parse(["session", "delete", "--name", "pizza"])
assert self.session_command.execute()
session_removed = True
for s in self.session_command.session_controller.list():
if s.name == 'pizza':
session_removed = False
assert session_removed
self.session_command.parse(["session", "ls"])
assert self.session_command.execute()
class TestWorkspace():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.cli_helper = Helper()
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.environment_command = EnvironmentCommand(self.cli_helper)
self.workspace_command = WorkspaceCommand(self.cli_helper)
self.run_command = RunCommand(self.cli_helper)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "wb") as f:
f.write(
to_bytes(str("FROM datmo/python-base:cpu-py27%s" %
os.linesep)))
def teardown_method(self):
pass
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_notebook(self):
self.__set_variables()
test_mem_limit = "4g"
# test single ports option before command
self.workspace_command.parse([
"notebook",
"--gpu",
"--environment-paths",
self.env_def_path,
"--mem-limit",
test_mem_limit,
])
# test for desired side effects
assert self.workspace_command.args.gpu == True
assert self.workspace_command.args.environment_paths == [
self.env_def_path
]
assert self.workspace_command.args.mem_limit == test_mem_limit
# test notebook command
self.workspace_command.parse(["notebook"])
assert self.workspace_command.args.gpu == False
@timeout_decorator.timeout(10, use_signals=False)
def timed_run(timed_run_result):
if self.workspace_command.execute():
return timed_run_result
timed_run_result = False
try:
timed_run_result = timed_run(timed_run_result)
except timeout_decorator.timeout_decorator.TimeoutError:
timed_run_result = True
assert timed_run_result
# Stop all running datmo task to not have any containers
self.run_command.parse(["stop", "--all"])
self.run_command.execute()
# creating and passing environment id
random_text = str(uuid.uuid1())
with open(self.env_def_path, "wb") as f:
f.write(
to_bytes(str("FROM datmo/python-base:cpu-py27%s" %
os.linesep)))
f.write(to_bytes(str("RUN echo " + random_text)))
self.environment_command.parse([
"environment", "create", "--name", "test", "--description",
"test description"
])
result = self.environment_command.execute()
# test notebook command
self.workspace_command.parse(
["notebook", "--environment-id", result.id])
@timeout_decorator.timeout(10, use_signals=False)
def timed_run(timed_run_result):
if self.workspace_command.execute():
return timed_run_result
timed_run_result = False
try:
timed_run_result = timed_run(timed_run_result)
except timeout_decorator.timeout_decorator.TimeoutError:
timed_run_result = True
assert timed_run_result
# Stop all running datmo task
self.run_command.parse(["stop", "--all"])
self.run_command.execute()
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_jupyterlab(self):
self.__set_variables()
test_mem_limit = "4g"
# test single ports option before command
self.workspace_command.parse([
"jupyterlab",
"--gpu",
"--environment-paths",
self.env_def_path,
"--mem-limit",
test_mem_limit,
])
# test for desired side effects
assert self.workspace_command.args.gpu == True
assert self.workspace_command.args.environment_paths == [
self.env_def_path
]
assert self.workspace_command.args.mem_limit == test_mem_limit
# test multiple ports option before command
self.workspace_command.parse(["jupyterlab"])
assert self.workspace_command.args.gpu == False
@timeout_decorator.timeout(10, use_signals=False)
def timed_run(timed_run_result):
if self.workspace_command.execute():
return timed_run_result
timed_run_result = False
try:
timed_run_result = timed_run(timed_run_result)
except timeout_decorator.timeout_decorator.TimeoutError:
timed_run_result = True
assert timed_run_result
# Stop all running datmo task to not have any containers
self.run_command.parse(["stop", "--all"])
self.run_command.execute()
# creating and passing environment id
random_text = str(uuid.uuid1())
with open(self.env_def_path, "wb") as f:
f.write(
to_bytes(str("FROM datmo/python-base:cpu-py27%s" %
os.linesep)))
f.write(to_bytes(str("RUN echo " + random_text)))
self.environment_command.parse([
"environment", "create", "--name", "test", "--description",
"test description"
])
result = self.environment_command.execute()
# test notebook command
self.workspace_command.parse(
["jupyterlab", "--environment-id", result.id])
@timeout_decorator.timeout(10, use_signals=False)
def timed_run(timed_run_result):
if self.workspace_command.execute():
return timed_run_result
timed_run_result = False
try:
timed_run_result = timed_run(timed_run_result)
except timeout_decorator.timeout_decorator.TimeoutError:
timed_run_result = True
assert timed_run_result
# Stop all running datmo task
self.run_command.parse(["stop", "--all"])
self.run_command.execute()
# Test doesn't take tty as True for docker
# @pytest_docker_environment_failed_instantiation(test_datmo_dir)
# def test_terminal(self):
# self.__set_variables()
# test_mem_limit = "4g"
# # test single ports option before command
# self.workspace_command.parse([
# "terminal",
# "--gpu",
# "--environment-paths",
# self.env_def_path,
# "--mem-limit",
# test_mem_limit,
# ])
#
# # test for desired side effects
# assert self.workspace_command.args.gpu == True
# assert self.workspace_command.args.environment_paths == [
# self.env_def_path
# ]
# assert self.workspace_command.args.mem_limit == test_mem_limit
#
# # test multiple ports option before command
# self.workspace_command.parse(["terminal"])
#
# assert self.workspace_command.args.gpu == False
# @timeout_decorator.timeout(10, use_signals=False)
# def timed_run(timed_run_result):
# if self.workspace_command.execute():
# return timed_run_result
#
# timed_run_result = False
# timed_run_result = timed_run(timed_run_result)
#
# assert timed_run_result
#
# # Stop all running datmo task
# self.run_command.parse(["stop", "--all"])
# self.run_command.execute()
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_rstudio(self):
self.__set_variables()
# Update environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "wb") as f:
f.write(to_bytes(str("FROM datmo/r-base:cpu%s" % os.linesep)))
test_mem_limit = "4g"
# test single ports option before command
self.workspace_command.parse([
"rstudio",
"--environment-paths",
self.env_def_path,
"--mem-limit",
test_mem_limit,
])
# test for desired side effects
assert self.workspace_command.args.environment_paths == [
self.env_def_path
]
assert self.workspace_command.args.mem_limit == test_mem_limit
# test multiple ports option before command
self.workspace_command.parse(["rstudio"])
@timeout_decorator.timeout(10, use_signals=False)
def timed_run(timed_run_result):
if self.workspace_command.execute():
return timed_run_result
timed_run_result = False
try:
timed_run_result = timed_run(timed_run_result)
except timeout_decorator.timeout_decorator.TimeoutError:
timed_run_result = True
assert timed_run_result
# Stop all running datmo task to not have any containers
self.run_command.parse(["stop", "--all"])
self.run_command.execute()
# creating and passing environment id
random_text = str(uuid.uuid1())
with open(self.env_def_path, "wb") as f:
f.write(to_bytes(str("FROM datmo/r-base:cpu%s" % os.linesep)))
f.write(to_bytes(str("RUN echo " + random_text)))
self.environment_command.parse([
"environment", "create", "--name", "test", "--description",
"test description"
])
result = self.environment_command.execute()
# test notebook command
self.workspace_command.parse(
["rstudio", "--environment-id", result.id])
@timeout_decorator.timeout(10, use_signals=False)
def timed_run(timed_run_result):
if self.workspace_command.execute():
return timed_run_result
timed_run_result = False
try:
timed_run_result = timed_run(timed_run_result)
except timeout_decorator.timeout_decorator.TimeoutError:
timed_run_result = True
assert timed_run_result
# Stop all running datmo task
self.run_command.parse(["stop", "--all"])
self.run_command.execute()
class TestSnapshot():
def setup_class(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.cli_helper = Helper()
def teardown_class(self):
pass
def __set_variables(self):
self.project = ProjectCommand(self.temp_dir, self.cli_helper)
self.project.parse(
["init", "--name", "foobar", "--description", "test model"])
self.project.execute()
self.snapshot = SnapshotCommand(self.temp_dir, self.cli_helper)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
# Create config
self.config_filepath = os.path.join(self.snapshot.home, "config.json")
with open(self.config_filepath, "w") as f:
f.write(to_unicode(str("{}")))
# Create stats
self.stats_filepath = os.path.join(self.snapshot.home, "stats.json")
with open(self.stats_filepath, "w") as f:
f.write(to_unicode(str("{}")))
# Create test file
self.filepath = os.path.join(self.snapshot.home, "file.txt")
with open(self.filepath, "w") as f:
f.write(to_unicode(str("test")))
# Create another test file
self.filepath_2 = os.path.join(self.snapshot.home, "file2.txt")
with open(self.filepath_2, "w") as f:
f.write(to_unicode(str("test")))
def test_snapshot_project_not_init(self):
failed = False
try:
self.snapshot = SnapshotCommand(self.temp_dir, self.cli_helper)
except ProjectNotInitializedException:
failed = True
assert failed
def test_snapshot_help(self):
self.__set_variables()
print(
"\n====================================== datmo snapshot ==========================\n"
)
self.snapshot.parse(["snapshot"])
assert self.snapshot.execute()
print(
"\n====================================== datmo snapshot --help ==========================\n"
)
self.snapshot.parse(["snapshot", "--help"])
assert self.snapshot.execute()
print(
"\n====================================== datmo snapshot create --help ==========================\n"
)
self.snapshot.parse(["snapshot", "create", "--help"])
assert self.snapshot.execute()
def test_snapshot_command(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_session_id = "test_session_id"
test_task_id = "test_task_id"
test_code_id = "test_code_id"
test_environment_definition_filepath = self.env_def_path
test_config_filepath = self.config_filepath
test_stats_filepath = self.config_filepath
test_filepaths = [self.filepath, self.filepath_2]
# try single filepath
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
test_task_id
])
# testing for proper parsing
assert self.snapshot.args.message == test_message
assert self.snapshot.args.task_id == test_task_id
# try single filepath
self.snapshot.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--code-id",
test_code_id,
"--environment-def",
test_environment_definition_filepath,
"--config-filepath",
test_config_filepath,
"--stats-filepath",
test_stats_filepath,
"--filepaths",
test_filepaths[0],
])
# test for desired side effects
assert self.snapshot.args.message == test_message
assert self.snapshot.args.label == test_label
assert self.snapshot.args.session_id == test_session_id
assert self.snapshot.args.code_id == test_code_id
assert self.snapshot.args.environment_definition_filepath == test_environment_definition_filepath
assert self.snapshot.args.config_filepath == test_config_filepath
assert self.snapshot.args.stats_filepath == test_stats_filepath
assert self.snapshot.args.filepaths == [test_filepaths[0]]
# test multiple filepaths
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--session-id", test_session_id, "--code-id",
test_code_id, "--environment-def",
test_environment_definition_filepath, "--config-filepath",
test_config_filepath, "--stats-filepath", test_stats_filepath,
"--filepaths", test_filepaths[0], "--filepaths", test_filepaths[1]
])
# test for desired side effects
assert self.snapshot.args.message == test_message
assert self.snapshot.args.label == test_label
assert self.snapshot.args.session_id == test_session_id
assert self.snapshot.args.code_id == test_code_id
assert self.snapshot.args.environment_definition_filepath == test_environment_definition_filepath
assert self.snapshot.args.config_filepath == test_config_filepath
assert self.snapshot.args.stats_filepath == test_stats_filepath
assert self.snapshot.args.filepaths == test_filepaths
snapshot_id_1 = self.snapshot.execute()
assert snapshot_id_1
def test_datmo_snapshot_create_from_task(self):
self.__set_variables()
test_message = "this is a test message"
test_code_id = "test_code_id"
# create task
test_command = "sh -c 'echo accuracy:0.45'"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.task = TaskCommand(self.temp_dir, self.cli_helper)
self.task.parse([
"task", "run", "--environment-def", test_dockerfile, test_command
])
# test proper execution of task run command
task_obj = self.task.execute()
task_id = task_obj.id
# test task id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id
])
# test for desired side effects
assert self.snapshot.args.message == test_message
snapshot_id = self.snapshot.execute()
assert snapshot_id
def test_snapshot_create_from_task_fail_user_inputs(self):
self.__set_variables()
test_message = "this is a test message"
# create task
test_command = "sh -c 'echo accuracy:0.45'"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.task = TaskCommand(self.temp_dir, self.cli_helper)
self.task.parse([
"task", "run", "--environment-def", test_dockerfile, test_command
])
# test proper execution of task run command
task_obj = self.task.execute()
task_id = task_obj.id
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--code-id", "test_code_id"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--commit-id", "test_commit_id"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--environment-id", "test_environment_id"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--environment-def", "test_environment_def"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--file-collection-id", "test_file_collection_id"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--filepaths", "mypath"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--config-filepath", "mypath"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--config-filename", "myname"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--stats-filepath", "mypath"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--stats-filename", "myname"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
def test_datmo_snapshot_create_fail_mutually_exclusive_args(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_session_id = "test_session_id"
test_code_id = "test_code_id"
test_commit_id = "test_commit_id"
test_environment_id = "test_environment_id"
test_environment_definition_filepath = self.env_def_path
test_file_collection_id = "test_file_collection_id"
test_filepaths = [self.filepath]
test_config_filename = "config.json"
test_config_filepath = self.config_filepath
test_stats_filename = "stats.json"
test_stats_filepath = self.config_filepath
# Code exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--session-id", test_session_id, "--code-id",
test_code_id, "--commit-id", test_commit_id
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Environment exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--environment-id",
test_environment_id,
"--environment-def",
test_environment_definition_filepath,
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# File exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--file-collection-id",
test_file_collection_id,
"--filepaths",
test_filepaths[0],
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Config exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--config-filename",
test_config_filename,
"--config-filepath",
test_config_filepath,
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Stats exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--stats-filename",
test_stats_filename,
"--stats-filepath",
test_stats_filepath,
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_create_default(self):
self.__set_variables()
self.snapshot.parse(["snapshot", "create", "-m", "my test snapshot"])
snapshot_id_2 = self.snapshot.execute()
assert snapshot_id_2
def test_datmo_snapshot_create_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot.parse(["snapshot", "create" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_delete(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "create", "-m", "my test snapshot"])
snapshot_id = self.snapshot.execute()
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "delete", "--id", snapshot_id])
result = self.snapshot.execute()
assert result
def test_datmo_snapshot_delete_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot.parse(["snapshot", "delete" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_ls(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "create", "-m", "my test snapshot"])
snapshot_id = self.snapshot.execute()
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "ls"])
result = self.snapshot.execute()
assert result
assert snapshot_id in result
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "ls", "-a"])
result = self.snapshot.execute()
assert result
assert snapshot_id in result
def test_datmo_snapshot_checkout_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot.parse(["snapshot", "checkout" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_checkout(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "create", "-m", "my test snapshot"])
snapshot_id = self.snapshot.execute()
# remove datmo_task folder to have no changes before checkout
datmo_tasks_dirpath = os.path.join(self.snapshot.home, "datmo_tasks")
shutil.rmtree(datmo_tasks_dirpath)
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "checkout", "--id", snapshot_id])
result = self.snapshot.execute()
assert result
class TestSnapshotCommand():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.cli_helper = Helper()
def teardown_method(self):
pass
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.snapshot_command = SnapshotCommand(self.cli_helper)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# Create config file
self.config_filepath = os.path.join(self.snapshot_command.home,
"config.json")
with open(self.config_filepath, "wb") as f:
f.write(to_bytes(str("{}")))
# Create stats file
self.stats_filepath = os.path.join(self.snapshot_command.home,
"stats.json")
with open(self.stats_filepath, "wb") as f:
f.write(to_bytes(str("{}")))
# Create test file
self.filepath = os.path.join(self.snapshot_command.home, "file.txt")
with open(self.filepath, "wb") as f:
f.write(to_bytes(str("test")))
# Create another test file
self.filepath_2 = os.path.join(self.snapshot_command.home, "file2.txt")
with open(self.filepath_2, "wb") as f:
f.write(to_bytes(str("test")))
# Create config
self.config = 'foo:bar'
self.config1 = "{'foo1':'bar1'}"
self.config2 = "this is test config blob"
# Create stats
self.stats = 'foo:bar'
self.stats1 = "{'foo1':'bar1'}"
self.stats2 = "this is test stats blob"
def test_snapshot_help(self):
self.__set_variables()
print(
"\n====================================== datmo snapshot ==========================\n"
)
self.snapshot_command.parse(["snapshot"])
assert self.snapshot_command.execute()
print(
"\n====================================== datmo snapshot --help ==========================\n"
)
self.snapshot_command.parse(["snapshot", "--help"])
assert self.snapshot_command.execute()
print(
"\n====================================== datmo snapshot create --help ==========================\n"
)
self.snapshot_command.parse(["snapshot", "create", "--help"])
assert self.snapshot_command.execute()
def test_snapshot_create(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_run_id = "test_run_id"
test_environment_definition_filepath = self.env_def_path
test_config_filepath = self.config_filepath
test_stats_filepath = self.config_filepath
test_paths = [self.filepath, self.filepath_2]
# try single filepath
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id",
test_run_id
])
# testing for proper parsing
assert self.snapshot_command.args.message == test_message
assert self.snapshot_command.args.run_id == test_run_id
# try single filepath
self.snapshot_command.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--environment-paths",
test_environment_definition_filepath,
"--config-filepath",
test_config_filepath,
"--stats-filepath",
test_stats_filepath,
"--paths",
test_paths[0],
])
# test for desired side effects
assert self.snapshot_command.args.message == test_message
assert self.snapshot_command.args.label == test_label
assert self.snapshot_command.args.environment_paths == [
test_environment_definition_filepath
]
assert self.snapshot_command.args.config_filepath == test_config_filepath
assert self.snapshot_command.args.stats_filepath == test_stats_filepath
assert self.snapshot_command.args.paths == [test_paths[0]]
# test multiple paths
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--environment-paths",
test_environment_definition_filepath, "--config-filepath",
test_config_filepath, "--stats-filepath", test_stats_filepath,
"--paths", test_paths[0], "--paths", test_paths[1]
])
# test for desired side effects
assert self.snapshot_command.args.message == test_message
assert self.snapshot_command.args.label == test_label
assert self.snapshot_command.args.environment_paths == [
test_environment_definition_filepath
]
assert self.snapshot_command.args.config_filepath == test_config_filepath
assert self.snapshot_command.args.stats_filepath == test_stats_filepath
assert self.snapshot_command.args.paths == test_paths
snapshot_obj_1 = self.snapshot_command.execute()
assert snapshot_obj_1
def test_snapshot_create_config_stats(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_config = self.config
test_stats = self.stats
# try config
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--config", test_config, "--stats", test_stats
])
# test for desired side effects
snapshot_obj = self.snapshot_command.execute()
assert snapshot_obj
test_config = self.config1
test_stats = self.stats1
# try config
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--config", test_config, "--stats", test_stats
])
# test for desired side effects
snapshot_obj = self.snapshot_command.execute()
assert snapshot_obj
test_config = self.config2
test_stats = self.stats2
# try config
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--config", test_config, "--stats", test_stats
])
# test for desired side effects
snapshot_obj = self.snapshot_command.execute()
assert snapshot_obj
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_snapshot_create_from_run(self):
self.__set_variables()
test_message = "this is a test message"
# create task
test_command = "sh -c 'echo accuracy:0.45'"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.run = RunCommand(self.cli_helper)
self.run.parse(
["run", "--environment-paths", test_dockerfile, test_command])
# test proper execution of task run command
run_obj = self.run.execute()
run_id = run_obj.id
# test run id
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id", run_id
])
# test for desired side effects
assert self.snapshot_command.args.message == test_message
snapshot_obj = self.snapshot_command.execute()
assert snapshot_obj
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_snapshot_create_from_run_fail_user_inputs(self):
self.__set_variables()
test_message = "this is a test message"
# create run
test_command = "sh -c 'echo accuracy:0.45'"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.run = RunCommand(self.cli_helper)
self.run.parse(
["run", "--environment-paths", test_dockerfile, test_command])
# test proper execution of task run command
run_obj = self.run.execute()
run_id = run_obj.id
failed = False
try:
# test run id with environment-id
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id",
run_id, "--environment-id", "test_environment_id"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test run id with environment-def
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id",
run_id, "--environment-paths", "test_environment_path"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test run id with filepaths
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id",
run_id, "--paths", "mypath"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test run id with config-filepath
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id",
run_id, "--config-filepath", "mypath"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test run id with config-filename
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id",
run_id, "--config-filename", "myname"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test run id with stats-filepath
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id",
run_id, "--stats-filepath", "mypath"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test run id with stats-filename
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id",
run_id, "--stats-filename", "myname"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
def test_snapshot_create_fail_mutually_exclusive_args(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_environment_id = "test_environment_id"
test_environment_definition_filepath = self.env_def_path
test_config_filename = "config.json"
test_config_filepath = self.config_filepath
test_stats_filename = "stats.json"
test_stats_filepath = self.config_filepath
# Environment exception
exception_thrown = False
try:
self.snapshot_command.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--environment-id",
test_environment_id,
"--environment-paths",
test_environment_definition_filepath,
])
_ = self.snapshot_command.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Config exception
exception_thrown = False
try:
self.snapshot_command.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--config-filename",
test_config_filename,
"--config-filepath",
test_config_filepath,
])
_ = self.snapshot_command.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Stats exception
exception_thrown = False
try:
self.snapshot_command.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--stats-filename",
test_stats_filename,
"--stats-filepath",
test_stats_filepath,
])
_ = self.snapshot_command.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
def test_snapshot_create_default(self):
self.__set_variables()
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj_2 = self.snapshot_command.execute()
assert snapshot_obj_2
def test_snapshot_create_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot_command.parse(
["snapshot", "create"
"--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_snapshot_update(self):
self.__set_variables()
test_config = ["depth: 10", "learning_rate:0.91"]
test_stats = ["acc: 91.34", "f1_score:0.91"]
test_config1 = "{'foo_config': 'bar_config'}"
test_stats1 = "{'foo_stats': 'bar_stats'}"
test_config2 = "this is a config blob"
test_stats2 = "this is a stats blob"
test_message = "test_message"
test_label = "test_label"
# 1. Updating both message and label
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse([
"snapshot", "update", snapshot_obj.id, "--message", test_message,
"--label", test_label
])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.message == test_message
assert result.label == test_label
# 2. Updating only message
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse(
["snapshot", "update", snapshot_obj.id, "--message", test_message])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.message == test_message
# Updating label
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse(
["snapshot", "update", snapshot_obj.id, "--label", test_label])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.label == test_label
# 3. Updating config, message and label
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse([
"snapshot", "update", snapshot_obj.id, "--config", test_config[0],
"--config", test_config[1], "--message", test_message, "--label",
test_label
])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.config == {"depth": "10", "learning_rate": "0.91"}
assert result.message == test_message
assert result.label == test_label
# 4. Updating stats, message and label
# Test when optional parameters are not given
self.snapshot_command.parse([
"snapshot", "update", snapshot_obj.id, "--stats", test_stats[0],
"--stats", test_stats[1], "--message", test_message, "--label",
test_label
])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.stats == {"acc": "91.34", "f1_score": "0.91"}
assert result.message == test_message
assert result.label == test_label
# Adding sleep due to issue with consistency in blitzdb database
time.sleep(2)
# 5. Updating config, stats
# Test when optional parameters are not given
self.snapshot_command.parse([
"snapshot", "update", snapshot_obj.id, "--config", test_config1,
"--stats", test_stats1
])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.config == {
"depth": "10",
"learning_rate": "0.91",
'foo_config': 'bar_config'
}
assert result.stats == {
"acc": "91.34",
"f1_score": "0.91",
'foo_stats': 'bar_stats'
}
assert result.message == test_message
assert result.label == test_label
# Test when optional parameters are not given
self.snapshot_command.parse([
"snapshot", "update", snapshot_obj.id, "--config", test_config2,
"--stats", test_stats2
])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.config == {
"depth": "10",
"learning_rate": "0.91",
'foo_config': 'bar_config',
'config': test_config2
}
assert result.stats == {
"acc": "91.34",
"f1_score": "0.91",
'foo_stats': 'bar_stats',
'stats': test_stats2
}
assert result.message == test_message
assert result.label == test_label
def test_snapshot_delete(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse(["snapshot", "delete", snapshot_obj.id])
result = self.snapshot_command.execute()
assert result
def test_snapshot_delete_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot_command.parse(
["snapshot", "delete"
"--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_snapshot_ls_invisible(self):
self.__set_variables()
# Test invisible snapshot from task
# create task
test_command = "sh -c 'echo accuracy:0.45'"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.run = RunCommand(self.cli_helper)
self.run.parse(
["run", "--environment-paths", test_dockerfile, test_command])
# test proper execution of task run command
task_obj = self.run.execute()
# test no visible snapshots
self.snapshot_command.parse(["snapshot", "ls"])
result = self.snapshot_command.execute()
assert not result
# test invisible snapshot from task_id was created
self.snapshot_command.parse(["snapshot", "ls", "--all"])
result = self.snapshot_command.execute()
assert result
assert task_obj.after_snapshot_id in [obj.id for obj in result]
def test_snapshot_ls(self):
self.__set_variables()
# create a visible snapshot
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
created_snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse(["snapshot", "ls"])
result = self.snapshot_command.execute()
assert result
assert created_snapshot_obj in result
# Check if current snapshot id is the same as the latest created id
current_snapshot_obj = self.snapshot_command.snapshot_controller.current_snapshot(
)
current_snapshot_id = current_snapshot_obj.id if current_snapshot_obj else None
assert current_snapshot_id == created_snapshot_obj.id
# Test when optional parameters are not given
self.snapshot_command.parse(["snapshot", "ls", "--details"])
result = self.snapshot_command.execute()
assert result
assert created_snapshot_obj in result
# Test failure (format)
failed = False
try:
self.snapshot_command.parse(["snapshot", "ls", "--format"])
except ArgumentError:
failed = True
assert failed
# Test success format csv
self.snapshot_command.parse(["snapshot", "ls", "--format", "csv"])
snapshot_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
# Test success format csv, download default
self.snapshot_command.parse(
["snapshot", "ls", "--format", "csv", "--download"])
snapshot_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
test_wildcard = os.path.join(
self.snapshot_command.snapshot_controller.home, "snapshot_ls_*")
paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
assert paths
assert open(paths[0], "r").read()
os.remove(paths[0])
# Test success format csv, download exact path
test_path = os.path.join(self.temp_dir, "my_output")
self.snapshot_command.parse([
"snapshot", "ls", "--format", "csv", "--download",
"--download-path", test_path
])
snapshot_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
assert os.path.isfile(test_path)
assert open(test_path, "r").read()
os.remove(test_path)
# Test success format table
self.snapshot_command.parse(["snapshot", "ls"])
environment_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
# Test success format table, download default
self.snapshot_command.parse(["snapshot", "ls", "--download"])
snapshot_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
test_wildcard = os.path.join(
self.snapshot_command.snapshot_controller.home, "snapshot_ls_*")
paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
assert paths
assert open(paths[0], "r").read()
os.remove(paths[0])
# Test success format table, download exact path
test_path = os.path.join(self.temp_dir, "my_output")
self.snapshot_command.parse(
["snapshot", "ls", "--download", "--download-path", test_path])
snapshot_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
assert os.path.isfile(test_path)
assert open(test_path, "r").read()
os.remove(test_path)
def test_snapshot_checkout_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot_command.parse(
["snapshot", "checkout"
"--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_snapshot_checkout(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse(["snapshot", "checkout", snapshot_obj.id])
result = self.snapshot_command.execute()
assert result
def test_snapshot_diff(self):
self.__set_variables()
# Create config file
with open(self.config_filepath, "wb") as f:
f.write(to_bytes(str('{"depth":6}')))
# Create stats file
with open(self.stats_filepath, "wb") as f:
f.write(to_bytes(str('{"acc":0.97}')))
# Create snapshots to test
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj_1 = self.snapshot_command.execute()
# Create another test file
self.filepath_3 = os.path.join(self.snapshot_command.home, "file3.txt")
with open(self.filepath_3, "wb") as f:
f.write(to_bytes(str("test")))
# Create config file
with open(self.config_filepath, "wb") as f:
f.write(to_bytes(str('{"depth":5}')))
# Create stats file
with open(self.stats_filepath, "wb") as f:
f.write(to_bytes(str('{"acc":0.91}')))
self.snapshot_command.parse(
["snapshot", "create", "-m", "my second snapshot"])
snapshot_obj_2 = self.snapshot_command.execute()
# Test diff with the above two snapshots
self.snapshot_command.parse(
["snapshot", "diff", snapshot_obj_1.id, snapshot_obj_2.id])
result = self.snapshot_command.execute()
assert result
assert "message" in result
assert "label" in result
assert "code_id" in result
assert "environment_id" in result
assert "file_collection_id" in result
assert "config" in result
assert "stats" in result
def test_snapshot_inspect(self):
self.__set_variables()
# Create snapshot to test
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test inspect for this snapshot
self.snapshot_command.parse(["snapshot", "inspect", snapshot_obj.id])
result = self.snapshot_command.execute()
assert result
assert "Code" in result
assert "Environment" in result
assert "Files" in result
assert "Config" in result
assert "Stats" in result
class TestTaskCommand():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.cli_helper = Helper()
def teardown_method(self):
pass
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.task_command = TaskCommand(self.cli_helper)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
def test_task_command(self):
self.__set_variables()
self.task_command.parse(["task"])
assert self.task_command.execute()
def test_task_run_should_fail1(self):
self.__set_variables()
# Test failure case
self.task_command.parse(["task", "run"])
result = self.task_command.execute()
assert not result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_run(self):
# TODO: Adding test with `--interactive` argument and terminate inside container
self.__set_variables()
# Test failure command execute
test_command = ["yo", "yo"]
self.task_command.parse(["task", "run", test_command])
result = self.task_command.execute()
assert result
# Test success case
test_command = ["sh", "-c", "echo accuracy:0.45"]
test_ports = ["8888:8888", "9999:9999"]
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
test_mem_limit = "4g"
# test for single set of ports
self.task_command.parse([
"task", "run", "--ports", test_ports[0], "--environment-paths",
test_dockerfile, "--mem-limit", test_mem_limit, test_command
])
# test for desired side effects
assert self.task_command.args.cmd == test_command
assert self.task_command.args.ports == [test_ports[0]]
assert self.task_command.args.environment_paths == [test_dockerfile]
assert self.task_command.args.mem_limit == test_mem_limit
self.task_command.parse([
"task", "run", "-p", test_ports[0], "-p", test_ports[1],
"--environment-paths", test_dockerfile, "--mem-limit",
test_mem_limit, test_command
])
# test for desired side effects
assert self.task_command.args.cmd == test_command
assert self.task_command.args.ports == test_ports
assert self.task_command.args.environment_paths == [test_dockerfile]
assert self.task_command.args.mem_limit == test_mem_limit
# test proper execution of task run command
result = self.task_command.execute()
time.sleep(1)
assert result
assert isinstance(result, CoreTask)
assert result.logs
assert "accuracy" in result.logs
assert result.results
assert result.results == {"accuracy": "0.45"}
assert result.status == "SUCCESS"
# teardown
self.task_command.parse(["task", "stop", "--all"])
# test when all is passed to stop all
task_stop_command = self.task_command.execute()
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_run_string_command(self):
# TODO: Adding test with `--interactive` argument and terminate inside container
self.__set_variables()
# Test success case
test_command = "sh -c 'echo accuracy:0.45'"
test_ports = ["8888:8888", "9999:9999"]
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
test_mem_limit = "4g"
self.task_command.parse([
"task", "run", "--ports", test_ports[0], "--ports", test_ports[1],
"--environment-paths", test_dockerfile, "--mem-limit",
test_mem_limit, test_command
])
# test for desired side effects
assert self.task_command.args.cmd == test_command
assert self.task_command.args.ports == test_ports
assert self.task_command.args.environment_paths == [test_dockerfile]
assert self.task_command.args.mem_limit == test_mem_limit
# test proper execution of task run command
result = self.task_command.execute()
assert result
assert isinstance(result, CoreTask)
assert result.logs
assert "accuracy" in result.logs
assert result.results
assert result.results == {"accuracy": "0.45"}
assert result.status == "SUCCESS"
# teardown
self.task_command.parse(["task", "stop", "--all"])
# test when all is passed to stop all
task_stop_command = self.task_command.execute()
# def test_multiple_concurrent_task_run_command(self):
# test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
# test_command = ["sh", "-c", "echo accuracy:0.45"]
# manager = Manager()
# return_dict = manager.dict()
#
# def task_exec_func(procnum, return_dict):
# print("Creating Task object")
# task = TaskCommand(self.cli_helper)
# print("Parsing command")
# task.parse(
# ["task", "run", "--environment-paths", test_dockerfile, test_command])
# print("Executing command")
# result = task.execute()
# return_dict[procnum] = result
#
# self.__set_variables()
# test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
#
# # Run all three tasks in parallel
# jobs = []
# number_tasks = 3
# for i in range(number_tasks):
# p = Process(target=task_exec_func, args=(i, return_dict))
# jobs.append(p)
# p.start()
#
# # Join
# for proc in jobs:
# proc.join()
#
# results = return_dict.values()
# assert len(results) == number_tasks
# for result in results:
# assert result
# assert isinstance(result, CoreTask)
# assert result.logs
# assert "accuracy" in result.logs
# assert result.results
# assert result.results == {"accuracy": "0.45"}
# assert result.status == "SUCCESS"
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_run_notebook(self):
self.__set_variables()
# Update the default Dockerfile to test with
with open(self.env_def_path, "wb") as f:
f.write(to_bytes("FROM nbgallery/jupyter-alpine:latest"))
# Test success case
test_command = ["jupyter", "notebook", "list"]
test_ports = ["8888:8888", "9999:9999"]
test_mem_limit = "4g"
# test single ports option before command
self.task_command.parse([
"task", "run", "--ports", test_ports[0], "--mem-limit",
test_mem_limit, test_command
])
# test for desired side effects
assert self.task_command.args.cmd == test_command
assert self.task_command.args.ports == [test_ports[0]]
assert self.task_command.args.mem_limit == test_mem_limit
# test multiple ports option before command
self.task_command.parse([
"task", "run", "--ports", test_ports[0], "--ports", test_ports[1],
"--mem-limit", test_mem_limit, test_command
])
# test for desired side effects
assert self.task_command.args.cmd == test_command
assert self.task_command.args.ports == test_ports
assert self.task_command.args.mem_limit == test_mem_limit
# test proper execution of task run command
result = self.task_command.execute()
assert result
assert isinstance(result, CoreTask)
assert result.logs
assert "Currently running servers" in result.logs
assert result.status == "SUCCESS"
# teardown
self.task_command.parse(["task", "stop", "--all"])
# test when all is passed to stop all
_ = self.task_command.execute()
def test_task_run_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.task_command.parse(["task", "run" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_task_ls(self):
self.__set_variables()
# Test defaults
self.task_command.parse(["task", "ls"])
task_objs = self.task_command.execute()
assert task_objs == []
test_session_id = 'test_session_id'
self.task_command.parse(
["task", "ls", "--session-id", test_session_id])
# test for desired side effects
assert self.task_command.args.session_id == test_session_id
# Test failure no session
failed = False
try:
_ = self.task_command.execute()
except SessionDoesNotExist:
failed = True
assert failed
# Test failure (format)
failed = False
try:
self.task_command.parse(["task", "ls", "--format"])
except ArgumentError:
failed = True
assert failed
# Test success format csv
self.task_command.parse(["task", "ls", "--format", "csv"])
task_objs = self.task_command.execute()
assert task_objs == []
# Test success format csv, download default
self.task_command.parse(
["task", "ls", "--format", "csv", "--download"])
task_objs = self.task_command.execute()
assert task_objs == []
test_wildcard = os.path.join(self.task_command.task_controller.home,
"task_ls_*")
paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
assert paths
assert open(paths[0], "r").read()
os.remove(paths[0])
# Test success format csv, download exact path
test_path = os.path.join(self.temp_dir, "my_output")
self.task_command.parse([
"task", "ls", "--format", "csv", "--download", "--download-path",
test_path
])
task_objs = self.task_command.execute()
assert task_objs == []
assert os.path.isfile(test_path)
assert open(test_path, "r").read()
os.remove(test_path)
# Test success format table
self.task_command.parse(["task", "ls"])
task_objs = self.task_command.execute()
assert task_objs == []
# Test success format table, download default
self.task_command.parse(["task", "ls", "--download"])
task_objs = self.task_command.execute()
assert task_objs == []
test_wildcard = os.path.join(self.task_command.task_controller.home,
"task_ls_*")
paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
assert paths
assert open(paths[0], "r").read()
os.remove(paths[0])
# Test success format table, download exact path
test_path = os.path.join(self.temp_dir, "my_output")
self.task_command.parse(
["task", "ls", "--download", "--download-path", test_path])
task_objs = self.task_command.execute()
assert task_objs == []
assert os.path.isfile(test_path)
assert open(test_path, "r").read()
os.remove(test_path)
def test_task_ls_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.task_command.parse(["task", "ls" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_stop_success(self):
# 1) Test stop with task_id
# 2) Test stop with all
self.__set_variables()
test_command = ["sh", "-c", "echo yo"]
test_ports = "8888:8888"
test_mem_limit = "4g"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.task_command.parse([
"task", "run", "--ports", test_ports, "--environment-paths",
test_dockerfile, "--mem-limit", test_mem_limit, test_command
])
test_task_obj = self.task_command.execute()
# 1) Test option 1
self.task_command.parse(["task", "stop", "--id", test_task_obj.id])
# test for desired side effects
assert self.task_command.args.id == test_task_obj.id
# test when task id is passed to stop it
task_stop_command = self.task_command.execute()
assert task_stop_command == True
# 2) Test option 2
self.task_command.parse(["task", "stop", "--all"])
# test when all is passed to stop all
task_stop_command = self.task_command.execute()
assert task_stop_command == True
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_stop_failure_required_args(self):
self.__set_variables()
# Passing wrong task id
self.task_command.parse(["task", "stop"])
failed = False
try:
_ = self.task_command.execute()
except RequiredArgumentMissing:
failed = True
assert failed
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_stop_failure_mutually_exclusive_vars(self):
self.__set_variables()
# Passing wrong task id
self.task_command.parse(
["task", "stop", "--id", "invalid-task-id", "--all"])
failed = False
try:
_ = self.task_command.execute()
except MutuallyExclusiveArguments:
failed = True
assert failed
def test_task_stop_failure_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.task_command.parse(["task", "stop" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_task_stop_invalid_task_id(self):
self.__set_variables()
# Passing wrong task id
self.task_command.parse(["task", "stop", "--id", "invalid-task-id"])
# test when wrong task id is passed to stop it
result = self.task_command.execute()
assert not result