def test_datmo_snapshot_create_from_task(self):
self.__set_variables()
test_message = "this is a test message"
test_code_id = "test_code_id"
# create task
test_command = "sh -c 'echo accuracy:0.45'"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.task = TaskCommand(self.temp_dir, self.cli_helper)
self.task.parse([
"task", "run", "--environment-def", test_dockerfile, test_command
])
# test proper execution of task run command
task_obj = self.task.execute()
task_id = task_obj.id
# test task id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id
])
# test for desired side effects
assert self.snapshot.args.message == test_message
snapshot_id = self.snapshot.execute()
assert snapshot_id
def test_task_project_not_init(self):
failed = False
try:
self.task = TaskCommand(self.temp_dir, self.cli_helper)
except ProjectNotInitializedException:
failed = True
assert failed
def __set_variables(self):
init = ProjectCommand(self.temp_dir, self.cli_helper)
init.parse(["init", "--name", "foobar", "--description", "test model"])
init.execute()
self.task = TaskCommand(self.temp_dir, self.cli_helper)
# Create environment_driver definition
env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.task_command = TaskCommand(self.cli_helper)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.environment_command = EnvironmentCommand(self.cli_helper)
self.task_command = TaskCommand(self.cli_helper)
self.snapshot_command = SnapshotCommand(self.cli_helper)
# Create test file
self.filepath = os.path.join(self.snapshot_command.home, "file.txt")
with open(self.filepath, "wb") as f:
f.write(to_bytes(str("test")))
class TestTaskCommand():
def setup_class(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.cli_helper = Helper()
def teardown_class(self):
pass
def __set_variables(self):
init = ProjectCommand(self.temp_dir, self.cli_helper)
init.parse(["init", "--name", "foobar", "--description", "test model"])
init.execute()
self.task = TaskCommand(self.temp_dir, self.cli_helper)
# Create environment_driver definition
env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
def test_task_project_not_init(self):
failed = False
try:
self.task = TaskCommand(self.temp_dir, self.cli_helper)
except ProjectNotInitializedException:
failed = True
assert failed
def test_datmo_task_run_should_fail1(self):
self.__set_variables()
# Test failure case
self.task.parse(["task", "run"])
failed = False
try:
_ = self.task.execute()
except:
failed = True
assert failed
def test_datmo_task_run_should_fail2(self):
self.__set_variables()
# Test failure case execute
test_command = ["yo", "yo"]
self.task.parse(["task", "run", test_command])
result = self.task.execute()
assert not result
def test_datmo_task_run(self):
self.__set_variables()
# Test success case
test_command = ["sh", "-c", "echo accuracy:0.45"]
test_gpu = True # TODO: implement in controller
test_ports = "8888:8888"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
test_interactive = True
self.task.parse([
"task", "run", "--gpu", "--ports", test_ports, "--env-def",
test_dockerfile, "--interactive", test_command
])
# test for desired side effects
assert self.task.args.cmd == test_command
assert self.task.args.gpu == test_gpu
assert self.task.args.ports == [test_ports]
assert self.task.args.environment_definition_filepath == test_dockerfile
assert self.task.args.interactive == test_interactive
# test proper execution of task run command
result = self.task.execute()
assert result
assert isinstance(result, CoreTask)
assert result.logs
assert "accuracy" in result.logs
assert result.results
assert result.results == {"accuracy": "0.45"}
assert result.status == "SUCCESS"
def test_task_run_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.task.parse(["task", "run" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_task_ls(self):
self.__set_variables()
test_session_id = 'test_session_id'
self.task.parse(["task", "ls", "--session-id", test_session_id])
# test for desired side effects
assert self.task.args.session_id == test_session_id
task_ls_command = self.task.execute()
assert task_ls_command == True
def test_task_ls_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.task.parse(["task", "ls" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_task_stop(self):
self.__set_variables()
test_command = ["sh", "-c", "echo yo"]
test_ports = "8888:8888"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.task.parse([
"task", "run", "--gpu", "--ports", test_ports, "--env-def",
test_dockerfile, "--interactive", test_command
])
test_task_obj = self.task.execute()
self.task.parse(["task", "stop", "--id", test_task_obj.id])
# test for desired side effects
assert self.task.args.id == test_task_obj.id
# test when task id is passed to stop it
task_stop_command = self.task.execute()
assert task_stop_command == True
def test_task_stop_invalid_task_id(self):
self.__set_variables()
# Passing wrong task id
self.task.parse(["task", "stop", "--id", "invalid-task-id"])
# test when wrong task id is passed to stop it
result = self.task.execute()
assert not result
def test_task_stop_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.task.parse(["task", "stop" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
class TestSnapshot():
def setup_class(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.cli_helper = Helper()
def teardown_class(self):
pass
def __set_variables(self):
self.project = ProjectCommand(self.temp_dir, self.cli_helper)
self.project.parse(
["init", "--name", "foobar", "--description", "test model"])
self.project.execute()
self.snapshot = SnapshotCommand(self.temp_dir, self.cli_helper)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
# Create config
self.config_filepath = os.path.join(self.snapshot.home, "config.json")
with open(self.config_filepath, "w") as f:
f.write(to_unicode(str("{}")))
# Create stats
self.stats_filepath = os.path.join(self.snapshot.home, "stats.json")
with open(self.stats_filepath, "w") as f:
f.write(to_unicode(str("{}")))
# Create test file
self.filepath = os.path.join(self.snapshot.home, "file.txt")
with open(self.filepath, "w") as f:
f.write(to_unicode(str("test")))
# Create another test file
self.filepath_2 = os.path.join(self.snapshot.home, "file2.txt")
with open(self.filepath_2, "w") as f:
f.write(to_unicode(str("test")))
def test_snapshot_project_not_init(self):
failed = False
try:
self.snapshot = SnapshotCommand(self.temp_dir, self.cli_helper)
except ProjectNotInitializedException:
failed = True
assert failed
def test_snapshot_help(self):
self.__set_variables()
print(
"\n====================================== datmo snapshot ==========================\n"
)
self.snapshot.parse(["snapshot"])
assert self.snapshot.execute()
print(
"\n====================================== datmo snapshot --help ==========================\n"
)
self.snapshot.parse(["snapshot", "--help"])
assert self.snapshot.execute()
print(
"\n====================================== datmo snapshot create --help ==========================\n"
)
self.snapshot.parse(["snapshot", "create", "--help"])
assert self.snapshot.execute()
def test_snapshot_command(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_session_id = "test_session_id"
test_task_id = "test_task_id"
test_code_id = "test_code_id"
test_environment_definition_filepath = self.env_def_path
test_config_filepath = self.config_filepath
test_stats_filepath = self.config_filepath
test_filepaths = [self.filepath, self.filepath_2]
# try single filepath
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
test_task_id
])
# testing for proper parsing
assert self.snapshot.args.message == test_message
assert self.snapshot.args.task_id == test_task_id
# try single filepath
self.snapshot.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--code-id",
test_code_id,
"--environment-def",
test_environment_definition_filepath,
"--config-filepath",
test_config_filepath,
"--stats-filepath",
test_stats_filepath,
"--filepaths",
test_filepaths[0],
])
# test for desired side effects
assert self.snapshot.args.message == test_message
assert self.snapshot.args.label == test_label
assert self.snapshot.args.session_id == test_session_id
assert self.snapshot.args.code_id == test_code_id
assert self.snapshot.args.environment_definition_filepath == test_environment_definition_filepath
assert self.snapshot.args.config_filepath == test_config_filepath
assert self.snapshot.args.stats_filepath == test_stats_filepath
assert self.snapshot.args.filepaths == [test_filepaths[0]]
# test multiple filepaths
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--session-id", test_session_id, "--code-id",
test_code_id, "--environment-def",
test_environment_definition_filepath, "--config-filepath",
test_config_filepath, "--stats-filepath", test_stats_filepath,
"--filepaths", test_filepaths[0], "--filepaths", test_filepaths[1]
])
# test for desired side effects
assert self.snapshot.args.message == test_message
assert self.snapshot.args.label == test_label
assert self.snapshot.args.session_id == test_session_id
assert self.snapshot.args.code_id == test_code_id
assert self.snapshot.args.environment_definition_filepath == test_environment_definition_filepath
assert self.snapshot.args.config_filepath == test_config_filepath
assert self.snapshot.args.stats_filepath == test_stats_filepath
assert self.snapshot.args.filepaths == test_filepaths
snapshot_id_1 = self.snapshot.execute()
assert snapshot_id_1
def test_datmo_snapshot_create_from_task(self):
self.__set_variables()
test_message = "this is a test message"
test_code_id = "test_code_id"
# create task
test_command = "sh -c 'echo accuracy:0.45'"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.task = TaskCommand(self.temp_dir, self.cli_helper)
self.task.parse([
"task", "run", "--environment-def", test_dockerfile, test_command
])
# test proper execution of task run command
task_obj = self.task.execute()
task_id = task_obj.id
# test task id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id
])
# test for desired side effects
assert self.snapshot.args.message == test_message
snapshot_id = self.snapshot.execute()
assert snapshot_id
def test_snapshot_create_from_task_fail_user_inputs(self):
self.__set_variables()
test_message = "this is a test message"
# create task
test_command = "sh -c 'echo accuracy:0.45'"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.task = TaskCommand(self.temp_dir, self.cli_helper)
self.task.parse([
"task", "run", "--environment-def", test_dockerfile, test_command
])
# test proper execution of task run command
task_obj = self.task.execute()
task_id = task_obj.id
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--code-id", "test_code_id"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--commit-id", "test_commit_id"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--environment-id", "test_environment_id"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--environment-def", "test_environment_def"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--file-collection-id", "test_file_collection_id"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--filepaths", "mypath"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--config-filepath", "mypath"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--config-filename", "myname"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--stats-filepath", "mypath"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--stats-filename", "myname"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
def test_datmo_snapshot_create_fail_mutually_exclusive_args(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_session_id = "test_session_id"
test_code_id = "test_code_id"
test_commit_id = "test_commit_id"
test_environment_id = "test_environment_id"
test_environment_definition_filepath = self.env_def_path
test_file_collection_id = "test_file_collection_id"
test_filepaths = [self.filepath]
test_config_filename = "config.json"
test_config_filepath = self.config_filepath
test_stats_filename = "stats.json"
test_stats_filepath = self.config_filepath
# Code exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--session-id", test_session_id, "--code-id",
test_code_id, "--commit-id", test_commit_id
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Environment exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--environment-id",
test_environment_id,
"--environment-def",
test_environment_definition_filepath,
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# File exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--file-collection-id",
test_file_collection_id,
"--filepaths",
test_filepaths[0],
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Config exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--config-filename",
test_config_filename,
"--config-filepath",
test_config_filepath,
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Stats exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--stats-filename",
test_stats_filename,
"--stats-filepath",
test_stats_filepath,
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_create_default(self):
self.__set_variables()
self.snapshot.parse(["snapshot", "create", "-m", "my test snapshot"])
snapshot_id_2 = self.snapshot.execute()
assert snapshot_id_2
def test_datmo_snapshot_create_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot.parse(["snapshot", "create" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_delete(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "create", "-m", "my test snapshot"])
snapshot_id = self.snapshot.execute()
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "delete", "--id", snapshot_id])
result = self.snapshot.execute()
assert result
def test_datmo_snapshot_delete_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot.parse(["snapshot", "delete" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_ls(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "create", "-m", "my test snapshot"])
snapshot_id = self.snapshot.execute()
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "ls"])
result = self.snapshot.execute()
assert result
assert snapshot_id in result
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "ls", "-a"])
result = self.snapshot.execute()
assert result
assert snapshot_id in result
def test_datmo_snapshot_checkout_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot.parse(["snapshot", "checkout" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_checkout(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "create", "-m", "my test snapshot"])
snapshot_id = self.snapshot.execute()
# remove datmo_task folder to have no changes before checkout
datmo_tasks_dirpath = os.path.join(self.snapshot.home, "datmo_tasks")
shutil.rmtree(datmo_tasks_dirpath)
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "checkout", "--id", snapshot_id])
result = self.snapshot.execute()
assert result
def test_snapshot_create_from_task_fail_user_inputs(self):
self.__set_variables()
test_message = "this is a test message"
# create task
test_command = "sh -c 'echo accuracy:0.45'"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.task = TaskCommand(self.temp_dir, self.cli_helper)
self.task.parse([
"task", "run", "--environment-def", test_dockerfile, test_command
])
# test proper execution of task run command
task_obj = self.task.execute()
task_id = task_obj.id
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--code-id", "test_code_id"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--commit-id", "test_commit_id"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--environment-id", "test_environment_id"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--environment-def", "test_environment_def"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--file-collection-id", "test_file_collection_id"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--filepaths", "mypath"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--config-filepath", "mypath"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--config-filename", "myname"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--stats-filepath", "mypath"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--stats-filename", "myname"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
class TestSnapshotCommand():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.cli_helper = Helper()
def teardown_method(self):
pass
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.snapshot_command = SnapshotCommand(self.cli_helper)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# Create config file
self.config_filepath = os.path.join(self.snapshot_command.home,
"config.json")
with open(self.config_filepath, "wb") as f:
f.write(to_bytes(str("{}")))
# Create stats file
self.stats_filepath = os.path.join(self.snapshot_command.home,
"stats.json")
with open(self.stats_filepath, "wb") as f:
f.write(to_bytes(str("{}")))
# Create test file
self.filepath = os.path.join(self.snapshot_command.home, "file.txt")
with open(self.filepath, "wb") as f:
f.write(to_bytes(str("test")))
# Create another test file
self.filepath_2 = os.path.join(self.snapshot_command.home, "file2.txt")
with open(self.filepath_2, "wb") as f:
f.write(to_bytes(str("test")))
# Create config
self.config = 'foo:bar'
self.config1 = "{'foo1':'bar1'}"
self.config2 = "this is test config blob"
# Create stats
self.stats = 'foo:bar'
self.stats1 = "{'foo1':'bar1'}"
self.stats2 = "this is test stats blob"
def test_snapshot_help(self):
self.__set_variables()
print(
"\n====================================== datmo snapshot ==========================\n"
)
self.snapshot_command.parse(["snapshot"])
assert self.snapshot_command.execute()
print(
"\n====================================== datmo snapshot --help ==========================\n"
)
self.snapshot_command.parse(["snapshot", "--help"])
assert self.snapshot_command.execute()
print(
"\n====================================== datmo snapshot create --help ==========================\n"
)
self.snapshot_command.parse(["snapshot", "create", "--help"])
assert self.snapshot_command.execute()
def test_snapshot_create(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_session_id = "test_session_id"
test_task_id = "test_task_id"
test_environment_definition_filepath = self.env_def_path
test_config_filepath = self.config_filepath
test_stats_filepath = self.config_filepath
test_paths = [self.filepath, self.filepath_2]
# try single filepath
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--task-id",
test_task_id
])
# testing for proper parsing
assert self.snapshot_command.args.message == test_message
assert self.snapshot_command.args.task_id == test_task_id
# try single filepath
self.snapshot_command.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--environment-paths",
test_environment_definition_filepath,
"--config-filepath",
test_config_filepath,
"--stats-filepath",
test_stats_filepath,
"--paths",
test_paths[0],
])
# test for desired side effects
assert self.snapshot_command.args.message == test_message
assert self.snapshot_command.args.label == test_label
assert self.snapshot_command.args.session_id == test_session_id
assert self.snapshot_command.args.environment_paths == [
test_environment_definition_filepath
]
assert self.snapshot_command.args.config_filepath == test_config_filepath
assert self.snapshot_command.args.stats_filepath == test_stats_filepath
assert self.snapshot_command.args.paths == [test_paths[0]]
# test multiple paths
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--session-id", test_session_id, "--environment-paths",
test_environment_definition_filepath, "--config-filepath",
test_config_filepath, "--stats-filepath", test_stats_filepath,
"--paths", test_paths[0], "--paths", test_paths[1]
])
# test for desired side effects
assert self.snapshot_command.args.message == test_message
assert self.snapshot_command.args.label == test_label
assert self.snapshot_command.args.session_id == test_session_id
assert self.snapshot_command.args.environment_paths == [
test_environment_definition_filepath
]
assert self.snapshot_command.args.config_filepath == test_config_filepath
assert self.snapshot_command.args.stats_filepath == test_stats_filepath
assert self.snapshot_command.args.paths == test_paths
snapshot_obj_1 = self.snapshot_command.execute()
assert snapshot_obj_1
def test_snapshot_create_config_stats(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_config = self.config
test_stats = self.stats
# try config
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--config", test_config, "--stats", test_stats
])
# test for desired side effects
snapshot_obj = self.snapshot_command.execute()
assert snapshot_obj
test_config = self.config1
test_stats = self.stats1
# try config
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--config", test_config, "--stats", test_stats
])
# test for desired side effects
snapshot_obj = self.snapshot_command.execute()
assert snapshot_obj
test_config = self.config2
test_stats = self.stats2
# try config
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--config", test_config, "--stats", test_stats
])
# test for desired side effects
snapshot_obj = self.snapshot_command.execute()
assert snapshot_obj
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_snapshot_create_from_task(self):
self.__set_variables()
test_message = "this is a test message"
# create task
test_command = "sh -c 'echo accuracy:0.45'"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.task = TaskCommand(self.cli_helper)
self.task.parse([
"task", "run", "--environment-paths", test_dockerfile, test_command
])
# test proper execution of task run command
task_obj = self.task.execute()
task_id = task_obj.id
# test task id
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id
])
# test for desired side effects
assert self.snapshot_command.args.message == test_message
snapshot_obj = self.snapshot_command.execute()
assert snapshot_obj
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_snapshot_create_from_task_fail_user_inputs(self):
self.__set_variables()
test_message = "this is a test message"
# create task
test_command = "sh -c 'echo accuracy:0.45'"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.task = TaskCommand(self.cli_helper)
self.task.parse([
"task", "run", "--environment-paths", test_dockerfile, test_command
])
# test proper execution of task run command
task_obj = self.task.execute()
task_id = task_obj.id
failed = False
try:
# test task id with environment-id
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--environment-id", "test_environment_id"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with environment-def
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--environment-paths", "test_environment_path"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with filepaths
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--paths", "mypath"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with config-filepath
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--config-filepath", "mypath"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with config-filename
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--config-filename", "myname"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with stats-filepath
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--stats-filepath", "mypath"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with stats-filename
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--stats-filename", "myname"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
def test_snapshot_create_fail_mutually_exclusive_args(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_session_id = "test_session_id"
test_environment_id = "test_environment_id"
test_environment_definition_filepath = self.env_def_path
test_config_filename = "config.json"
test_config_filepath = self.config_filepath
test_stats_filename = "stats.json"
test_stats_filepath = self.config_filepath
# Environment exception
exception_thrown = False
try:
self.snapshot_command.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--environment-id",
test_environment_id,
"--environment-paths",
test_environment_definition_filepath,
])
_ = self.snapshot_command.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Config exception
exception_thrown = False
try:
self.snapshot_command.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--config-filename",
test_config_filename,
"--config-filepath",
test_config_filepath,
])
_ = self.snapshot_command.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Stats exception
exception_thrown = False
try:
self.snapshot_command.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--stats-filename",
test_stats_filename,
"--stats-filepath",
test_stats_filepath,
])
_ = self.snapshot_command.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
def test_snapshot_create_default(self):
self.__set_variables()
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj_2 = self.snapshot_command.execute()
assert snapshot_obj_2
def test_snapshot_create_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot_command.parse(
["snapshot", "create"
"--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_snapshot_update(self):
self.__set_variables()
test_config = ["depth: 10", "learning_rate:0.91"]
test_stats = ["acc: 91.34", "f1_score:0.91"]
test_config1 = "{'foo_config': 'bar_config'}"
test_stats1 = "{'foo_stats': 'bar_stats'}"
test_config2 = "this is a config blob"
test_stats2 = "this is a stats blob"
test_message = "test_message"
test_label = "test_label"
# 1. Updating both message and label
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse([
"snapshot", "update", snapshot_obj.id, "--message", test_message,
"--label", test_label
])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.message == test_message
assert result.label == test_label
# 2. Updating only message
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse(
["snapshot", "update", snapshot_obj.id, "--message", test_message])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.message == test_message
# Updating label
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse(
["snapshot", "update", snapshot_obj.id, "--label", test_label])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.label == test_label
# 3. Updating config, message and label
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse([
"snapshot", "update", snapshot_obj.id, "--config", test_config[0],
"--config", test_config[1], "--message", test_message, "--label",
test_label
])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.config == {"depth": "10", "learning_rate": "0.91"}
assert result.message == test_message
assert result.label == test_label
# 4. Updating stats, message and label
# Test when optional parameters are not given
self.snapshot_command.parse([
"snapshot", "update", snapshot_obj.id, "--stats", test_stats[0],
"--stats", test_stats[1], "--message", test_message, "--label",
test_label
])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.stats == {"acc": "91.34", "f1_score": "0.91"}
assert result.message == test_message
assert result.label == test_label
# Adding sleep due to issue with consistency in blitzdb database
time.sleep(2)
# 5. Updating config, stats
# Test when optional parameters are not given
self.snapshot_command.parse([
"snapshot", "update", snapshot_obj.id, "--config", test_config1,
"--stats", test_stats1
])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.config == {
"depth": "10",
"learning_rate": "0.91",
'foo_config': 'bar_config'
}
assert result.stats == {
"acc": "91.34",
"f1_score": "0.91",
'foo_stats': 'bar_stats'
}
assert result.message == test_message
assert result.label == test_label
# Test when optional parameters are not given
self.snapshot_command.parse([
"snapshot", "update", snapshot_obj.id, "--config", test_config2,
"--stats", test_stats2
])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.config == {
"depth": "10",
"learning_rate": "0.91",
'foo_config': 'bar_config',
'config': test_config2
}
assert result.stats == {
"acc": "91.34",
"f1_score": "0.91",
'foo_stats': 'bar_stats',
'stats': test_stats2
}
assert result.message == test_message
assert result.label == test_label
def test_snapshot_delete(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse(["snapshot", "delete", snapshot_obj.id])
result = self.snapshot_command.execute()
assert result
def test_snapshot_delete_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot_command.parse(
["snapshot", "delete"
"--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_snapshot_ls(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
created_snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse(["snapshot", "ls"])
result = self.snapshot_command.execute()
assert result
assert created_snapshot_obj in result
# Test when optional parameters are not given
self.snapshot_command.parse(["snapshot", "ls", "-a"])
result = self.snapshot_command.execute()
assert result
assert created_snapshot_obj in result
# Test failure (format)
failed = False
try:
self.snapshot_command.parse(["snapshot", "ls", "--format"])
except ArgumentError:
failed = True
assert failed
# Test success format csv
self.snapshot_command.parse(["snapshot", "ls", "--format", "csv"])
snapshot_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
# Test success format csv, download default
self.snapshot_command.parse(
["snapshot", "ls", "--format", "csv", "--download"])
snapshot_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
test_wildcard = os.path.join(
self.snapshot_command.snapshot_controller.home, "snapshot_ls_*")
paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
assert paths
assert open(paths[0], "r").read()
os.remove(paths[0])
# Test success format csv, download exact path
test_path = os.path.join(self.temp_dir, "my_output")
self.snapshot_command.parse([
"snapshot", "ls", "--format", "csv", "--download",
"--download-path", test_path
])
snapshot_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
assert os.path.isfile(test_path)
assert open(test_path, "r").read()
os.remove(test_path)
# Test success format table
self.snapshot_command.parse(["snapshot", "ls"])
environment_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
# Test success format table, download default
self.snapshot_command.parse(["snapshot", "ls", "--download"])
snapshot_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
test_wildcard = os.path.join(
self.snapshot_command.snapshot_controller.home, "snapshot_ls_*")
paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
assert paths
assert open(paths[0], "r").read()
os.remove(paths[0])
# Test success format table, download exact path
test_path = os.path.join(self.temp_dir, "my_output")
self.snapshot_command.parse(
["snapshot", "ls", "--download", "--download-path", test_path])
snapshot_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
assert os.path.isfile(test_path)
assert open(test_path, "r").read()
os.remove(test_path)
def test_snapshot_checkout_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot_command.parse(
["snapshot", "checkout"
"--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_snapshot_checkout(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse(["snapshot", "checkout", snapshot_obj.id])
result = self.snapshot_command.execute()
assert result
def test_snapshot_diff(self):
self.__set_variables()
# Create snapshots to test
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj_1 = self.snapshot_command.execute()
# Create another test file
self.filepath_3 = os.path.join(self.snapshot_command.home, "file3.txt")
with open(self.filepath_3, "wb") as f:
f.write(to_bytes(str("test")))
self.snapshot_command.parse(
["snapshot", "create", "-m", "my second snapshot"])
snapshot_obj_2 = self.snapshot_command.execute()
# Test diff with the above two snapshots
self.snapshot_command.parse(
["snapshot", "diff", snapshot_obj_1.id, snapshot_obj_2.id])
result = self.snapshot_command.execute()
assert result
def test_snapshot_inspect(self):
self.__set_variables()
# Create snapshot to test
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test inspect for this snapshot
self.snapshot_command.parse(["snapshot", "inspect", snapshot_obj.id])
result = self.snapshot_command.execute()
assert result
class TestFlow():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.cli_helper = Helper()
def teardown_method(self):
pass
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.environment_command = EnvironmentCommand(self.cli_helper)
self.task_command = TaskCommand(self.cli_helper)
self.snapshot_command = SnapshotCommand(self.cli_helper)
# Create test file
self.filepath = os.path.join(self.snapshot_command.home, "file.txt")
with open(self.filepath, "wb") as f:
f.write(to_bytes(str("test")))
def __environment_setup(self):
self.environment_command.parse(["environment", "setup"])
@self.environment_command.cli_helper.input("1\n")
def dummy(self):
return self.environment_command.execute()
environment_setup_result = dummy(self)
return environment_setup_result
def __task_run(self):
test_command = ["sh", "-c", "echo accuracy:0.45"]
test_ports = ["8888:8888"]
self.task_command.parse(
["task", "run", "-p", test_ports[0], test_command])
task_run_result = self.task_command.execute()
return task_run_result
def __task_ls(self):
self.task_command.parse(["task", "ls"])
task_ls_result = self.task_command.execute()
return task_ls_result
def __snapshot_create(self):
test_message = "creating a snapshot"
self.snapshot_command.parse(["snapshot", "create", "-m", test_message])
snapshot_create_result = self.snapshot_command.execute()
return snapshot_create_result
def __snapshot_ls(self):
self.snapshot_command.parse(["snapshot", "ls"])
snapshot_ls_result = self.snapshot_command.execute()
return snapshot_ls_result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_flow_1(self):
# Flow
# Step 1: environment setup
# Step 2: task run
# Step 3: task ls
# Step 4: snapshot create
# Step 5: snapshot ls
self.__set_variables()
# Step 1: environment setup
environment_setup_result = self.__environment_setup()
assert environment_setup_result
# Step 2: task run
task_run_result = self.__task_run()
assert task_run_result
# Step 3: task ls
task_ls_result = self.__task_ls()
assert task_ls_result
# Step 4: snapshot create
snapshot_create_result = self.__snapshot_create()
assert snapshot_create_result
# Step 5: snapshot ls
snapshot_ls_result = self.__snapshot_ls()
assert snapshot_ls_result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_flow_2(self):
# Flow interruption in environment
# Step 1: interrupted environment setup
# Step 2: environment setup
# Step 3: task run
# Step 4: task ls
# Step 5: snapshot create
# Step 6: snapshot ls
self.__set_variables()
# TODO: TEST more fundamental functions first
# Step 1: interrupted environment setup
# @timeout_decorator.timeout(0.0001, use_signals=False)
# def timed_command_with_interuption():
# result = self.__environment_setup()
# return result
#
# failed = False
# try:
# timed_command_with_interuption()
# except timeout_decorator.timeout_decorator.TimeoutError:
# failed = True
# assert failed
# Step 2: environment setup
environment_setup_result = self.__environment_setup()
assert environment_setup_result
# Step 3: task run
task_run_result = self.__task_run()
assert task_run_result
# Step 4: task ls
task_ls_result = self.__task_ls()
assert task_ls_result
# Step 5: snapshot create
snapshot_create_result = self.__snapshot_create()
assert snapshot_create_result
# Step 6: snapshot ls
snapshot_ls_result = self.__snapshot_ls()
assert snapshot_ls_result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_flow_3(self):
# Flow interruption in task run
# Step 1: environment setup
# Step 2: interrupted task run
# Step 3: task run
# Step 4: task ls
# Step 5: snapshot create
# Step 6: snapshot ls
self.__set_variables()
# Step 1: environment setup
environment_setup_result = self.__environment_setup()
assert environment_setup_result
# TODO: Test more fundamental functions first
# Step 2: interrupted task run
# @timeout_decorator.timeout(0.0001, use_signals=False)
# def timed_command_with_interuption():
# result = self.__task_run()
# return result
#
# failed = False
# try:
# timed_command_with_interuption()
# except timeout_decorator.timeout_decorator.TimeoutError:
# failed = True
# assert failed
# Step 3: task run
task_run_result = self.__task_run()
assert task_run_result
# Step 4: task ls
task_ls_result = self.__task_ls()
assert task_ls_result
# Step 5: snapshot create
snapshot_create_result = self.__snapshot_create()
assert snapshot_create_result
# Step 6: snapshot ls
snapshot_ls_result = self.__snapshot_ls()
assert snapshot_ls_result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_flow_4(self):
# Flow interruption in snapshot create
# Step 1: environment setup
# Step 2: task run
# Step 3: task ls
# Step 4: interrupted snapshot create
# Step 5: snapshot create
# Step 6: snapshot ls
self.__set_variables()
# Step 1: environment setup
environment_setup_result = self.__environment_setup()
assert environment_setup_result
# Step 2: task run
task_run_result = self.__task_run()
assert task_run_result
# Step 3: task ls
task_ls_result = self.__task_ls()
assert task_ls_result
# TODO: Test more fundamental functions first
# Step 4: interrupted snapshot create
# @timeout_decorator.timeout(0.0001, use_signals=False)
# def timed_command_with_interuption():
# result = self.__snapshot_create()
# return result
#
# failed = False
# try:
# timed_command_with_interuption()
# except timeout_decorator.timeout_decorator.TimeoutError:
# failed = True
# assert failed
# snapshot_ls_result = self.__snapshot_ls()
# assert not snapshot_ls_result
# Step 5: snapshot create
snapshot_create_result = self.__snapshot_create()
assert snapshot_create_result
# Step 6: snapshot ls
snapshot_ls_result = self.__snapshot_ls()
assert snapshot_ls_result
class TestTaskCommand():
def setup_class(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.cli_helper = Helper()
def teardown_class(self):
pass
def __set_variables(self):
self.project = ProjectCommand(self.temp_dir, self.cli_helper)
self.project.parse(
["init", "--name", "foobar", "--description", "test model"])
self.project.execute()
self.task = TaskCommand(self.temp_dir, self.cli_helper)
# Create environment_driver definition
env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
def test_task_project_not_init(self):
failed = False
try:
self.task = TaskCommand(self.temp_dir, self.cli_helper)
except ProjectNotInitializedException:
failed = True
assert failed
def test_snapshot_command(self):
self.__set_variables()
self.task.parse(["task"])
assert self.task.execute()
def test_task_run_should_fail1(self):
self.__set_variables()
# Test failure case
self.task.parse(["task", "run"])
result = self.task.execute()
assert not result
def test_task_run(self):
# TODO: Adding test with `--interactive` argument and terminate inside container
self.__set_variables()
# Test failure command execute
test_command = ["yo", "yo"]
self.task.parse(["task", "run", test_command])
result = self.task.execute()
assert result
# Test success case
test_command = ["sh", "-c", "echo accuracy:0.45"]
test_ports = ["8888:8888", "9999:9999"]
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
# test for single set of ports
self.task.parse([
"task", "run", "--ports", test_ports[0], "--environment-def",
test_dockerfile, test_command
])
# test for desired side effects
assert self.task.args.cmd == test_command
assert self.task.args.ports == [test_ports[0]]
assert self.task.args.environment_definition_filepath == test_dockerfile
self.task.parse([
"task", "run", "-p", test_ports[0], "-p", test_ports[1],
"--environment-def", test_dockerfile, test_command
])
# test for desired side effects
assert self.task.args.cmd == test_command
assert self.task.args.ports == test_ports
assert self.task.args.environment_definition_filepath == test_dockerfile
# test proper execution of task run command
result = self.task.execute()
time.sleep(1)
assert result
assert isinstance(result, CoreTask)
assert result.logs
assert "accuracy" in result.logs
assert result.results
assert result.results == {"accuracy": "0.45"}
assert result.status == "SUCCESS"
def test_task_run_string_command(self):
# TODO: Adding test with `--interactive` argument and terminate inside container
self.__set_variables()
# Test success case
test_command = "sh -c 'echo accuracy:0.45'"
test_ports = ["8888:8888", "9999:9999"]
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.task.parse([
"task", "run", "--ports", test_ports[0], "--ports", test_ports[1],
"--environment-def", test_dockerfile, test_command
])
# test for desired side effects
assert self.task.args.cmd == test_command
assert self.task.args.ports == test_ports
assert self.task.args.environment_definition_filepath == test_dockerfile
# test proper execution of task run command
result = self.task.execute()
assert result
assert isinstance(result, CoreTask)
assert result.logs
assert "accuracy" in result.logs
assert result.results
assert result.results == {"accuracy": "0.45"}
assert result.status == "SUCCESS"
# def test_multiple_concurrent_task_run_command(self):
# test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
# test_command = ["sh", "-c", "echo accuracy:0.45"]
# manager = Manager()
# return_dict = manager.dict()
#
# def task_exec_func(procnum, return_dict):
# print("Creating Task object")
# task = TaskCommand(self.temp_dir, self.cli_helper)
# print("Parsing command")
# task.parse(
# ["task", "run", "--environment-def", test_dockerfile, test_command])
# print("Executing command")
# result = task.execute()
# return_dict[procnum] = result
#
# self.__set_variables()
# test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
#
# # Run all three tasks in parallel
# jobs = []
# number_tasks = 3
# for i in range(number_tasks):
# p = Process(target=task_exec_func, args=(i, return_dict))
# jobs.append(p)
# p.start()
#
# # Join
# for proc in jobs:
# proc.join()
#
# results = return_dict.values()
# assert len(results) == number_tasks
# for result in results:
# assert result
# assert isinstance(result, CoreTask)
# assert result.logs
# assert "accuracy" in result.logs
# assert result.results
# assert result.results == {"accuracy": "0.45"}
# assert result.status == "SUCCESS"
def test_task_run_notebook(self):
self.__set_variables()
# Test success case
test_command = ["jupyter", "notebook", "list"]
test_ports = ["8888:8888", "9999:9999"]
# test single ports option before command
self.task.parse(
["task", "run", "--ports", test_ports[0], test_command])
# test for desired side effects
assert self.task.args.cmd == test_command
assert self.task.args.ports == [test_ports[0]]
# test multiple ports option before command
self.task.parse([
"task", "run", "--ports", test_ports[0], "--ports", test_ports[1],
test_command
])
# test for desired side effects
assert self.task.args.cmd == test_command
assert self.task.args.ports == test_ports
# test proper execution of task run command
result = self.task.execute()
assert result
assert isinstance(result, CoreTask)
assert result.logs
assert "Currently running servers" in result.logs
assert result.status == "SUCCESS"
def test_task_run_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.task.parse(["task", "run" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_task_ls(self):
self.__set_variables()
self.task.parse(["task", "ls"])
task_ls_command = self.task.execute()
assert task_ls_command == True
test_session_id = 'test_session_id'
self.task.parse(["task", "ls", "--session-id", test_session_id])
# test for desired side effects
assert self.task.args.session_id == test_session_id
task_ls_command = self.task.execute()
assert task_ls_command == True
def test_task_ls_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.task.parse(["task", "ls" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_task_stop_success(self):
# 1) Test stop with task_id
# 2) Test stop with all
self.__set_variables()
test_command = ["sh", "-c", "echo yo"]
test_ports = "8888:8888"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.task.parse([
"task", "run", "--ports", test_ports, "--environment-def",
test_dockerfile, test_command
])
test_task_obj = self.task.execute()
# 1) Test option 1
self.task.parse(["task", "stop", "--id", test_task_obj.id])
# test for desired side effects
assert self.task.args.id == test_task_obj.id
# test when task id is passed to stop it
task_stop_command = self.task.execute()
assert task_stop_command == True
# 2) Test option 2
self.task.parse(["task", "stop", "--all"])
# test when all is passed to stop all
task_stop_command = self.task.execute()
assert task_stop_command == True
def test_task_stop_failure_required_args(self):
self.__set_variables()
# Passing wrong task id
self.task.parse(["task", "stop"])
failed = False
try:
_ = self.task.execute()
except RequiredArgumentMissing:
failed = True
assert failed
def test_task_stop_failure_mutually_exclusive_vars(self):
self.__set_variables()
# Passing wrong task id
self.task.parse(["task", "stop", "--id", "invalid-task-id", "--all"])
failed = False
try:
_ = self.task.execute()
except MutuallyExclusiveArguments:
failed = True
assert failed
def test_task_stop_failure_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.task.parse(["task", "stop" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_task_stop_invalid_task_id(self):
self.__set_variables()
# Passing wrong task id
self.task.parse(["task", "stop", "--id", "invalid-task-id"])
# test when wrong task id is passed to stop it
result = self.task.execute()
assert not result
class TestWorkspace():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.cli_helper = Helper()
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.workspace_command = WorkspaceCommand(self.cli_helper)
self.task_command = TaskCommand(self.cli_helper)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "wb") as f:
f.write(to_bytes(str("FROM datmo/xgboost:cpu\n")))
def teardown_method(self):
pass
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_notebook(self):
self.__set_variables()
test_mem_limit = "4g"
# test single ports option before command
self.workspace_command.parse([
"notebook",
"--gpu",
"--environment-paths",
self.env_def_path,
"--mem-limit",
test_mem_limit,
])
# test for desired side effects
assert self.workspace_command.args.gpu == True
assert self.workspace_command.args.environment_paths == [
self.env_def_path
]
assert self.workspace_command.args.mem_limit == test_mem_limit
# test multiple ports option before command
self.workspace_command.parse(["notebook"])
assert self.workspace_command.args.gpu == False
@timeout_decorator.timeout(10, use_signals=False)
def timed_run(timed_run_result):
if self.workspace_command.execute():
return timed_run_result
timed_run_result = False
try:
timed_run_result = timed_run(timed_run_result)
except timeout_decorator.timeout_decorator.TimeoutError:
timed_run_result = True
assert timed_run_result
# Stop all running datmo task
self.task_command.parse(["task", "stop", "--all"])
self.task_command.execute()
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_rstudio(self):
self.__set_variables()
# Update environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "wb") as f:
f.write(to_bytes(str("FROM datmo/rstudio:latest\n")))
test_mem_limit = "4g"
# test single ports option before command
self.workspace_command.parse([
"rstudio",
"--environment-paths",
self.env_def_path,
"--mem-limit",
test_mem_limit,
])
# test for desired side effects
assert self.workspace_command.args.environment_paths == [
self.env_def_path
]
assert self.workspace_command.args.mem_limit == test_mem_limit
# test multiple ports option before command
self.workspace_command.parse(["rstudio"])
@timeout_decorator.timeout(10, use_signals=False)
def timed_run(timed_run_result):
if self.workspace_command.execute():
return timed_run_result
timed_run_result = False
try:
timed_run_result = timed_run(timed_run_result)
except timeout_decorator.timeout_decorator.TimeoutError:
timed_run_result = True
assert timed_run_result
# Stop all running datmo task
self.task_command.parse(["task", "stop", "--all"])
self.task_command.execute()
class TestTaskCommand():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.cli_helper = Helper()
def teardown_method(self):
pass
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.task_command = TaskCommand(self.cli_helper)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
def test_task_command(self):
self.__set_variables()
self.task_command.parse(["task"])
assert self.task_command.execute()
def test_task_run_should_fail1(self):
self.__set_variables()
# Test failure case
self.task_command.parse(["task", "run"])
result = self.task_command.execute()
assert not result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_run(self):
# TODO: Adding test with `--interactive` argument and terminate inside container
self.__set_variables()
# Test failure command execute
test_command = ["yo", "yo"]
self.task_command.parse(["task", "run", test_command])
result = self.task_command.execute()
assert result
# Test success case
test_command = ["sh", "-c", "echo accuracy:0.45"]
test_ports = ["8888:8888", "9999:9999"]
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
test_mem_limit = "4g"
# test for single set of ports
self.task_command.parse([
"task", "run", "--ports", test_ports[0], "--environment-paths",
test_dockerfile, "--mem-limit", test_mem_limit, test_command
])
# test for desired side effects
assert self.task_command.args.cmd == test_command
assert self.task_command.args.ports == [test_ports[0]]
assert self.task_command.args.environment_paths == [test_dockerfile]
assert self.task_command.args.mem_limit == test_mem_limit
self.task_command.parse([
"task", "run", "-p", test_ports[0], "-p", test_ports[1],
"--environment-paths", test_dockerfile, "--mem-limit",
test_mem_limit, test_command
])
# test for desired side effects
assert self.task_command.args.cmd == test_command
assert self.task_command.args.ports == test_ports
assert self.task_command.args.environment_paths == [test_dockerfile]
assert self.task_command.args.mem_limit == test_mem_limit
# test proper execution of task run command
result = self.task_command.execute()
time.sleep(1)
assert result
assert isinstance(result, CoreTask)
assert result.logs
assert "accuracy" in result.logs
assert result.results
assert result.results == {"accuracy": "0.45"}
assert result.status == "SUCCESS"
# teardown
self.task_command.parse(["task", "stop", "--all"])
# test when all is passed to stop all
task_stop_command = self.task_command.execute()
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_run_string_command(self):
# TODO: Adding test with `--interactive` argument and terminate inside container
self.__set_variables()
# Test success case
test_command = "sh -c 'echo accuracy:0.45'"
test_ports = ["8888:8888", "9999:9999"]
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
test_mem_limit = "4g"
self.task_command.parse([
"task", "run", "--ports", test_ports[0], "--ports", test_ports[1],
"--environment-paths", test_dockerfile, "--mem-limit",
test_mem_limit, test_command
])
# test for desired side effects
assert self.task_command.args.cmd == test_command
assert self.task_command.args.ports == test_ports
assert self.task_command.args.environment_paths == [test_dockerfile]
assert self.task_command.args.mem_limit == test_mem_limit
# test proper execution of task run command
result = self.task_command.execute()
assert result
assert isinstance(result, CoreTask)
assert result.logs
assert "accuracy" in result.logs
assert result.results
assert result.results == {"accuracy": "0.45"}
assert result.status == "SUCCESS"
# teardown
self.task_command.parse(["task", "stop", "--all"])
# test when all is passed to stop all
task_stop_command = self.task_command.execute()
# def test_multiple_concurrent_task_run_command(self):
# test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
# test_command = ["sh", "-c", "echo accuracy:0.45"]
# manager = Manager()
# return_dict = manager.dict()
#
# def task_exec_func(procnum, return_dict):
# print("Creating Task object")
# task = TaskCommand(self.cli_helper)
# print("Parsing command")
# task.parse(
# ["task", "run", "--environment-paths", test_dockerfile, test_command])
# print("Executing command")
# result = task.execute()
# return_dict[procnum] = result
#
# self.__set_variables()
# test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
#
# # Run all three tasks in parallel
# jobs = []
# number_tasks = 3
# for i in range(number_tasks):
# p = Process(target=task_exec_func, args=(i, return_dict))
# jobs.append(p)
# p.start()
#
# # Join
# for proc in jobs:
# proc.join()
#
# results = return_dict.values()
# assert len(results) == number_tasks
# for result in results:
# assert result
# assert isinstance(result, CoreTask)
# assert result.logs
# assert "accuracy" in result.logs
# assert result.results
# assert result.results == {"accuracy": "0.45"}
# assert result.status == "SUCCESS"
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_run_notebook(self):
self.__set_variables()
# Update the default Dockerfile to test with
with open(self.env_def_path, "wb") as f:
f.write(to_bytes("FROM nbgallery/jupyter-alpine:latest"))
# Test success case
test_command = ["jupyter", "notebook", "list"]
test_ports = ["8888:8888", "9999:9999"]
test_mem_limit = "4g"
# test single ports option before command
self.task_command.parse([
"task", "run", "--ports", test_ports[0], "--mem-limit",
test_mem_limit, test_command
])
# test for desired side effects
assert self.task_command.args.cmd == test_command
assert self.task_command.args.ports == [test_ports[0]]
assert self.task_command.args.mem_limit == test_mem_limit
# test multiple ports option before command
self.task_command.parse([
"task", "run", "--ports", test_ports[0], "--ports", test_ports[1],
"--mem-limit", test_mem_limit, test_command
])
# test for desired side effects
assert self.task_command.args.cmd == test_command
assert self.task_command.args.ports == test_ports
assert self.task_command.args.mem_limit == test_mem_limit
# test proper execution of task run command
result = self.task_command.execute()
assert result
assert isinstance(result, CoreTask)
assert result.logs
assert "Currently running servers" in result.logs
assert result.status == "SUCCESS"
# teardown
self.task_command.parse(["task", "stop", "--all"])
# test when all is passed to stop all
_ = self.task_command.execute()
def test_task_run_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.task_command.parse(["task", "run" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_task_ls(self):
self.__set_variables()
# Test defaults
self.task_command.parse(["task", "ls"])
task_objs = self.task_command.execute()
assert task_objs == []
test_session_id = 'test_session_id'
self.task_command.parse(
["task", "ls", "--session-id", test_session_id])
# test for desired side effects
assert self.task_command.args.session_id == test_session_id
# Test failure no session
failed = False
try:
_ = self.task_command.execute()
except SessionDoesNotExist:
failed = True
assert failed
# Test failure (format)
failed = False
try:
self.task_command.parse(["task", "ls", "--format"])
except ArgumentError:
failed = True
assert failed
# Test success format csv
self.task_command.parse(["task", "ls", "--format", "csv"])
task_objs = self.task_command.execute()
assert task_objs == []
# Test success format csv, download default
self.task_command.parse(
["task", "ls", "--format", "csv", "--download"])
task_objs = self.task_command.execute()
assert task_objs == []
test_wildcard = os.path.join(self.task_command.task_controller.home,
"task_ls_*")
paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
assert paths
assert open(paths[0], "r").read()
os.remove(paths[0])
# Test success format csv, download exact path
test_path = os.path.join(self.temp_dir, "my_output")
self.task_command.parse([
"task", "ls", "--format", "csv", "--download", "--download-path",
test_path
])
task_objs = self.task_command.execute()
assert task_objs == []
assert os.path.isfile(test_path)
assert open(test_path, "r").read()
os.remove(test_path)
# Test success format table
self.task_command.parse(["task", "ls"])
task_objs = self.task_command.execute()
assert task_objs == []
# Test success format table, download default
self.task_command.parse(["task", "ls", "--download"])
task_objs = self.task_command.execute()
assert task_objs == []
test_wildcard = os.path.join(self.task_command.task_controller.home,
"task_ls_*")
paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
assert paths
assert open(paths[0], "r").read()
os.remove(paths[0])
# Test success format table, download exact path
test_path = os.path.join(self.temp_dir, "my_output")
self.task_command.parse(
["task", "ls", "--download", "--download-path", test_path])
task_objs = self.task_command.execute()
assert task_objs == []
assert os.path.isfile(test_path)
assert open(test_path, "r").read()
os.remove(test_path)
def test_task_ls_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.task_command.parse(["task", "ls" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_stop_success(self):
# 1) Test stop with task_id
# 2) Test stop with all
self.__set_variables()
test_command = ["sh", "-c", "echo yo"]
test_ports = "8888:8888"
test_mem_limit = "4g"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.task_command.parse([
"task", "run", "--ports", test_ports, "--environment-paths",
test_dockerfile, "--mem-limit", test_mem_limit, test_command
])
test_task_obj = self.task_command.execute()
# 1) Test option 1
self.task_command.parse(["task", "stop", "--id", test_task_obj.id])
# test for desired side effects
assert self.task_command.args.id == test_task_obj.id
# test when task id is passed to stop it
task_stop_command = self.task_command.execute()
assert task_stop_command == True
# 2) Test option 2
self.task_command.parse(["task", "stop", "--all"])
# test when all is passed to stop all
task_stop_command = self.task_command.execute()
assert task_stop_command == True
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_stop_failure_required_args(self):
self.__set_variables()
# Passing wrong task id
self.task_command.parse(["task", "stop"])
failed = False
try:
_ = self.task_command.execute()
except RequiredArgumentMissing:
failed = True
assert failed
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_stop_failure_mutually_exclusive_vars(self):
self.__set_variables()
# Passing wrong task id
self.task_command.parse(
["task", "stop", "--id", "invalid-task-id", "--all"])
failed = False
try:
_ = self.task_command.execute()
except MutuallyExclusiveArguments:
failed = True
assert failed
def test_task_stop_failure_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.task_command.parse(["task", "stop" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_task_stop_invalid_task_id(self):
self.__set_variables()
# Passing wrong task id
self.task_command.parse(["task", "stop", "--id", "invalid-task-id"])
# test when wrong task id is passed to stop it
result = self.task_command.execute()
assert not result