class TestSnapshotCommand():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.cli_helper = Helper()
def teardown_method(self):
pass
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.snapshot_command = SnapshotCommand(self.cli_helper)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# Create config file
self.config_filepath = os.path.join(self.snapshot_command.home,
"config.json")
with open(self.config_filepath, "wb") as f:
f.write(to_bytes(str("{}")))
# Create stats file
self.stats_filepath = os.path.join(self.snapshot_command.home,
"stats.json")
with open(self.stats_filepath, "wb") as f:
f.write(to_bytes(str("{}")))
# Create test file
self.filepath = os.path.join(self.snapshot_command.home, "file.txt")
with open(self.filepath, "wb") as f:
f.write(to_bytes(str("test")))
# Create another test file
self.filepath_2 = os.path.join(self.snapshot_command.home, "file2.txt")
with open(self.filepath_2, "wb") as f:
f.write(to_bytes(str("test")))
# Create config
self.config = 'foo:bar'
self.config1 = "{'foo1':'bar1'}"
self.config2 = "this is test config blob"
# Create stats
self.stats = 'foo:bar'
self.stats1 = "{'foo1':'bar1'}"
self.stats2 = "this is test stats blob"
def test_snapshot_help(self):
self.__set_variables()
print(
"\n====================================== datmo snapshot ==========================\n"
)
self.snapshot_command.parse(["snapshot"])
assert self.snapshot_command.execute()
print(
"\n====================================== datmo snapshot --help ==========================\n"
)
self.snapshot_command.parse(["snapshot", "--help"])
assert self.snapshot_command.execute()
print(
"\n====================================== datmo snapshot create --help ==========================\n"
)
self.snapshot_command.parse(["snapshot", "create", "--help"])
assert self.snapshot_command.execute()
def test_snapshot_create(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_run_id = "test_run_id"
test_environment_definition_filepath = self.env_def_path
test_config_filepath = self.config_filepath
test_stats_filepath = self.config_filepath
test_paths = [self.filepath, self.filepath_2]
# try single filepath
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id",
test_run_id
])
# testing for proper parsing
assert self.snapshot_command.args.message == test_message
assert self.snapshot_command.args.run_id == test_run_id
# try single filepath
self.snapshot_command.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--environment-paths",
test_environment_definition_filepath,
"--config-filepath",
test_config_filepath,
"--stats-filepath",
test_stats_filepath,
"--paths",
test_paths[0],
])
# test for desired side effects
assert self.snapshot_command.args.message == test_message
assert self.snapshot_command.args.label == test_label
assert self.snapshot_command.args.environment_paths == [
test_environment_definition_filepath
]
assert self.snapshot_command.args.config_filepath == test_config_filepath
assert self.snapshot_command.args.stats_filepath == test_stats_filepath
assert self.snapshot_command.args.paths == [test_paths[0]]
# test multiple paths
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--environment-paths",
test_environment_definition_filepath, "--config-filepath",
test_config_filepath, "--stats-filepath", test_stats_filepath,
"--paths", test_paths[0], "--paths", test_paths[1]
])
# test for desired side effects
assert self.snapshot_command.args.message == test_message
assert self.snapshot_command.args.label == test_label
assert self.snapshot_command.args.environment_paths == [
test_environment_definition_filepath
]
assert self.snapshot_command.args.config_filepath == test_config_filepath
assert self.snapshot_command.args.stats_filepath == test_stats_filepath
assert self.snapshot_command.args.paths == test_paths
snapshot_obj_1 = self.snapshot_command.execute()
assert snapshot_obj_1
def test_snapshot_create_config_stats(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_config = self.config
test_stats = self.stats
# try config
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--config", test_config, "--stats", test_stats
])
# test for desired side effects
snapshot_obj = self.snapshot_command.execute()
assert snapshot_obj
test_config = self.config1
test_stats = self.stats1
# try config
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--config", test_config, "--stats", test_stats
])
# test for desired side effects
snapshot_obj = self.snapshot_command.execute()
assert snapshot_obj
test_config = self.config2
test_stats = self.stats2
# try config
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--config", test_config, "--stats", test_stats
])
# test for desired side effects
snapshot_obj = self.snapshot_command.execute()
assert snapshot_obj
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_snapshot_create_from_run(self):
self.__set_variables()
test_message = "this is a test message"
# create task
test_command = "sh -c 'echo accuracy:0.45'"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.run = RunCommand(self.cli_helper)
self.run.parse(
["run", "--environment-paths", test_dockerfile, test_command])
# test proper execution of task run command
run_obj = self.run.execute()
run_id = run_obj.id
# test run id
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id", run_id
])
# test for desired side effects
assert self.snapshot_command.args.message == test_message
snapshot_obj = self.snapshot_command.execute()
assert snapshot_obj
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_snapshot_create_from_run_fail_user_inputs(self):
self.__set_variables()
test_message = "this is a test message"
# create run
test_command = "sh -c 'echo accuracy:0.45'"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.run = RunCommand(self.cli_helper)
self.run.parse(
["run", "--environment-paths", test_dockerfile, test_command])
# test proper execution of task run command
run_obj = self.run.execute()
run_id = run_obj.id
failed = False
try:
# test run id with environment-id
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id",
run_id, "--environment-id", "test_environment_id"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test run id with environment-def
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id",
run_id, "--environment-paths", "test_environment_path"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test run id with filepaths
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id",
run_id, "--paths", "mypath"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test run id with config-filepath
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id",
run_id, "--config-filepath", "mypath"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test run id with config-filename
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id",
run_id, "--config-filename", "myname"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test run id with stats-filepath
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id",
run_id, "--stats-filepath", "mypath"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test run id with stats-filename
self.snapshot_command.parse([
"snapshot", "create", "--message", test_message, "--run-id",
run_id, "--stats-filename", "myname"
])
_ = self.snapshot_command.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
def test_snapshot_create_fail_mutually_exclusive_args(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_environment_id = "test_environment_id"
test_environment_definition_filepath = self.env_def_path
test_config_filename = "config.json"
test_config_filepath = self.config_filepath
test_stats_filename = "stats.json"
test_stats_filepath = self.config_filepath
# Environment exception
exception_thrown = False
try:
self.snapshot_command.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--environment-id",
test_environment_id,
"--environment-paths",
test_environment_definition_filepath,
])
_ = self.snapshot_command.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Config exception
exception_thrown = False
try:
self.snapshot_command.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--config-filename",
test_config_filename,
"--config-filepath",
test_config_filepath,
])
_ = self.snapshot_command.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Stats exception
exception_thrown = False
try:
self.snapshot_command.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--stats-filename",
test_stats_filename,
"--stats-filepath",
test_stats_filepath,
])
_ = self.snapshot_command.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
def test_snapshot_create_default(self):
self.__set_variables()
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj_2 = self.snapshot_command.execute()
assert snapshot_obj_2
def test_snapshot_create_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot_command.parse(
["snapshot", "create"
"--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_snapshot_update(self):
self.__set_variables()
test_config = ["depth: 10", "learning_rate:0.91"]
test_stats = ["acc: 91.34", "f1_score:0.91"]
test_config1 = "{'foo_config': 'bar_config'}"
test_stats1 = "{'foo_stats': 'bar_stats'}"
test_config2 = "this is a config blob"
test_stats2 = "this is a stats blob"
test_message = "test_message"
test_label = "test_label"
# 1. Updating both message and label
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse([
"snapshot", "update", snapshot_obj.id, "--message", test_message,
"--label", test_label
])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.message == test_message
assert result.label == test_label
# 2. Updating only message
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse(
["snapshot", "update", snapshot_obj.id, "--message", test_message])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.message == test_message
# Updating label
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse(
["snapshot", "update", snapshot_obj.id, "--label", test_label])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.label == test_label
# 3. Updating config, message and label
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse([
"snapshot", "update", snapshot_obj.id, "--config", test_config[0],
"--config", test_config[1], "--message", test_message, "--label",
test_label
])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.config == {"depth": "10", "learning_rate": "0.91"}
assert result.message == test_message
assert result.label == test_label
# 4. Updating stats, message and label
# Test when optional parameters are not given
self.snapshot_command.parse([
"snapshot", "update", snapshot_obj.id, "--stats", test_stats[0],
"--stats", test_stats[1], "--message", test_message, "--label",
test_label
])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.stats == {"acc": "91.34", "f1_score": "0.91"}
assert result.message == test_message
assert result.label == test_label
# Adding sleep due to issue with consistency in blitzdb database
time.sleep(2)
# 5. Updating config, stats
# Test when optional parameters are not given
self.snapshot_command.parse([
"snapshot", "update", snapshot_obj.id, "--config", test_config1,
"--stats", test_stats1
])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.config == {
"depth": "10",
"learning_rate": "0.91",
'foo_config': 'bar_config'
}
assert result.stats == {
"acc": "91.34",
"f1_score": "0.91",
'foo_stats': 'bar_stats'
}
assert result.message == test_message
assert result.label == test_label
# Test when optional parameters are not given
self.snapshot_command.parse([
"snapshot", "update", snapshot_obj.id, "--config", test_config2,
"--stats", test_stats2
])
result = self.snapshot_command.execute()
assert result.id == snapshot_obj.id
assert result.config == {
"depth": "10",
"learning_rate": "0.91",
'foo_config': 'bar_config',
'config': test_config2
}
assert result.stats == {
"acc": "91.34",
"f1_score": "0.91",
'foo_stats': 'bar_stats',
'stats': test_stats2
}
assert result.message == test_message
assert result.label == test_label
def test_snapshot_delete(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse(["snapshot", "delete", snapshot_obj.id])
result = self.snapshot_command.execute()
assert result
def test_snapshot_delete_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot_command.parse(
["snapshot", "delete"
"--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_snapshot_ls_invisible(self):
self.__set_variables()
# Test invisible snapshot from task
# create task
test_command = "sh -c 'echo accuracy:0.45'"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.run = RunCommand(self.cli_helper)
self.run.parse(
["run", "--environment-paths", test_dockerfile, test_command])
# test proper execution of task run command
task_obj = self.run.execute()
# test no visible snapshots
self.snapshot_command.parse(["snapshot", "ls"])
result = self.snapshot_command.execute()
assert not result
# test invisible snapshot from task_id was created
self.snapshot_command.parse(["snapshot", "ls", "--all"])
result = self.snapshot_command.execute()
assert result
assert task_obj.after_snapshot_id in [obj.id for obj in result]
def test_snapshot_ls(self):
self.__set_variables()
# create a visible snapshot
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
created_snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse(["snapshot", "ls"])
result = self.snapshot_command.execute()
assert result
assert created_snapshot_obj in result
# Check if current snapshot id is the same as the latest created id
current_snapshot_obj = self.snapshot_command.snapshot_controller.current_snapshot(
)
current_snapshot_id = current_snapshot_obj.id if current_snapshot_obj else None
assert current_snapshot_id == created_snapshot_obj.id
# Test when optional parameters are not given
self.snapshot_command.parse(["snapshot", "ls", "--details"])
result = self.snapshot_command.execute()
assert result
assert created_snapshot_obj in result
# Test failure (format)
failed = False
try:
self.snapshot_command.parse(["snapshot", "ls", "--format"])
except ArgumentError:
failed = True
assert failed
# Test success format csv
self.snapshot_command.parse(["snapshot", "ls", "--format", "csv"])
snapshot_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
# Test success format csv, download default
self.snapshot_command.parse(
["snapshot", "ls", "--format", "csv", "--download"])
snapshot_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
test_wildcard = os.path.join(
self.snapshot_command.snapshot_controller.home, "snapshot_ls_*")
paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
assert paths
assert open(paths[0], "r").read()
os.remove(paths[0])
# Test success format csv, download exact path
test_path = os.path.join(self.temp_dir, "my_output")
self.snapshot_command.parse([
"snapshot", "ls", "--format", "csv", "--download",
"--download-path", test_path
])
snapshot_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
assert os.path.isfile(test_path)
assert open(test_path, "r").read()
os.remove(test_path)
# Test success format table
self.snapshot_command.parse(["snapshot", "ls"])
environment_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
# Test success format table, download default
self.snapshot_command.parse(["snapshot", "ls", "--download"])
snapshot_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
test_wildcard = os.path.join(
self.snapshot_command.snapshot_controller.home, "snapshot_ls_*")
paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
assert paths
assert open(paths[0], "r").read()
os.remove(paths[0])
# Test success format table, download exact path
test_path = os.path.join(self.temp_dir, "my_output")
self.snapshot_command.parse(
["snapshot", "ls", "--download", "--download-path", test_path])
snapshot_objs = self.snapshot_command.execute()
assert created_snapshot_obj in snapshot_objs
assert os.path.isfile(test_path)
assert open(test_path, "r").read()
os.remove(test_path)
def test_snapshot_checkout_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot_command.parse(
["snapshot", "checkout"
"--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_snapshot_checkout(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test when optional parameters are not given
self.snapshot_command.parse(["snapshot", "checkout", snapshot_obj.id])
result = self.snapshot_command.execute()
assert result
def test_snapshot_diff(self):
self.__set_variables()
# Create config file
with open(self.config_filepath, "wb") as f:
f.write(to_bytes(str('{"depth":6}')))
# Create stats file
with open(self.stats_filepath, "wb") as f:
f.write(to_bytes(str('{"acc":0.97}')))
# Create snapshots to test
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj_1 = self.snapshot_command.execute()
# Create another test file
self.filepath_3 = os.path.join(self.snapshot_command.home, "file3.txt")
with open(self.filepath_3, "wb") as f:
f.write(to_bytes(str("test")))
# Create config file
with open(self.config_filepath, "wb") as f:
f.write(to_bytes(str('{"depth":5}')))
# Create stats file
with open(self.stats_filepath, "wb") as f:
f.write(to_bytes(str('{"acc":0.91}')))
self.snapshot_command.parse(
["snapshot", "create", "-m", "my second snapshot"])
snapshot_obj_2 = self.snapshot_command.execute()
# Test diff with the above two snapshots
self.snapshot_command.parse(
["snapshot", "diff", snapshot_obj_1.id, snapshot_obj_2.id])
result = self.snapshot_command.execute()
assert result
assert "message" in result
assert "label" in result
assert "code_id" in result
assert "environment_id" in result
assert "file_collection_id" in result
assert "config" in result
assert "stats" in result
def test_snapshot_inspect(self):
self.__set_variables()
# Create snapshot to test
self.snapshot_command.parse(
["snapshot", "create", "-m", "my test snapshot"])
snapshot_obj = self.snapshot_command.execute()
# Test inspect for this snapshot
self.snapshot_command.parse(["snapshot", "inspect", snapshot_obj.id])
result = self.snapshot_command.execute()
assert result
assert "Code" in result
assert "Environment" in result
assert "Files" in result
assert "Config" in result
assert "Stats" in result
class TestFlow():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.cli_helper = Helper()
def teardown_method(self):
pass
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.environment_command = EnvironmentCommand(self.cli_helper)
self.run_command = RunCommand(self.cli_helper)
self.snapshot_command = SnapshotCommand(self.cli_helper)
# Create test file
self.filepath = os.path.join(self.snapshot_command.home, "file.txt")
with open(self.filepath, "wb") as f:
f.write(to_bytes(str("test")))
def __environment_setup(self):
self.environment_command.parse(["environment", "setup"])
@self.environment_command.cli_helper.input("\n\n\n")
def dummy(self):
return self.environment_command.execute()
environment_setup_result = dummy(self)
return environment_setup_result
def __run(self):
test_command = ["sh", "-c", "echo accuracy:0.45"]
test_ports = ["8888:8888"]
self.run_command.parse(["run", "-p", test_ports[0], test_command])
run_result = self.run_command.execute()
return run_result
def __run_ls(self):
self.run_command.parse(["ls"])
run_ls_result = self.run_command.execute()
return run_ls_result
def __snapshot_create(self):
test_message = "creating a snapshot"
self.snapshot_command.parse(["snapshot", "create", "-m", test_message])
snapshot_create_result = self.snapshot_command.execute()
return snapshot_create_result
def __snapshot_ls(self):
self.snapshot_command.parse(["snapshot", "ls"])
snapshot_ls_result = self.snapshot_command.execute()
return snapshot_ls_result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_flow_1(self):
# Flow
# Step 1: environment setup
# Step 2: run
# Step 3: ls
# Step 4: snapshot create
# Step 5: snapshot ls
self.__set_variables()
# Step 1: environment setup
environment_setup_result = self.__environment_setup()
assert environment_setup_result
# Step 2: run command
run_result = self.__run()
assert run_result
# Step 3: ls
run_ls_result = self.__run_ls()
assert run_ls_result
# Step 4: snapshot create
snapshot_create_result = self.__snapshot_create()
assert snapshot_create_result
# Step 5: snapshot ls
snapshot_ls_result = self.__snapshot_ls()
assert snapshot_ls_result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_flow_2(self):
# Flow interruption in environment
# Step 1: interrupted environment setup
# Step 2: environment setup
# Step 3: run
# Step 4: ls
# Step 5: snapshot create
# Step 6: snapshot ls
self.__set_variables()
# TODO: TEST more fundamental functions first
# Step 1: interrupted environment setup
# @timeout_decorator.timeout(0.0001, use_signals=False)
# def timed_command_with_interuption():
# result = self.__environment_setup()
# return result
#
# failed = False
# try:
# timed_command_with_interuption()
# except timeout_decorator.timeout_decorator.TimeoutError:
# failed = True
# assert failed
# Step 2: environment setup
environment_setup_result = self.__environment_setup()
assert environment_setup_result
# Step 3: run
run_result = self.__run()
assert run_result
# Step 4: ls
run_ls_result = self.__run_ls()
assert run_ls_result
# Step 5: snapshot create
snapshot_create_result = self.__snapshot_create()
assert snapshot_create_result
# Step 6: snapshot ls
snapshot_ls_result = self.__snapshot_ls()
assert snapshot_ls_result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_flow_3(self):
# Flow interruption in run
# Step 1: environment setup
# Step 2: interrupted run
# Step 3: run
# Step 4: ls
# Step 5: snapshot create
# Step 6: snapshot ls
self.__set_variables()
# Step 1: environment setup
environment_setup_result = self.__environment_setup()
assert environment_setup_result
# TODO: Test more fundamental functions first
# Step 2: interrupted run
# @timeout_decorator.timeout(0.0001, use_signals=False)
# def timed_command_with_interuption():
# result = self.__run()
# return result
#
# failed = False
# try:
# timed_command_with_interuption()
# except timeout_decorator.timeout_decorator.TimeoutError:
# failed = True
# assert failed
# Step 3: run
run_result = self.__run()
assert run_result
# Step 4: task ls
run_ls_result = self.__run_ls()
assert run_ls_result
# Step 5: snapshot create
snapshot_create_result = self.__snapshot_create()
assert snapshot_create_result
# Step 6: snapshot ls
snapshot_ls_result = self.__snapshot_ls()
assert snapshot_ls_result
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_flow_4(self):
# Flow interruption in snapshot create
# Step 1: environment setup
# Step 2: run
# Step 3: ls
# Step 4: interrupted snapshot create
# Step 5: snapshot create
# Step 6: snapshot ls
self.__set_variables()
# Step 1: environment setup
environment_setup_result = self.__environment_setup()
assert environment_setup_result
# Step 2: run
run_result = self.__run()
assert run_result
# Step 3: ls
run_ls_result = self.__run_ls()
assert run_ls_result
# TODO: Test more fundamental functions first
# Step 4: interrupted snapshot create
# @timeout_decorator.timeout(0.0001, use_signals=False)
# def timed_command_with_interuption():
# result = self.__snapshot_create()
# return result
#
# failed = False
# try:
# timed_command_with_interuption()
# except timeout_decorator.timeout_decorator.TimeoutError:
# failed = True
# assert failed
# snapshot_ls_result = self.__snapshot_ls()
# assert not snapshot_ls_result
# Step 5: snapshot create
snapshot_create_result = self.__snapshot_create()
assert snapshot_create_result
# Step 6: snapshot ls
snapshot_ls_result = self.__snapshot_ls()
assert snapshot_ls_result
class TestProjectCommand():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.cli_helper = Helper()
self.project_command = ProjectCommand(self.cli_helper)
def teardown_method(self):
pass
def test_init_create_success_default_name_no_description_no_environment(
self):
self.project_command.parse(["init"])
# Test when environment is created
@self.project_command.cli_helper.input("\n\ny\n\n\n\n")
def dummy(self):
return self.project_command.execute()
result = dummy(self)
# Ensure that the name and description are current
_, default_name = os.path.split(
self.project_command.project_controller.home)
assert result
assert result.name == default_name
assert result.description == None
# Ensure environment is correct
definition_filepath = os.path.join(
self.temp_dir, self.project_command.project_controller.file_driver.
environment_directory, "Dockerfile")
assert os.path.isfile(definition_filepath)
assert "FROM datmo/python-base:cpu-py27" in open(
definition_filepath, "r").read()
def test_init_create_success_no_environment(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
# Test when environment is not created
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
result = dummy(self)
definition_filepath = os.path.join(
self.temp_dir, self.project_command.project_controller.file_driver.
environment_directory, "Dockerfile")
assert result
assert not os.path.isfile(definition_filepath)
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result.name == test_name
assert result.description == test_description
def test_init_create_success_environment(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
# Test when environment is created
@self.project_command.cli_helper.input("y\n\n\n\n")
def dummy(self):
return self.project_command.execute()
result = dummy(self)
definition_filepath = os.path.join(
self.temp_dir, self.project_command.project_controller.file_driver.
environment_directory, "Dockerfile")
assert result
assert os.path.isfile(definition_filepath)
assert "FROM datmo/python-base:cpu-py27" in open(
definition_filepath, "r").read()
# test for desired side effects
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result.name == test_name
assert result.description == test_description
def test_init_create_success_blank(self):
self.project_command.parse(["init", "--name", "", "--description", ""])
# test if prompt opens
@self.project_command.cli_helper.input("\n\n\n")
def dummy(self):
return self.project_command.execute()
result = dummy(self)
assert result
assert result.name
assert not result.description
def test_init_update_success(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
result_1 = dummy(self)
updated_name = "foobar2"
updated_description = "test model 2"
self.project_command.parse([
"init", "--name", updated_name, "--description",
updated_description
])
result_2 = dummy(self)
# test for desired side effects
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result_2.id == result_1.id
assert result_2.name == updated_name
assert result_2.description == updated_description
def test_init_update_success_only_name(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
result_1 = dummy(self)
updated_name = "foobar2"
self.project_command.parse(
["init", "--name", updated_name, "--description", ""])
@self.project_command.cli_helper.input("\n\n")
def dummy(self):
return self.project_command.execute()
result_2 = dummy(self)
# test for desired side effects
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result_2.id == result_1.id
assert result_2.name == updated_name
assert result_2.description == result_1.description
def test_init_update_success_only_description(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
result_1 = dummy(self)
updated_description = "test model 2"
self.project_command.parse(
["init", "--name", "", "--description", updated_description])
@self.project_command.cli_helper.input("\n\n")
def dummy(self):
return self.project_command.execute()
result_2 = dummy(self)
# test for desired side effects
assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert result_2.id == result_1.id
assert result_2.name == result_1.name
assert result_2.description == updated_description
def test_init_invalid_arg(self):
exception_thrown = False
try:
self.project_command.parse(["init", "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_version(self):
self.project_command.parse(["version"])
result = self.project_command.execute()
# test for desired side effects
assert __version__ in result
def test_version_invalid_arg(self):
exception_thrown = False
try:
self.project_command.parse(["version", "--foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_status_basic(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
_ = dummy(self)
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = result
assert isinstance(status_dict, dict)
assert not current_snapshot
assert not latest_snapshot_user_generated
assert not latest_snapshot_auto_generated
assert unstaged_code
assert not unstaged_environment
assert not unstaged_files
def test_status_user_generated_snapshot(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
_ = dummy(self)
# Create a snapshot
self.snapshot_command = SnapshotCommand(self.cli_helper)
with open(os.path.join(self.project_command.home, "test.py"),
"wb") as f:
f.write(to_bytes(str("import xgboost")))
self.snapshot_command.parse(
["snapshot", "create", "--message", "test"])
snapshot_obj = self.snapshot_command.execute()
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = result
assert isinstance(status_dict, dict)
assert isinstance(current_snapshot, Snapshot)
assert isinstance(latest_snapshot_user_generated, Snapshot)
assert snapshot_obj == latest_snapshot_user_generated
assert current_snapshot == latest_snapshot_user_generated
assert not latest_snapshot_auto_generated
assert not unstaged_code
assert not unstaged_environment
assert not unstaged_files
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_status_autogenerated_snapshot(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
_ = dummy(self)
self.run_command = RunCommand(self.cli_helper)
self.task_controller = TaskController()
# Create and run a task and test if task is shown
test_command = ["sh", "-c", "echo accuracy:0.45"]
self.run_command.parse(["run", test_command])
updated_first_task = self.run_command.execute()
before_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_first_task.before_snapshot_id)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_first_task.after_snapshot_id)
before_environment_obj = self.task_controller.dal.environment.get_by_id(
before_snapshot_obj.environment_id)
after_environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
assert before_environment_obj == after_environment_obj
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
result
assert status_dict
assert isinstance(status_dict, dict)
assert status_dict['name'] == test_name
assert status_dict['description'] == test_description
assert isinstance(status_dict['config'], dict)
assert isinstance(current_snapshot, Snapshot)
assert not latest_snapshot_user_generated
assert isinstance(latest_snapshot_auto_generated, Snapshot)
# current snapshot is the before snapshot for the run
assert current_snapshot == before_snapshot_obj
assert current_snapshot != latest_snapshot_auto_generated
# latest autogenerated snapshot is the after snapshot id
assert latest_snapshot_auto_generated == after_snapshot_obj
# autogenerated snapshot is after the user generated snapshot
assert not unstaged_code
assert not unstaged_environment
assert not unstaged_files
# Create a snapshot
self.snapshot_command = SnapshotCommand(self.cli_helper)
with open(os.path.join(self.project_command.home, "test.py"),
"wb") as f:
f.write(to_bytes(str("import xgboost")))
self.snapshot_command.parse(
["snapshot", "create", "--message", "test"])
first_snapshot = self.snapshot_command.execute()
# Create and run a task and test if task is shown
test_command = ["sh", "-c", "echo accuracy:0.45"]
self.run_command.parse(["run", test_command])
updated_second_task = self.run_command.execute()
before_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_second_task.before_snapshot_id)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_second_task.after_snapshot_id)
before_environment_obj = self.task_controller.dal.environment.get_by_id(
before_snapshot_obj.environment_id)
after_environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
assert before_environment_obj == after_environment_obj
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
result
assert status_dict
assert isinstance(status_dict, dict)
assert status_dict['name'] == test_name
assert status_dict['description'] == test_description
assert isinstance(status_dict['config'], dict)
assert isinstance(current_snapshot, Snapshot)
assert isinstance(latest_snapshot_user_generated, Snapshot)
assert latest_snapshot_user_generated == first_snapshot
assert isinstance(latest_snapshot_auto_generated, Snapshot)
# current snapshot is the before snapshot for the run
assert current_snapshot == before_snapshot_obj
assert current_snapshot == latest_snapshot_user_generated
assert current_snapshot != latest_snapshot_auto_generated
# latest autogenerated snapshot is the after snapshot id
assert latest_snapshot_auto_generated == after_snapshot_obj
assert latest_snapshot_auto_generated != latest_snapshot_user_generated
# user generated snapshot is not associated with any before or after snapshot
assert latest_snapshot_user_generated == before_snapshot_obj
assert latest_snapshot_user_generated != after_snapshot_obj
# autogenerated snapshot is after the user generated snapshot
assert latest_snapshot_auto_generated.created_at > latest_snapshot_user_generated.created_at
assert not unstaged_code
assert not unstaged_environment
assert not unstaged_files
# Check if the same snapshot is given back when created
self.snapshot_command.parse(
["snapshot", "create", "--message", "test"])
new_snapshot = self.snapshot_command.execute()
assert current_snapshot == new_snapshot
# Create a user generated snapshot after some changes
with open(os.path.join(self.project_command.home, "new.py"),
"wb") as f:
f.write(to_bytes(str("import xgboost")))
self.snapshot_command.parse(
["snapshot", "create", "--message", "test"])
new_snapshot = self.snapshot_command.execute()
# ensure we created a new snapshot
assert current_snapshot != new_snapshot
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
result
assert status_dict
assert isinstance(status_dict, dict)
assert status_dict['name'] == test_name
assert status_dict['description'] == test_description
assert isinstance(status_dict['config'], dict)
assert isinstance(current_snapshot, Snapshot)
assert isinstance(latest_snapshot_user_generated, Snapshot)
# current snapshot is the latest user generated one
assert current_snapshot == latest_snapshot_user_generated
# latest autogenerated snapshot is not the user generated one
assert latest_snapshot_auto_generated != latest_snapshot_user_generated
# autogenerated snapshot is after the user generated snapshot
assert latest_snapshot_user_generated.created_at > latest_snapshot_auto_generated.created_at
assert not unstaged_code
assert not unstaged_environment
assert not unstaged_files
# Create a snapshot from the auto generated one and ensure they are the same
self.run_command = RunCommand(self.cli_helper)
self.task_controller = TaskController()
test_command = ["sh", "-c", "echo accuracy:0.45"]
self.run_command.parse(["run", test_command])
updated_third_task = self.run_command.execute()
before_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_third_task.before_snapshot_id)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_third_task.after_snapshot_id)
self.snapshot_command.parse([
"snapshot", "create", "--run-id", updated_third_task.id,
"--message", "test"
])
converted_snapshot = self.snapshot_command.execute()
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
result
assert status_dict
assert isinstance(status_dict, dict)
assert status_dict['name'] == test_name
assert status_dict['description'] == test_description
assert isinstance(status_dict['config'], dict)
assert isinstance(current_snapshot, Snapshot)
assert isinstance(latest_snapshot_user_generated, Snapshot)
assert isinstance(latest_snapshot_auto_generated, Snapshot)
# current snapshot is the before snapshot for the run
assert current_snapshot == before_snapshot_obj
assert current_snapshot != latest_snapshot_user_generated
# latest user generated snapshot is the same as that created by the run
assert converted_snapshot == latest_snapshot_user_generated
assert latest_snapshot_user_generated == after_snapshot_obj
# latest user generated converted the latest task so latest generate is from the last task
assert latest_snapshot_user_generated.created_at > latest_snapshot_auto_generated.created_at
assert not unstaged_code
assert not unstaged_environment
assert not unstaged_files
def test_status_invalid_arg(self):
exception_thrown = False
try:
self.project_command.parse(["status", "--foobar"])
except UnrecognizedCLIArgument:
exception_thrown = True
assert exception_thrown
def test_cleanup(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
_ = dummy(self)
self.project_command.parse(["cleanup"])
@self.project_command.cli_helper.input("y\n\n")
def dummy(self):
return self.project_command.execute()
result = dummy(self)
assert not os.path.exists(os.path.join(self.temp_dir, '.datmo'))
assert isinstance(result, bool)
assert result
def test_cleanup_invalid_arg(self):
exception_thrown = False
try:
self.project_command.parse(["cleanup", "--foobar"])
except UnrecognizedCLIArgument:
exception_thrown = True
assert exception_thrown
class TestSnapshot():
def setup_class(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.cli_helper = Helper()
def teardown_class(self):
pass
def __set_variables(self):
self.project = ProjectCommand(self.temp_dir, self.cli_helper)
self.project.parse(
["init", "--name", "foobar", "--description", "test model"])
self.project.execute()
self.snapshot = SnapshotCommand(self.temp_dir, self.cli_helper)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
# Create config
self.config_filepath = os.path.join(self.snapshot.home, "config.json")
with open(self.config_filepath, "w") as f:
f.write(to_unicode(str("{}")))
# Create stats
self.stats_filepath = os.path.join(self.snapshot.home, "stats.json")
with open(self.stats_filepath, "w") as f:
f.write(to_unicode(str("{}")))
# Create test file
self.filepath = os.path.join(self.snapshot.home, "file.txt")
with open(self.filepath, "w") as f:
f.write(to_unicode(str("test")))
# Create another test file
self.filepath_2 = os.path.join(self.snapshot.home, "file2.txt")
with open(self.filepath_2, "w") as f:
f.write(to_unicode(str("test")))
def test_snapshot_project_not_init(self):
failed = False
try:
self.snapshot = SnapshotCommand(self.temp_dir, self.cli_helper)
except ProjectNotInitializedException:
failed = True
assert failed
def test_snapshot_help(self):
self.__set_variables()
print(
"\n====================================== datmo snapshot ==========================\n"
)
self.snapshot.parse(["snapshot"])
assert self.snapshot.execute()
print(
"\n====================================== datmo snapshot --help ==========================\n"
)
self.snapshot.parse(["snapshot", "--help"])
assert self.snapshot.execute()
print(
"\n====================================== datmo snapshot create --help ==========================\n"
)
self.snapshot.parse(["snapshot", "create", "--help"])
assert self.snapshot.execute()
def test_snapshot_command(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_session_id = "test_session_id"
test_task_id = "test_task_id"
test_code_id = "test_code_id"
test_environment_definition_filepath = self.env_def_path
test_config_filepath = self.config_filepath
test_stats_filepath = self.config_filepath
test_filepaths = [self.filepath, self.filepath_2]
# try single filepath
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
test_task_id
])
# testing for proper parsing
assert self.snapshot.args.message == test_message
assert self.snapshot.args.task_id == test_task_id
# try single filepath
self.snapshot.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--code-id",
test_code_id,
"--environment-def",
test_environment_definition_filepath,
"--config-filepath",
test_config_filepath,
"--stats-filepath",
test_stats_filepath,
"--filepaths",
test_filepaths[0],
])
# test for desired side effects
assert self.snapshot.args.message == test_message
assert self.snapshot.args.label == test_label
assert self.snapshot.args.session_id == test_session_id
assert self.snapshot.args.code_id == test_code_id
assert self.snapshot.args.environment_definition_filepath == test_environment_definition_filepath
assert self.snapshot.args.config_filepath == test_config_filepath
assert self.snapshot.args.stats_filepath == test_stats_filepath
assert self.snapshot.args.filepaths == [test_filepaths[0]]
# test multiple filepaths
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--session-id", test_session_id, "--code-id",
test_code_id, "--environment-def",
test_environment_definition_filepath, "--config-filepath",
test_config_filepath, "--stats-filepath", test_stats_filepath,
"--filepaths", test_filepaths[0], "--filepaths", test_filepaths[1]
])
# test for desired side effects
assert self.snapshot.args.message == test_message
assert self.snapshot.args.label == test_label
assert self.snapshot.args.session_id == test_session_id
assert self.snapshot.args.code_id == test_code_id
assert self.snapshot.args.environment_definition_filepath == test_environment_definition_filepath
assert self.snapshot.args.config_filepath == test_config_filepath
assert self.snapshot.args.stats_filepath == test_stats_filepath
assert self.snapshot.args.filepaths == test_filepaths
snapshot_id_1 = self.snapshot.execute()
assert snapshot_id_1
def test_datmo_snapshot_create_from_task(self):
self.__set_variables()
test_message = "this is a test message"
test_code_id = "test_code_id"
# create task
test_command = "sh -c 'echo accuracy:0.45'"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.task = TaskCommand(self.temp_dir, self.cli_helper)
self.task.parse([
"task", "run", "--environment-def", test_dockerfile, test_command
])
# test proper execution of task run command
task_obj = self.task.execute()
task_id = task_obj.id
# test task id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id
])
# test for desired side effects
assert self.snapshot.args.message == test_message
snapshot_id = self.snapshot.execute()
assert snapshot_id
def test_snapshot_create_from_task_fail_user_inputs(self):
self.__set_variables()
test_message = "this is a test message"
# create task
test_command = "sh -c 'echo accuracy:0.45'"
test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
self.task = TaskCommand(self.temp_dir, self.cli_helper)
self.task.parse([
"task", "run", "--environment-def", test_dockerfile, test_command
])
# test proper execution of task run command
task_obj = self.task.execute()
task_id = task_obj.id
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--code-id", "test_code_id"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--commit-id", "test_commit_id"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--environment-id", "test_environment_id"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--environment-def", "test_environment_def"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--file-collection-id", "test_file_collection_id"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--filepaths", "mypath"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--config-filepath", "mypath"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--config-filename", "myname"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--stats-filepath", "mypath"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
# test task id with code id
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--task-id",
task_id, "--stats-filename", "myname"
])
_ = self.snapshot.execute()
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
def test_datmo_snapshot_create_fail_mutually_exclusive_args(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_session_id = "test_session_id"
test_code_id = "test_code_id"
test_commit_id = "test_commit_id"
test_environment_id = "test_environment_id"
test_environment_definition_filepath = self.env_def_path
test_file_collection_id = "test_file_collection_id"
test_filepaths = [self.filepath]
test_config_filename = "config.json"
test_config_filepath = self.config_filepath
test_stats_filename = "stats.json"
test_stats_filepath = self.config_filepath
# Code exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot", "create", "--message", test_message, "--label",
test_label, "--session-id", test_session_id, "--code-id",
test_code_id, "--commit-id", test_commit_id
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Environment exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--environment-id",
test_environment_id,
"--environment-def",
test_environment_definition_filepath,
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# File exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--file-collection-id",
test_file_collection_id,
"--filepaths",
test_filepaths[0],
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Config exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--config-filename",
test_config_filename,
"--config-filepath",
test_config_filepath,
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Stats exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message",
test_message,
"--label",
test_label,
"--session-id",
test_session_id,
"--stats-filename",
test_stats_filename,
"--stats-filepath",
test_stats_filepath,
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_create_default(self):
self.__set_variables()
self.snapshot.parse(["snapshot", "create", "-m", "my test snapshot"])
snapshot_id_2 = self.snapshot.execute()
assert snapshot_id_2
def test_datmo_snapshot_create_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot.parse(["snapshot", "create" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_delete(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "create", "-m", "my test snapshot"])
snapshot_id = self.snapshot.execute()
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "delete", "--id", snapshot_id])
result = self.snapshot.execute()
assert result
def test_datmo_snapshot_delete_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot.parse(["snapshot", "delete" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_ls(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "create", "-m", "my test snapshot"])
snapshot_id = self.snapshot.execute()
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "ls"])
result = self.snapshot.execute()
assert result
assert snapshot_id in result
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "ls", "-a"])
result = self.snapshot.execute()
assert result
assert snapshot_id in result
def test_datmo_snapshot_checkout_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot.parse(["snapshot", "checkout" "--foobar", "foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_checkout(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "create", "-m", "my test snapshot"])
snapshot_id = self.snapshot.execute()
# remove datmo_task folder to have no changes before checkout
datmo_tasks_dirpath = os.path.join(self.snapshot.home, "datmo_tasks")
shutil.rmtree(datmo_tasks_dirpath)
# Test when optional parameters are not given
self.snapshot.parse(["snapshot", "checkout", "--id", snapshot_id])
result = self.snapshot.execute()
assert result
class TestSnapshot():
def setup_class(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system() == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.cli_helper = Helper()
def teardown_class(self):
pass
def __set_variables(self):
self.init = ProjectCommand(self.temp_dir, self.cli_helper)
self.init.parse([
"init",
"--name", "foobar",
"--description", "test model"])
self.init.execute()
self.snapshot = SnapshotCommand(self.temp_dir, self.cli_helper)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir,
"Dockerfile")
with open(self.env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
# Create config
self.config_filepath = os.path.join(self.snapshot.home,
"config.json")
with open(self.config_filepath, "w") as f:
f.write(to_unicode(str("{}")))
# Create stats
self.stats_filepath = os.path.join(self.snapshot.home,
"stats.json")
with open(self.stats_filepath, "w") as f:
f.write(to_unicode(str("{}")))
# Create test file
self.filepath = os.path.join(self.snapshot.home,
"file.txt")
with open(self.filepath, "w") as f:
f.write(to_unicode(str("test")))
def test_snapshot_project_not_init(self):
failed = False
try:
self.snapshot = SnapshotCommand(self.temp_dir, self.cli_helper)
except ProjectNotInitializedException:
failed = True
assert failed
def test_datmo_snapshot_create(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_session_id = "test_session_id"
test_task_id = "test_task_id"
test_code_id = "test_code_id"
test_environment_def_path = self.env_def_path
test_config_filepath = self.config_filepath
test_stats_filepath = self.config_filepath
test_filepaths = [self.filepath]
self.snapshot.parse([
"snapshot",
"create",
"--message", test_message,
"--label", test_label,
"--session-id", test_session_id,
"--task-id", test_task_id,
"--code-id", test_code_id,
"--environment-def-path", test_environment_def_path,
"--config-filepath", test_config_filepath,
"--stats-filepath", test_stats_filepath,
"--filepaths", test_filepaths[0],
])
# test for desired side effects
assert self.snapshot.args.message == test_message
assert self.snapshot.args.label == test_label
assert self.snapshot.args.session_id == test_session_id
assert self.snapshot.args.task_id == test_task_id
assert self.snapshot.args.code_id == test_code_id
assert self.snapshot.args.environment_def_path == test_environment_def_path
assert self.snapshot.args.config_filepath == test_config_filepath
assert self.snapshot.args.stats_filepath == test_stats_filepath
assert self.snapshot.args.filepaths == test_filepaths
snapshot_id_1 = self.snapshot.execute()
assert snapshot_id_1
def test_datmo_snapshot_create_fail_mutually_exclusive_args(self):
self.__set_variables()
test_message = "this is a test message"
test_label = "test label"
test_session_id = "test_session_id"
test_task_id = "test_task_id"
test_code_id = "test_code_id"
test_commit_id = "test_commit_id"
test_environment_id = "test_environment_id"
test_environment_def_path = self.env_def_path
test_file_collection_id = "test_file_collection_id"
test_filepaths = [self.filepath]
test_config_filename = "config.json"
test_config_filepath = self.config_filepath
test_stats_filename = "stats.json"
test_stats_filepath = self.config_filepath
# Code exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message", test_message,
"--label", test_label,
"--session-id", test_session_id,
"--task-id", test_task_id,
"--code-id", test_code_id,
"--commit-id", test_commit_id
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Environment exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message", test_message,
"--label", test_label,
"--session-id", test_session_id,
"--task-id", test_task_id,
"--environment-id", test_environment_id,
"--environment-def-path", test_environment_def_path,
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# File exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message", test_message,
"--label", test_label,
"--session-id", test_session_id,
"--task-id", test_task_id,
"--file-collection-id", test_file_collection_id,
"--filepaths", test_filepaths[0],
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Config exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message", test_message,
"--label", test_label,
"--session-id", test_session_id,
"--task-id", test_task_id,
"--config-filename", test_config_filename,
"--config-filepath", test_config_filepath,
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
# Stats exception
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create",
"--message", test_message,
"--label", test_label,
"--session-id", test_session_id,
"--task-id", test_task_id,
"--stats-filename", test_stats_filename,
"--stats-filepath", test_stats_filepath,
])
_ = self.snapshot.execute()
except MutuallyExclusiveArguments:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_create_default(self):
self.__set_variables()
self.snapshot.parse([
"snapshot",
"create",
"-m", "my test snapshot"
])
snapshot_id_2 = self.snapshot.execute()
assert snapshot_id_2
def test_datmo_snapshot_create_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"create"
"--foobar","foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_delete(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot.parse([
"snapshot",
"create",
"-m", "my test snapshot"
])
snapshot_id = self.snapshot.execute()
# Test when optional parameters are not given
self.snapshot.parse([
"snapshot",
"delete",
"--id", snapshot_id
])
result = self.snapshot.execute()
assert result
def test_datmo_snapshot_delete_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"delete"
"--foobar","foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_ls(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot.parse([
"snapshot",
"create",
"-m", "my test snapshot"
])
self.snapshot.execute()
# Test when optional parameters are not given
self.snapshot.parse([
"snapshot",
"ls"
])
result = self.snapshot.execute()
assert result
# Test when optional parameters are not given
self.snapshot.parse([
"snapshot",
"ls", "-a"
])
result = self.snapshot.execute()
assert result
def test_datmo_snapshot_checkout_invalid_arg(self):
self.__set_variables()
exception_thrown = False
try:
self.snapshot.parse([
"snapshot",
"checkout"
"--foobar","foobar"])
except Exception:
exception_thrown = True
assert exception_thrown
def test_datmo_snapshot_checkout(self):
self.__set_variables()
# Test when optional parameters are not given
self.snapshot.parse([
"snapshot",
"create",
"-m", "my test snapshot"
])
snapshot_id = self.snapshot.execute()
# Test when optional parameters are not given
self.snapshot.parse([
"snapshot",
"checkout",
"--id", snapshot_id
])
result = self.snapshot.execute()
assert result