def test_snapshot_project_not_init(self):
failed = False
try:
self.snapshot = SnapshotCommand(self.temp_dir, self.cli_helper)
except ProjectNotInitializedException:
failed = True
assert failed
def __set_variables(self):
self.project = ProjectCommand(self.temp_dir, self.cli_helper)
self.project.parse(
["init", "--name", "foobar", "--description", "test model"])
self.project.execute()
self.snapshot = SnapshotCommand(self.temp_dir, self.cli_helper)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
# Create config
self.config_filepath = os.path.join(self.snapshot.home, "config.json")
with open(self.config_filepath, "w") as f:
f.write(to_unicode(str("{}")))
# Create stats
self.stats_filepath = os.path.join(self.snapshot.home, "stats.json")
with open(self.stats_filepath, "w") as f:
f.write(to_unicode(str("{}")))
# Create test file
self.filepath = os.path.join(self.snapshot.home, "file.txt")
with open(self.filepath, "w") as f:
f.write(to_unicode(str("test")))
# Create another test file
self.filepath_2 = os.path.join(self.snapshot.home, "file2.txt")
with open(self.filepath_2, "w") as f:
f.write(to_unicode(str("test")))
def test_status_user_generated_snapshot(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
_ = dummy(self)
# Create a snapshot
self.snapshot_command = SnapshotCommand(self.cli_helper)
with open(os.path.join(self.project_command.home, "test.py"),
"wb") as f:
f.write(to_bytes(str("import xgboost")))
self.snapshot_command.parse(
["snapshot", "create", "--message", "test"])
snapshot_obj = self.snapshot_command.execute()
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = result
assert isinstance(status_dict, dict)
assert isinstance(current_snapshot, Snapshot)
assert isinstance(latest_snapshot_user_generated, Snapshot)
assert snapshot_obj == latest_snapshot_user_generated
assert current_snapshot == latest_snapshot_user_generated
assert not latest_snapshot_auto_generated
assert not unstaged_code
assert not unstaged_environment
assert not unstaged_files
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.snapshot_command = SnapshotCommand(self.cli_helper)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# Create config file
self.config_filepath = os.path.join(self.snapshot_command.home,
"config.json")
with open(self.config_filepath, "wb") as f:
f.write(to_bytes(str("{}")))
# Create stats file
self.stats_filepath = os.path.join(self.snapshot_command.home,
"stats.json")
with open(self.stats_filepath, "wb") as f:
f.write(to_bytes(str("{}")))
# Create test file
self.filepath = os.path.join(self.snapshot_command.home, "file.txt")
with open(self.filepath, "wb") as f:
f.write(to_bytes(str("test")))
# Create another test file
self.filepath_2 = os.path.join(self.snapshot_command.home, "file2.txt")
with open(self.filepath_2, "wb") as f:
f.write(to_bytes(str("test")))
# Create config
self.config = 'foo:bar'
self.config1 = "{'foo1':'bar1'}"
self.config2 = "this is test config blob"
# Create stats
self.stats = 'foo:bar'
self.stats1 = "{'foo1':'bar1'}"
self.stats2 = "this is test stats blob"
def __set_variables(self):
self.project_command = ProjectCommand(self.cli_helper)
self.project_command.parse(
["init", "--name", "foobar", "--description", "test model"])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
dummy(self)
self.environment_command = EnvironmentCommand(self.cli_helper)
self.run_command = RunCommand(self.cli_helper)
self.snapshot_command = SnapshotCommand(self.cli_helper)
# Create test file
self.filepath = os.path.join(self.snapshot_command.home, "file.txt")
with open(self.filepath, "wb") as f:
f.write(to_bytes(str("test")))
def test_status_autogenerated_snapshot(self):
test_name = "foobar"
test_description = "test model"
self.project_command.parse(
["init", "--name", test_name, "--description", test_description])
@self.project_command.cli_helper.input("\n")
def dummy(self):
return self.project_command.execute()
_ = dummy(self)
self.run_command = RunCommand(self.cli_helper)
self.task_controller = TaskController()
# Create and run a task and test if task is shown
test_command = ["sh", "-c", "echo accuracy:0.45"]
self.run_command.parse(["run", test_command])
updated_first_task = self.run_command.execute()
before_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_first_task.before_snapshot_id)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_first_task.after_snapshot_id)
before_environment_obj = self.task_controller.dal.environment.get_by_id(
before_snapshot_obj.environment_id)
after_environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
assert before_environment_obj == after_environment_obj
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
result
assert status_dict
assert isinstance(status_dict, dict)
assert status_dict['name'] == test_name
assert status_dict['description'] == test_description
assert isinstance(status_dict['config'], dict)
assert isinstance(current_snapshot, Snapshot)
assert not latest_snapshot_user_generated
assert isinstance(latest_snapshot_auto_generated, Snapshot)
# current snapshot is the before snapshot for the run
assert current_snapshot == before_snapshot_obj
assert current_snapshot != latest_snapshot_auto_generated
# latest autogenerated snapshot is the after snapshot id
assert latest_snapshot_auto_generated == after_snapshot_obj
# autogenerated snapshot is after the user generated snapshot
assert not unstaged_code
assert not unstaged_environment
assert not unstaged_files
# Create a snapshot
self.snapshot_command = SnapshotCommand(self.cli_helper)
with open(os.path.join(self.project_command.home, "test.py"),
"wb") as f:
f.write(to_bytes(str("import xgboost")))
self.snapshot_command.parse(
["snapshot", "create", "--message", "test"])
first_snapshot = self.snapshot_command.execute()
# Create and run a task and test if task is shown
test_command = ["sh", "-c", "echo accuracy:0.45"]
self.run_command.parse(["run", test_command])
updated_second_task = self.run_command.execute()
before_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_second_task.before_snapshot_id)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_second_task.after_snapshot_id)
before_environment_obj = self.task_controller.dal.environment.get_by_id(
before_snapshot_obj.environment_id)
after_environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
assert before_environment_obj == after_environment_obj
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
result
assert status_dict
assert isinstance(status_dict, dict)
assert status_dict['name'] == test_name
assert status_dict['description'] == test_description
assert isinstance(status_dict['config'], dict)
assert isinstance(current_snapshot, Snapshot)
assert isinstance(latest_snapshot_user_generated, Snapshot)
assert latest_snapshot_user_generated == first_snapshot
assert isinstance(latest_snapshot_auto_generated, Snapshot)
# current snapshot is the before snapshot for the run
assert current_snapshot == before_snapshot_obj
assert current_snapshot == latest_snapshot_user_generated
assert current_snapshot != latest_snapshot_auto_generated
# latest autogenerated snapshot is the after snapshot id
assert latest_snapshot_auto_generated == after_snapshot_obj
assert latest_snapshot_auto_generated != latest_snapshot_user_generated
# user generated snapshot is not associated with any before or after snapshot
assert latest_snapshot_user_generated == before_snapshot_obj
assert latest_snapshot_user_generated != after_snapshot_obj
# autogenerated snapshot is after the user generated snapshot
assert latest_snapshot_auto_generated.created_at > latest_snapshot_user_generated.created_at
assert not unstaged_code
assert not unstaged_environment
assert not unstaged_files
# Check if the same snapshot is given back when created
self.snapshot_command.parse(
["snapshot", "create", "--message", "test"])
new_snapshot = self.snapshot_command.execute()
assert current_snapshot == new_snapshot
# Create a user generated snapshot after some changes
with open(os.path.join(self.project_command.home, "new.py"),
"wb") as f:
f.write(to_bytes(str("import xgboost")))
self.snapshot_command.parse(
["snapshot", "create", "--message", "test"])
new_snapshot = self.snapshot_command.execute()
# ensure we created a new snapshot
assert current_snapshot != new_snapshot
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
result
assert status_dict
assert isinstance(status_dict, dict)
assert status_dict['name'] == test_name
assert status_dict['description'] == test_description
assert isinstance(status_dict['config'], dict)
assert isinstance(current_snapshot, Snapshot)
assert isinstance(latest_snapshot_user_generated, Snapshot)
# current snapshot is the latest user generated one
assert current_snapshot == latest_snapshot_user_generated
# latest autogenerated snapshot is not the user generated one
assert latest_snapshot_auto_generated != latest_snapshot_user_generated
# autogenerated snapshot is after the user generated snapshot
assert latest_snapshot_user_generated.created_at > latest_snapshot_auto_generated.created_at
assert not unstaged_code
assert not unstaged_environment
assert not unstaged_files
# Create a snapshot from the auto generated one and ensure they are the same
self.run_command = RunCommand(self.cli_helper)
self.task_controller = TaskController()
test_command = ["sh", "-c", "echo accuracy:0.45"]
self.run_command.parse(["run", test_command])
updated_third_task = self.run_command.execute()
before_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_third_task.before_snapshot_id)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_third_task.after_snapshot_id)
self.snapshot_command.parse([
"snapshot", "create", "--run-id", updated_third_task.id,
"--message", "test"
])
converted_snapshot = self.snapshot_command.execute()
self.project_command.parse(["status"])
result = self.project_command.execute()
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
result
assert status_dict
assert isinstance(status_dict, dict)
assert status_dict['name'] == test_name
assert status_dict['description'] == test_description
assert isinstance(status_dict['config'], dict)
assert isinstance(current_snapshot, Snapshot)
assert isinstance(latest_snapshot_user_generated, Snapshot)
assert isinstance(latest_snapshot_auto_generated, Snapshot)
# current snapshot is the before snapshot for the run
assert current_snapshot == before_snapshot_obj
assert current_snapshot != latest_snapshot_user_generated
# latest user generated snapshot is the same as that created by the run
assert converted_snapshot == latest_snapshot_user_generated
assert latest_snapshot_user_generated == after_snapshot_obj
# latest user generated converted the latest task so latest generate is from the last task
assert latest_snapshot_user_generated.created_at > latest_snapshot_auto_generated.created_at
assert not unstaged_code
assert not unstaged_environment
assert not unstaged_files