def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.project = ProjectController(self.temp_dir)
self.project.init("test", "test description")
self.snapshot = SnapshotController(self.temp_dir)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "w") as f:
f.write(str("FROM datmo/xgboost:cpu"))
# Create config
self.config_filepath = os.path.join(self.snapshot.home, "config.json")
with open(self.config_filepath, "w") as f:
f.write(str('{"foo":1}'))
# Create stats
self.stats_filepath = os.path.join(self.snapshot.home, "stats.json")
with open(self.stats_filepath, "w") as f:
f.write(str('{"bar":1}'))
# Create test file
self.filepath = os.path.join(self.snapshot.home, "file.txt")
with open(self.filepath, "w") as f:
f.write(str("test"))
def __init__(self, home, cli_helper):
super(ProjectCommand, self).__init__(home, cli_helper)
init_parser = self.subparsers.add_parser("init",
help="Initialize project")
init_parser.add_argument("--name", default=None)
init_parser.add_argument("--description", default=None)
self.project_controller = ProjectController(home=home)
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system() == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.project = ProjectController(self.temp_dir)
def teardown_method(self):
if not check_docker_inactive(test_datmo_dir):
self.project_controller = ProjectController()
if self.project_controller.is_initialized:
self.environment_controller = EnvironmentController()
for env_id in list(set(self.environment_ids)):
if not self.environment_controller.delete(env_id):
raise Exception
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = '/tmp'
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.project = ProjectController(self.temp_dir)
self.project.init("test", "test description")
self.session = SessionController(self.temp_dir)
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.project_controller = ProjectController()
_ = self.project_controller.init("test", "test description")
self.input_dict = {
"id": "test",
"model_id": "my_model",
"session_id": "my_session",
"command": "python test.py"
}
class TestProjectController():
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system() == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.project = ProjectController(self.temp_dir)
def teardown_method(self):
pass
def test_init(self):
# Test failed case
failed = False
try:
self.project.init(None, None)
except RequiredArgumentMissing:
failed = True
assert failed
result = self.project.init("test", "test description")
# Tested with is_initialized
assert self.project.model.name == "test"
assert self.project.model.description == "test description"
assert self.project.code_driver.is_initialized
assert self.project.file_driver.is_initialized
assert self.project.environment_driver.is_initialized
assert result and self.project.is_initialized
# Changeable by user, not tested in is_initialized
assert self.project.current_session.name == "default"
# Check Project template if user specified template
# TODO: Add in Project template if user specifies
# Test out functionality for re-initialize project
result = self.project.init("anything", "else")
assert self.project.model.name == "anything"
assert self.project.model.description == "else"
assert result == True
def test_cleanup(self):
self.project.init("test", "test description")
result = self.project.cleanup()
assert not self.project.code_driver.exists_code_refs_dir()
assert not self.project.file_driver.exists_datmo_file_structure()
assert not self.project.environment_driver.list_images("datmo-test2")
# Ensure that containers built with this image do not exist
# assert not self.project.environment_driver.list_containers(filters={
# "ancestor": image_id
# })
assert result == True
def test_status(self):
pass
class ProjectCommand(BaseCommand):
# NOTE: dal_driver is not passed into the project because it is created
# first by ProjectController and then passed down to all other Controllers
def __init__(self, home, cli_helper):
super(ProjectCommand, self).__init__(home, cli_helper)
init_parser = self.subparsers.add_parser("init",
help="Initialize project")
init_parser.add_argument("--name", default=None)
init_parser.add_argument("--description", default=None)
self.project_controller = ProjectController(home=home)
def init(self, name, description):
if not name:
name = self.cli_helper.prompt(__("prompt",
"cli.project.init.name"))
if not description:
description = self.cli_helper.prompt(
__("prompt", "cli.project.init.description"))
self.cli_helper.echo(
__("info", "cli.project.init", {
"name": name,
"path": self.home
}))
self.project_controller.init(name, description)
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
_ = ProjectController().init("test", "test description")
self.input_dict = {
"id": "test",
"model_id": "my_model",
"message": "my message",
"code_id": "my_code_id",
"environment_id": "my_environment_id",
"file_collection_id": "my file collection",
"config": {
"test": 0.56
},
"stats": {
"test": 0.34
}
}
class TestTaskController():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.environment_ids = []
def teardown_method(self):
if not check_docker_inactive(test_datmo_dir,
Config().datmo_directory_name):
self.__setup()
self.environment_controller = EnvironmentController()
for env_id in list(set(self.environment_ids)):
if not self.environment_controller.delete(env_id):
raise Exception
def __setup(self):
Config().set_home(self.temp_dir)
self.project_controller = ProjectController()
self.project_controller.init("test", "test description")
self.environment_controller = EnvironmentController()
self.task_controller = TaskController()
def test_init_fail_project_not_init(self):
Config().set_home(self.temp_dir)
failed = False
try:
TaskController()
except ProjectNotInitialized:
failed = True
assert failed
def test_init_fail_invalid_path(self):
test_home = "some_random_dir"
Config().set_home(test_home)
failed = False
try:
TaskController()
except InvalidProjectPath:
failed = True
assert failed
def test_create(self):
self.__setup()
# Create task in the project
task_obj = self.task_controller.create()
assert isinstance(task_obj, Task)
assert task_obj.created_at
assert task_obj.updated_at
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_run_helper(self):
self.__setup()
# TODO: Try out more options (see below)
# Create environment_driver id
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
paths = [env_def_path]
environment_obj = self.environment_controller.create({"paths": paths})
self.environment_ids.append(environment_obj.id)
# Set log filepath
log_filepath = os.path.join(self.task_controller.home, "test.log")
# create volume to mount
temp_test_dirpath = os.path.join(self.temp_dir, "temp")
os.makedirs(temp_test_dirpath)
# Test option set 1
random_name = str(uuid.uuid1())
options_dict = {
"command": ["sh", "-c", "echo accuracy:0.45"],
"ports": ["8888:8888"],
"name": random_name,
"volumes": {
temp_test_dirpath: {
'bind': '/task/',
'mode': 'rw'
}
},
"mem_limit": "4g",
"detach": False,
"stdin_open": False,
"tty": False,
"api": False,
"interactive": False
}
return_code, run_id, logs = \
self.task_controller._run_helper(environment_obj.id,
options_dict, log_filepath)
assert return_code == 0
assert run_id and \
self.task_controller.environment_driver.get_container(run_id)
assert logs and \
os.path.exists(log_filepath)
self.task_controller.environment_driver.stop_remove_containers_by_term(
term=random_name)
# Test option set 2
random_name_2 = str(uuid.uuid1())
options_dict = {
"command": ["sh", "-c", "echo accuracy:0.45"],
"ports": ["8888:8888"],
"name": random_name_2,
"volumes": {
temp_test_dirpath: {
'bind': '/task/',
'mode': 'rw'
}
},
"mem_limit": "4g",
"detach": True,
"stdin_open": False,
"tty": False,
"api": True,
"interactive": False
}
return_code, run_id, logs = \
self.task_controller._run_helper(environment_obj.id,
options_dict, log_filepath)
assert return_code == 0
assert run_id and \
self.task_controller.environment_driver.get_container(run_id)
assert logs and \
os.path.exists(log_filepath)
self.task_controller.environment_driver.stop_remove_containers_by_term(
term=random_name_2)
def test_parse_logs_for_results(self):
self.__setup()
test_logs = """
this is a log
accuracy is good
accuracy : 0.94
this did not work
validation : 0.32
model_type : logistic regression
"""
result = self.task_controller._parse_logs_for_results(test_logs)
assert isinstance(result, dict)
assert result['accuracy'] == "0.94"
assert result['validation'] == "0.32"
assert result['model_type'] == "logistic regression"
test_logs = """test"""
result = self.task_controller._parse_logs_for_results(test_logs)
assert result is None
def test_update_environment_run_options(self):
self.__setup()
environment_run_option = {
"command": ["python", "script.py"],
"volumes": {
os.path.join(self.temp_dir, "/temp_task"): {
'bind': '/task/',
'mode': 'rw'
},
self.temp_dir: {
'bind': '/home/',
'mode': 'rw'
}
}
}
# create data file
data_dirpath = os.path.join(self.temp_dir, "data")
data_file_dirpath = os.path.join(self.temp_dir, "data_folder")
data_filepath = os.path.join(data_file_dirpath, "data_file.txt")
os.mkdir(data_dirpath)
os.mkdir(data_file_dirpath)
with open(data_filepath, "wb") as f:
f.write(to_bytes("data file"))
data_file_path_map = [(data_filepath, "data_file.txt")]
data_directory_path_map = [(data_dirpath, "data_directory")]
environment_run_option = self.task_controller._update_environment_run_options(
environment_run_option, data_file_path_map,
data_directory_path_map)
assert environment_run_option["volumes"][data_file_dirpath] == {
'bind': '/data/',
'mode': 'rw'
}
assert environment_run_option["volumes"][data_dirpath] == {
'bind': '/data/data_directory',
'mode': 'rw'
}
# Error by passing directory which does not exist
data_dirpath = os.path.join(self.temp_dir, "data_dne")
data_filepath = os.path.join(self.temp_dir, "data_dne",
"data_file.txt")
data_file_path_map = [(data_filepath, "data_file.txt")]
data_directory_path_map = [(data_dirpath, "data_directory")]
failed = False
try:
self.task_controller._update_environment_run_options(
environment_run_option, data_file_path_map,
data_directory_path_map)
except TaskRunError:
failed = True
assert failed
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_run(self):
self.__setup()
# 0) Test failure case without command and without interactive
# 1) Test success case with default values and env def file
# 2) Test failure case if running same task (conflicting containers)
# 3) Test failure case if running same task with snapshot_dict (conflicting containers)
# 4) Test success case with snapshot_dict
# 5) Test success case with saved file during task run
# 6) Test success case with data file path being passed
# 7) Test success case with data directory path being passed
# TODO: look into log filepath randomness, sometimes logs are not written
# Create task in the project
task_obj = self.task_controller.create()
# Create environment definition
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# 0) Test option 0
failed = False
try:
self.task_controller.run(task_obj.id)
except RequiredArgumentMissing:
failed = True
assert failed
failed = False
try:
self.task_controller.run(
task_obj.id,
task_dict={
"command": None,
"interactive": False,
"ports": None
})
except RequiredArgumentMissing:
failed = True
assert failed
# Create task_dict
task_command = ["sh", "-c", "echo accuracy:0.45"]
task_dict = {"command_list": task_command}
# 1) Test option 1
updated_task_obj = self.task_controller.run(
task_obj.id, task_dict=task_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
assert isinstance(updated_task_obj, Task)
assert task_obj.id == updated_task_obj.id
assert updated_task_obj.before_snapshot_id
assert updated_task_obj.ports == None
assert updated_task_obj.interactive == False
assert updated_task_obj.task_dirpath
assert updated_task_obj.log_filepath
assert updated_task_obj.start_time
assert updated_task_obj.after_snapshot_id
assert updated_task_obj.run_id
assert updated_task_obj.logs
assert "accuracy" in updated_task_obj.logs
assert updated_task_obj.results
assert updated_task_obj.results == {"accuracy": "0.45"}
assert after_snapshot_obj.stats == {"accuracy": "0.45"}
assert updated_task_obj.status == "SUCCESS"
assert updated_task_obj.end_time
assert updated_task_obj.duration
self.task_controller.stop(task_obj.id)
# 2) Test option 2
failed = False
try:
self.task_controller.run(task_obj.id)
except TaskRunError:
failed = True
assert failed
# 3) Test option 3
# Create files to add
self.project_controller.file_driver.create("dirpath1", directory=True)
self.project_controller.file_driver.create("dirpath2", directory=True)
self.project_controller.file_driver.create("filepath1")
# Snapshot dictionary
snapshot_dict = {
"paths": [
os.path.join(self.project_controller.home, "dirpath1"),
os.path.join(self.project_controller.home, "dirpath2"),
os.path.join(self.project_controller.home, "filepath1")
],
}
# Run a basic task in the project
failed = False
try:
self.task_controller.run(task_obj.id, snapshot_dict=snapshot_dict)
except TaskRunError:
failed = True
assert failed
# Test when the specific task id is already RUNNING
# Create task in the project
task_obj_1 = self.task_controller.create()
self.task_controller.dal.task.update({
"id": task_obj_1.id,
"status": "RUNNING"
})
# Create environment_driver definition
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
failed = False
try:
self.task_controller.run(task_obj_1.id, task_dict=task_dict)
except TaskRunError:
failed = True
assert failed
# 4) Test option 4
# Create a new task in the project
task_obj_2 = self.task_controller.create()
# Run another task in the project
updated_task_obj_2 = self.task_controller.run(
task_obj_2.id, task_dict=task_dict, snapshot_dict=snapshot_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj_2.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
assert isinstance(updated_task_obj_2, Task)
assert task_obj_2.id == updated_task_obj_2.id
assert updated_task_obj_2.before_snapshot_id
assert updated_task_obj_2.ports == None
assert updated_task_obj_2.interactive == False
assert updated_task_obj_2.task_dirpath
assert updated_task_obj_2.log_filepath
assert updated_task_obj_2.start_time
assert updated_task_obj_2.after_snapshot_id
assert updated_task_obj_2.run_id
assert updated_task_obj_2.logs
assert "accuracy" in updated_task_obj_2.logs
assert updated_task_obj_2.results
assert updated_task_obj_2.results == {"accuracy": "0.45"}
assert updated_task_obj_2.status == "SUCCESS"
assert updated_task_obj_2.end_time
assert updated_task_obj_2.duration
self.task_controller.stop(task_obj_2.id)
# 5) Test option 5
# Create a basic script
test_filepath = os.path.join(self.temp_dir, "script.py")
with open(test_filepath, "wb") as f:
f.write(to_bytes("import os\n"))
f.write(to_bytes("import shutil\n"))
f.write(to_bytes("print('hello')\n"))
f.write(to_bytes("print(' accuracy: 0.56 ')\n"))
f.write(
to_bytes(
"with open(os.path.join('/task', 'new_file.txt'), 'a') as f:\n"
))
f.write(to_bytes(" f.write('my test file')\n"))
# Create task in the project
task_obj_2 = self.task_controller.create()
# Create task_dict
task_command = ["python", "script.py"]
task_dict = {"command_list": task_command}
# Create environment definition
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
updated_task_obj_2 = self.task_controller.run(
task_obj_2.id, task_dict=task_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj_2.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
assert isinstance(updated_task_obj_2, Task)
assert updated_task_obj_2.before_snapshot_id
assert updated_task_obj_2.ports == None
assert updated_task_obj_2.interactive == False
assert updated_task_obj_2.task_dirpath
assert updated_task_obj_2.log_filepath
assert updated_task_obj_2.start_time
assert updated_task_obj_2.after_snapshot_id
assert updated_task_obj_2.run_id
assert updated_task_obj_2.logs
assert "accuracy" in updated_task_obj_2.logs
assert updated_task_obj_2.results
assert updated_task_obj_2.results == {"accuracy": "0.56"}
assert updated_task_obj_2.status == "SUCCESS"
assert updated_task_obj_2.end_time
assert updated_task_obj_2.duration
self.task_controller.stop(task_obj_2.id)
# test if after snapshot has the file written
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj_2.after_snapshot_id)
file_collection_obj = self.task_controller.dal.file_collection.get_by_id(
after_snapshot_obj.file_collection_id)
files_absolute_path = os.path.join(self.task_controller.home,
file_collection_obj.path)
assert os.path.isfile(os.path.join(files_absolute_path, "task.log"))
assert os.path.isfile(
os.path.join(files_absolute_path, "new_file.txt"))
# 6) Test Option 6
self.project_controller.file_driver.create("dirpath1", directory=True)
self.project_controller.file_driver.create(
os.path.join("dirpath1", "file.txt"))
with open(
os.path.join(self.project_controller.home, "dirpath1",
"file.txt"), "wb") as f:
f.write(to_bytes('my initial line\n'))
test_filename = "script.py"
test_filepath = os.path.join(self.temp_dir, test_filename)
with open(test_filepath, "wb") as f:
f.write(to_bytes("import os\n"))
f.write(to_bytes("print('hello')\n"))
f.write(to_bytes("import shutil\n"))
f.write(
to_bytes(
"with open(os.path.join('/data', 'file.txt'), 'a') as f:\n"
))
f.write(to_bytes(" f.write('my test file')\n"))
# Create task in the project
task_obj_3 = self.task_controller.create()
# Create task_dict
task_command = ["python", test_filename]
task_dict = {
"command_list":
task_command,
"data_file_path_map": [(os.path.join(self.project_controller.home,
"dirpath1", "file.txt"),
'file.txt')]
}
# Create environment definition
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
updated_task_obj_3 = self.task_controller.run(
task_obj_3.id, task_dict=task_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj_3.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
assert isinstance(updated_task_obj_3, Task)
assert updated_task_obj_3.before_snapshot_id
assert updated_task_obj_3.ports == None
assert updated_task_obj_3.interactive == False
assert updated_task_obj_3.task_dirpath
assert updated_task_obj_3.log_filepath
assert updated_task_obj_3.start_time
assert updated_task_obj_3.after_snapshot_id
assert updated_task_obj_3.run_id
assert updated_task_obj_3.logs
assert updated_task_obj_3.status == "SUCCESS"
assert updated_task_obj_3.end_time
assert updated_task_obj_3.duration
self.task_controller.stop(task_obj_3.id)
# test if after snapshot has the file written
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj_3.after_snapshot_id)
file_collection_obj = self.task_controller.dal.file_collection.get_by_id(
after_snapshot_obj.file_collection_id)
files_absolute_path = os.path.join(self.task_controller.home,
file_collection_obj.path)
assert os.path.isfile(os.path.join(files_absolute_path, "task.log"))
assert os.path.isfile(
os.path.join(self.project_controller.home, "dirpath1", "file.txt"))
assert "my initial line" in open(
os.path.join(self.project_controller.home, "dirpath1", "file.txt"),
"r").read()
assert "my test file" in open(
os.path.join(self.project_controller.home, "dirpath1", "file.txt"),
"r").read()
# 7) Test Option 7
self.project_controller.file_driver.create("dirpath1", directory=True)
self.project_controller.file_driver.create(
os.path.join("dirpath1", "file.txt"))
with open(
os.path.join(self.project_controller.home, "dirpath1",
"file.txt"), "wb") as f:
f.write(to_bytes('my initial line\n'))
test_filename = "script.py"
test_filepath = os.path.join(self.temp_dir, test_filename)
with open(test_filepath, "wb") as f:
f.write(to_bytes("import os\n"))
f.write(to_bytes("print('hello')\n"))
f.write(to_bytes("import shutil\n"))
f.write(
to_bytes(
"with open(os.path.join('/data', 'dirpath1', 'file.txt'), 'a') as f:\n"
))
f.write(to_bytes(" f.write('my test file')\n"))
# Create task in the project
task_obj_4 = self.task_controller.create()
# Create task_dict
task_command = ["python", test_filename]
task_dict = {
"command_list":
task_command,
"data_directory_path_map": [(os.path.join(
self.project_controller.home, "dirpath1"), 'dirpath1')]
}
# Create environment definition
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
updated_task_obj_4 = self.task_controller.run(
task_obj_4.id, task_dict=task_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj_4.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
assert isinstance(updated_task_obj_4, Task)
assert updated_task_obj_4.before_snapshot_id
assert updated_task_obj_4.ports == None
assert updated_task_obj_4.interactive == False
assert updated_task_obj_4.task_dirpath
assert updated_task_obj_4.log_filepath
assert updated_task_obj_4.start_time
assert updated_task_obj_4.after_snapshot_id
assert updated_task_obj_4.run_id
assert updated_task_obj_4.logs
assert updated_task_obj_4.status == "SUCCESS"
assert updated_task_obj_4.end_time
assert updated_task_obj_4.duration
self.task_controller.stop(task_obj_4.id)
# test if after snapshot has the file written
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj_4.after_snapshot_id)
file_collection_obj = self.task_controller.dal.file_collection.get_by_id(
after_snapshot_obj.file_collection_id)
files_absolute_path = os.path.join(self.task_controller.home,
file_collection_obj.path)
assert os.path.isfile(os.path.join(files_absolute_path, "task.log"))
assert os.path.isfile(
os.path.join(self.project_controller.home, "dirpath1", "file.txt"))
assert "my initial line" in open(
os.path.join(self.project_controller.home, "dirpath1", "file.txt"),
"r").read()
assert "my test file" in open(
os.path.join(self.project_controller.home, "dirpath1", "file.txt"),
"r").read()
def test_list(self):
self.__setup()
# Create tasks in the project
task_obj_1 = self.task_controller.create()
task_obj_2 = self.task_controller.create()
# List all tasks regardless of filters
result = self.task_controller.list()
assert len(result) == 2 and \
task_obj_1 in result and \
task_obj_2 in result
# List all tasks regardless of filters in ascending
result = self.task_controller.list(
sort_key='created_at', sort_order='ascending')
assert len(result) == 2 and \
task_obj_1 in result and \
task_obj_2 in result
assert result[0].created_at <= result[-1].created_at
# List all tasks regardless of filters in descending
result = self.task_controller.list(
sort_key='created_at', sort_order='descending')
assert len(result) == 2 and \
task_obj_1 in result and \
task_obj_2 in result
assert result[0].created_at >= result[-1].created_at
# Wrong order being passed in
failed = False
try:
_ = self.task_controller.list(
sort_key='created_at', sort_order='wrong_order')
except InvalidArgumentType:
failed = True
assert failed
# Wrong key and order being passed in
failed = False
try:
_ = self.task_controller.list(
sort_key='wrong_key', sort_order='wrong_order')
except InvalidArgumentType:
failed = True
assert failed
# wrong key and right order being passed in
expected_result = self.task_controller.list(
sort_key='created_at', sort_order='ascending')
result = self.task_controller.list(
sort_key='wrong_key', sort_order='ascending')
expected_ids = [item.id for item in expected_result]
ids = [item.id for item in result]
assert set(expected_ids) == set(ids)
# List all tasks
result = self.task_controller.list()
assert len(result) == 2 and \
task_obj_1 in result and \
task_obj_2 in result
def test_get(self):
self.__setup()
# Test failure for no task
failed = False
try:
self.task_controller.get("random")
except DoesNotExist:
failed = True
assert failed
# Test success for task
task_obj = self.task_controller.create()
task_obj_returned = self.task_controller.get(task_obj.id)
assert task_obj == task_obj_returned
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_get_files(self):
self.__setup()
# Test failure case
failed = False
try:
self.task_controller.get_files("random")
except DoesNotExist:
failed = True
assert failed
# Create task in the project
task_obj = self.task_controller.create()
# Create environment definition
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# Create file to add
self.project_controller.file_driver.create("dirpath1", directory=True)
self.project_controller.file_driver.create(
os.path.join("dirpath1", "filepath1"))
# Snapshot dictionary
snapshot_dict = {
"paths": [
os.path.join(self.project_controller.home, "dirpath1",
"filepath1")
],
}
# Create task_dict
task_command = ["sh", "-c", "echo accuracy:0.45"]
task_dict = {"command_list": task_command}
# Test the default values
updated_task_obj = self.task_controller.run(
task_obj.id, task_dict=task_dict, snapshot_dict=snapshot_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
# TODO: Test case for during run and before_snapshot run
# Get files for the task after run is complete (default)
result = self.task_controller.get_files(updated_task_obj.id)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj.after_snapshot_id)
file_collection_obj = self.task_controller.dal.file_collection.get_by_id(
after_snapshot_obj.file_collection_id)
file_names = [item.name for item in result]
assert len(result) == 2
for item in result:
assert isinstance(item, TextIOWrapper)
assert item.mode == "r"
assert os.path.join(self.task_controller.home, ".datmo", "collections",
file_collection_obj.filehash,
"task.log") in file_names
assert os.path.join(self.task_controller.home, ".datmo", "collections",
file_collection_obj.filehash,
"filepath1") in file_names
# Get files for the task after run is complete for different mode
result = self.task_controller.get_files(updated_task_obj.id, mode="a")
assert len(result) == 2
for item in result:
assert isinstance(item, TextIOWrapper)
assert item.mode == "a"
assert os.path.join(self.task_controller.home, ".datmo", "collections",
file_collection_obj.filehash,
"task.log") in file_names
assert os.path.join(self.task_controller.home, ".datmo", "collections",
file_collection_obj.filehash,
"filepath1") in file_names
self.task_controller.stop(task_obj.id)
def test_update(self):
self.__setup()
# Create task in the project
task_obj = self.task_controller.create()
assert isinstance(task_obj, Task)
# Test 1: When no meta data is passed
updated_task_obj = self.task_controller.update(task_obj.id)
assert updated_task_obj.workspace is None
# Test 2: When meta data for workspace is passed
test_workspace = "notebook"
test_command = "python script.py"
updated_task_obj = self.task_controller.update(
task_obj.id, workspace=test_workspace, command=test_command)
assert updated_task_obj.workspace == test_workspace
assert updated_task_obj.command == test_command
assert updated_task_obj.command_list == ["python", "script.py"]
# Test 3: When meta data for workspace is passed
test_interactive = True
updated_task_obj = self.task_controller.update(
task_obj.id, interactive=test_interactive)
assert updated_task_obj.interactive == test_interactive
# Test 4: When meta data for workspace is passed
test_command_list = ["python", "script.py"]
updated_task_obj = self.task_controller.update(
task_obj.id, command_list=test_command_list)
assert updated_task_obj.command_list == ["python", "script.py"]
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_delete(self):
self.__setup()
# Create tasks in the project
task_obj = self.task_controller.create()
# Delete task from the project
result = self.task_controller.delete(task_obj.id)
# Check if task retrieval throws error
thrown = False
try:
self.task_controller.dal.snapshot.get_by_id(task_obj.id)
except EntityNotFound:
thrown = True
assert result == True and \
thrown == True
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_stop_failure(self):
self.__setup()
# 1) Test required arguments not provided
# 2) Test too many arguments found
# 3) Test incorrect task id given
# 1) Test option 1
failed = False
try:
self.task_controller.stop()
except RequiredArgumentMissing:
failed = True
assert failed
# 2) Test option 2
failed = False
try:
self.task_controller.stop(task_id="test_task_id", all=True)
except TooManyArgumentsFound:
failed = True
assert failed
# 3) Test option 3
thrown = False
try:
self.task_controller.stop(task_id="incorrect_task_id")
except DoesNotExist:
thrown = True
assert thrown
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_stop_success(self):
self.__setup()
# 1) Test stop with task_id
# 2) Test stop with all given
# Create task in the project
task_obj = self.task_controller.create()
# Create environment driver definition
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# Create task_dict
task_command = ["sh", "-c", "echo accuracy:0.45"]
task_dict = {"command_list": task_command}
# 1) Test option 1
updated_task_obj = self.task_controller.run(
task_obj.id, task_dict=task_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
task_id = updated_task_obj.id
result = self.task_controller.stop(task_id=task_id)
after_task_obj = self.task_controller.dal.task.get_by_id(task_id)
assert result
assert after_task_obj.status == "STOPPED"
# 2) Test option 2
task_obj_2 = self.task_controller.create()
updated_task_obj = self.task_controller.run(
task_obj_2.id, task_dict=task_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
result = self.task_controller.stop(all=True)
all_task_objs = self.task_controller.dal.task.query({})
assert result
for task_obj in all_task_objs:
assert task_obj.status == "STOPPED"
class ProjectCommand(BaseCommand):
def __init__(self, cli_helper):
super(ProjectCommand, self).__init__(cli_helper)
self.project_controller = ProjectController()
def init(self, name, description):
"""Initialize command
Parameters
----------
name : str
name for the project
description : str
description of the project
Returns
-------
datmo.core.entity.model.Model
"""
# Check if project already exists
is_new_model = False
if not self.project_controller.model:
is_new_model = True
if is_new_model: # Initialize a new project
self.cli_helper.echo(
__("info", "cli.project.init.create",
{"path": self.project_controller.home}))
if not name:
_, default_name = os.path.split(self.project_controller.home)
name = self.cli_helper.prompt(__("prompt",
"cli.project.init.name"),
default=default_name)
if not description:
description = self.cli_helper.prompt(
__("prompt", "cli.project.init.description"))
try:
success = self.project_controller.init(name, description)
if success:
self.cli_helper.echo(
__("info", "cli.project.init.create.success", {
"name": name,
"path": self.project_controller.home
}))
except Exception:
self.cli_helper.echo(
__("info", "cli.project.init.create.failure", {
"name": name,
"path": self.project_controller.home
}))
return None
else: # Update the current project
self.cli_helper.echo(
__(
"info", "cli.project.init.update", {
"name": self.project_controller.model.name,
"path": self.project_controller.home
}))
# Prompt for the name and description and add default if not given
if not name:
name = self.cli_helper.prompt(
__("prompt", "cli.project.init.name"),
default=self.project_controller.model.name)
if not description:
description = self.cli_helper.prompt(
__("prompt", "cli.project.init.description"),
default=self.project_controller.model.description)
# Update the project with the values given
try:
success = self.project_controller.init(name, description)
if success:
self.cli_helper.echo(
__("info", "cli.project.init.update.success", {
"name": name,
"path": self.project_controller.home
}))
except Exception:
self.cli_helper.echo(
__("info", "cli.project.init.update.failure", {
"name": name,
"path": self.project_controller.home
}))
return None
self.cli_helper.echo("")
# Print out simple project meta data
for k, v in self.project_controller.model.to_dictionary().items():
if k != "config":
self.cli_helper.echo(str(k) + ": " + str(v))
# Ask question if the user would like to setup environment
environment_setup = self.cli_helper.prompt_bool(
__("prompt", "cli.project.environment.setup"))
if environment_setup:
# TODO: ramove business logic from here and create common helper
# Setting up the environment definition file
self.environment_controller = EnvironmentController()
environment_types = self.environment_controller.get_environment_types(
)
environment_type = self.cli_helper.prompt_available_options(
environment_types, option_type="type")
available_environment_frameworks = self.environment_controller.get_supported_frameworks(
environment_type)
environment_framework = self.cli_helper.prompt_available_options(
available_environment_frameworks, option_type="framework")
available_environment_languages = self.environment_controller.get_supported_languages(
environment_type, environment_framework)
environment_language = self.cli_helper.prompt_available_options(
available_environment_languages, option_type="language")
options = {
"environment_type": environment_type,
"environment_framework": environment_framework,
"environment_language": environment_language
}
environment_obj = self.environment_controller.setup(
options=options)
self.cli_helper.echo(
__("info", "cli.environment.setup.success",
(environment_obj.name, environment_obj.id)))
else:
self.cli_helper.echo(
"there was no environment setup. you can get information"
" here: https://datmo.readthedocs.io/en/latest/env-setup.html")
return self.project_controller.model
def version(self):
return self.cli_helper.echo("datmo version: %s" % __version__)
@Helper.notify_no_project_found
def status(self):
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
self.project_controller.status()
# Print out simple project meta data
for k, v in status_dict.items():
if k != "config":
self.cli_helper.echo(str(k) + ": " + str(v))
self.cli_helper.echo("")
# Print out any unstaged changes else print out the latest snapshot state of the repository
if not unstaged_code and not unstaged_environment and not unstaged_files:
self.cli_helper.echo(
"all changes have been saved, no unstaged changes")
self.cli_helper.echo("")
self.cli_helper.echo("current snapshot state of the repository: ")
if current_snapshot:
self.cli_helper.echo(current_snapshot)
else:
# Print out the unstaged components if unstaged
self.cli_helper.echo("unstaged changes since latest snapshot:")
if unstaged_code:
self.cli_helper.echo("code has been changed")
if unstaged_environment:
self.cli_helper.echo("environment has been changed")
if unstaged_files:
self.cli_helper.echo("files have been changed")
# Print out info for the latest snapshot (the most recent first, and state if autogenerated or by user)
if latest_snapshot_user_generated and not latest_snapshot_auto_generated:
self.cli_helper.echo("latest snapshot generated by the user: "******"no snapshot autogenerated by datmo")
elif latest_snapshot_auto_generated and not latest_snapshot_user_generated:
self.cli_helper.echo("latest snapshot autogenerated by datmo: ")
self.cli_helper.echo(latest_snapshot_auto_generated)
self.cli_helper.echo("no snapshot generated by the user")
elif not latest_snapshot_user_generated and not latest_snapshot_auto_generated:
self.cli_helper.echo("no snapshots created yet")
elif latest_snapshot_user_generated.created_at > latest_snapshot_auto_generated.created_at:
self.cli_helper.echo("latest snapshot generated by the user: "******"latest snapshot autogenerated by datmo: ")
self.cli_helper.echo(latest_snapshot_auto_generated)
elif latest_snapshot_user_generated.created_at < latest_snapshot_auto_generated.created_at:
self.cli_helper.echo("latest snapshot autogenerated by datmo: ")
self.cli_helper.echo(latest_snapshot_auto_generated)
self.cli_helper.echo("latest snapshot generated by the user: "******"prompt", "cli.project.cleanup.confirm"))
# Cleanup datmo project if user specifies
if response:
self.cli_helper.echo(
__(
"info", "cli.project.cleanup", {
"name": self.project_controller.model.name,
"path": self.project_controller.home
}))
try:
success = self.project_controller.cleanup()
if success:
self.cli_helper.echo(
__(
"info", "cli.project.cleanup.success", {
"name": self.project_controller.model.name,
"path": self.project_controller.home
}))
return success
except Exception:
self.cli_helper.echo(
__(
"info", "cli.project.cleanup.failure", {
"name": self.project_controller.model.name,
"path": self.project_controller.home
}))
return False
class TestEnvironmentController():
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.project = ProjectController(self.temp_dir)
self.environment = EnvironmentController(self.temp_dir)
def teardown_method(self):
pass
def test_create(self):
# 0) Test create when unsupported language given
# 1) Test create when NO file exists and NO definition path exists
# 2) Test create when NO file exists and definition path exists
# 3) Test create when definition path exists and given
# 4) Test create when file exists and definition path exists
# 5) Test create when file exists but NO definition path exists
# 6) Test create when definition path exists and given for NEW definition filepath
self.project.init("test3", "test description")
# 0) Test option 0
try:
self.environment.create({"language": "java"})
except EnvironmentDoesNotExist:
failed = True
assert failed
# 1) Test option 1
failed = False
try:
self.environment.create({})
except EnvironmentDoesNotExist:
failed = True
assert failed
# Create environment definition
definition_filepath = os.path.join(self.environment.home, "Dockerfile")
with open(definition_filepath, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
# 2) Test option 2
environment_obj_1 = self.environment.create({})
# Get file collection path
file_collection_obj = self.environment.dal.file_collection. \
get_by_id(environment_obj_1.file_collection_id)
file_collection_dir = self.environment.file_driver. \
get_collection_path(file_collection_obj.filehash)
assert environment_obj_1
assert environment_obj_1.id
assert environment_obj_1.driver_type == "docker"
assert environment_obj_1.file_collection_id
assert environment_obj_1.definition_filename
assert environment_obj_1.hardware_info
assert environment_obj_1.unique_hash == file_collection_obj.filehash
assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile"))
assert os.path.isfile(
os.path.join(file_collection_dir, "datmoDockerfile"))
assert os.path.isfile(
os.path.join(file_collection_dir, "hardware_info"))
# 3) Test option 3
input_dict = {
"definition_filepath": definition_filepath,
}
# Create environment in the project
environment_obj_2 = self.environment.create(input_dict)
# Get file collection path
file_collection_obj = self.environment.dal.file_collection. \
get_by_id(environment_obj_2.file_collection_id)
file_collection_dir = self.environment.file_driver. \
get_collection_path(file_collection_obj.filehash)
assert environment_obj_2
assert environment_obj_2.id
assert environment_obj_2.driver_type == "docker"
assert environment_obj_2.file_collection_id
assert environment_obj_2.definition_filename
assert environment_obj_2.hardware_info
assert environment_obj_2.unique_hash == file_collection_obj.filehash
assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile"))
assert os.path.isfile(
os.path.join(file_collection_dir, "datmoDockerfile"))
assert os.path.isfile(
os.path.join(file_collection_dir, "hardware_info"))
# Create script to test
test_filepath = os.path.join(self.environment.home, "script.py")
with open(test_filepath, "w") as f:
f.write(to_unicode("import numpy\n"))
f.write(to_unicode("import sklearn\n"))
f.write(to_unicode("print('hello')\n"))
# 4) Test option 4
environment_obj_3 = self.environment.create({})
# Get file collection path
file_collection_obj = self.environment.dal.file_collection. \
get_by_id(environment_obj_3.file_collection_id)
file_collection_dir = self.environment.file_driver. \
get_collection_path(file_collection_obj.filehash)
assert environment_obj_3
assert environment_obj_3.id
assert environment_obj_3.driver_type == "docker"
assert environment_obj_3.file_collection_id
assert environment_obj_3.definition_filename
assert environment_obj_3.hardware_info
assert environment_obj_3.unique_hash == file_collection_obj.filehash
assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile"))
assert os.path.isfile(
os.path.join(file_collection_dir, "datmoDockerfile"))
assert os.path.isfile(
os.path.join(file_collection_dir, "hardware_info"))
# Remove definition filepath
os.remove(definition_filepath)
assert environment_obj_1.id == environment_obj_2.id
assert environment_obj_2.id == environment_obj_3.id
# 5) Test option 5
environment_obj_4 = self.environment.create({})
# Get file collection path
file_collection_obj = self.environment.dal.file_collection. \
get_by_id(environment_obj_4.file_collection_id)
file_collection_dir = self.environment.file_driver. \
get_collection_path(file_collection_obj.filehash)
assert environment_obj_4
assert environment_obj_4.id
assert environment_obj_4.driver_type == "docker"
assert environment_obj_4.file_collection_id
assert environment_obj_4.definition_filename
assert environment_obj_4.hardware_info
assert environment_obj_4.unique_hash == file_collection_obj.filehash
assert os.path.isfile(
os.path.join(file_collection_dir, "requirements.txt"))
assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile"))
assert os.path.isfile(
os.path.join(file_collection_dir, "datmoDockerfile"))
assert os.path.isfile(
os.path.join(file_collection_dir, "hardware_info"))
assert environment_obj_1.id != environment_obj_4.id
# 6) Test option 6
# Create environment definition
definition_filepath = os.path.join(self.environment.home, "Dockerfile")
with open(definition_filepath, "w") as f:
f.write(to_unicode(str("FROM cloudgear/ubuntu:14.04")))
input_dict = {
"definition_filepath": definition_filepath,
}
# Create a new environment obj
environment_obj_5 = self.environment.create(input_dict)
# Get file collection path
file_collection_obj = self.environment.dal.file_collection. \
get_by_id(environment_obj_5.file_collection_id)
file_collection_dir = self.environment.file_driver. \
get_collection_path(file_collection_obj.filehash)
assert environment_obj_5
assert environment_obj_5.id
assert environment_obj_5.driver_type == "docker"
assert environment_obj_5.file_collection_id
assert environment_obj_5.definition_filename
assert environment_obj_5.hardware_info
assert environment_obj_5.unique_hash == file_collection_obj.filehash
assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile"))
assert os.path.isfile(
os.path.join(file_collection_dir, "datmoDockerfile"))
assert os.path.isfile(
os.path.join(file_collection_dir, "hardware_info"))
assert environment_obj_5.id != environment_obj_1.id
assert environment_obj_5.id != environment_obj_4.id
def test_build(self):
# 1) Test build when no environment given
# 2) Test build when definition path exists and given
# 3) Test build when NO file exists and definition path exists
# 4) Test build when file exists and definition path exists
# 5) Test build when file exists but NO definition path exists
self.project.init("test5", "test description")
# 1) Test option 1
failed = False
try:
_ = self.environment.build("does_not_exist")
except EntityNotFound:
failed = True
assert failed
# Create environment definition
definition_filepath = os.path.join(self.environment.home, "Dockerfile")
with open(definition_filepath, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
input_dict = {
"definition_filepath": definition_filepath,
}
# 2) Test option 2
# Create environment in the project
environment_obj_1 = self.environment.create(input_dict)
result = self.environment.build(environment_obj_1.id)
assert result
# 3) Test option 3
# Create environment in the project
environment_obj_2 = self.environment.create({})
result = self.environment.build(environment_obj_2.id)
assert result
# Create script to test
test_filepath = os.path.join(self.environment.home, "script.py")
with open(test_filepath, "w") as f:
f.write(to_unicode("import numpy\n"))
f.write(to_unicode("import sklearn\n"))
f.write(to_unicode("print('hello')\n"))
# 4) Test option 4
environment_obj_3 = self.environment.create({})
result = self.environment.build(environment_obj_3.id)
assert result
# test 2), 3), and 4) will result in the same environment
assert environment_obj_1.id == environment_obj_2.id
assert environment_obj_2.id == environment_obj_3.id
# Test for building dockerfile when there exists not
os.remove(definition_filepath)
# 5) Test option 5
environment_obj_4 = self.environment.create({})
result = self.environment.build(environment_obj_4.id)
assert result
assert environment_obj_4.id != environment_obj_1.id
# teardown
self.environment.delete(environment_obj_1.id)
self.environment.delete(environment_obj_4.id)
def test_run(self):
# 1) Test run simple command with simple Dockerfile
# 2) Test run script, with autogenerated definition
self.project.init("test5", "test description")
# 1) Test option 1
# Create environment definition
definition_filepath = os.path.join(self.environment.home, "Dockerfile")
with open(definition_filepath, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
run_options = {
"command": ["sh", "-c", "echo yo"],
"ports": ["8888:8888"],
"name": None,
"volumes": None,
"detach": False,
"stdin_open": False,
"tty": False,
"gpu": False,
"api": False
}
input_dict = {
"definition_filepath": definition_filepath,
}
# Create environment in the project
environment_obj = self.environment.create(input_dict)
log_filepath = os.path.join(self.project.home, "task.log")
# Build environment in the project
_ = self.environment.build(environment_obj.id)
# Run environment in the project
return_code, run_id, logs = \
self.environment.run(environment_obj.id, run_options, log_filepath)
assert return_code == 0
assert run_id
assert logs
# teardown
self.environment.delete(environment_obj.id)
# 2) Test option 2
os.remove(definition_filepath)
# Create script to test
test_filepath = os.path.join(self.environment.home, "script.py")
with open(test_filepath, "w") as f:
f.write(to_unicode("import numpy\n"))
f.write(to_unicode("import sklearn\n"))
f.write(to_unicode("print('hello')\n"))
# Create environment in the project
environment_obj = self.environment.create({})
self.environment.build(environment_obj.id)
run_options = {
"command": ["python", "script.py"],
"ports": ["8888:8888"],
"name": None,
"volumes": {
self.environment.home: {
'bind': '/home/',
'mode': 'rw'
}
},
"detach": False,
"stdin_open": False,
"tty": False,
"gpu": False,
"api": False
}
# Run environment in the project
return_code, run_id, logs = \
self.environment.run(environment_obj.id, run_options, log_filepath)
assert return_code == 0
assert run_id
assert logs
# teardown
self.environment.delete(environment_obj.id)
def test_list(self):
self.project.init("test4", "test description")
# Create environment definition for object 1
definition_path_1 = os.path.join(self.environment.home, "Dockerfile")
with open(definition_path_1, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
input_dict_1 = {
"definition_filepath": definition_path_1,
}
# Create environment in the project
environment_obj_1 = self.environment.create(input_dict_1)
# Create environment definition for object 2
definition_path_2 = os.path.join(self.environment.home, "Dockerfile2")
with open(definition_path_2, "w") as f:
f.write(to_unicode(str("FROM datmo/scikit-opencv")))
input_dict_2 = {
"definition_filepath": definition_path_2,
}
# Create second environment in the project
environment_obj_2 = self.environment.create(input_dict_2)
# List all environments and ensure they exist
result = self.environment.list()
assert len(result) == 2 and \
environment_obj_1 in result and \
environment_obj_2 in result
def test_delete(self):
self.project.init("test5", "test description")
# Create environment definition
definition_filepath = os.path.join(self.environment.home, "Dockerfile")
with open(definition_filepath, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
input_dict = {
"definition_filepath": definition_filepath,
}
# Create environment in the project
environment_obj = self.environment.create(input_dict)
# Delete environment in the project
result = self.environment.delete(environment_obj.id)
# Check if environment retrieval throws error
thrown = False
try:
self.environment.dal.environment.get_by_id(environment_obj.id)
except EntityNotFound:
thrown = True
assert result == True and \
thrown == True
def test_stop(self):
self.project.init("test5", "test description")
# Create environment definition
definition_filepath = os.path.join(self.environment.home, "Dockerfile")
with open(definition_filepath, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
run_options = {
"command": ["sh", "-c", "echo yo"],
"ports": ["8888:8888"],
"name": None,
"volumes": None,
"detach": False,
"stdin_open": False,
"tty": False,
"gpu": False,
"api": False
}
# Create environment_driver definition
env_def_path = os.path.join(self.project.home, "Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
input_dict = {
"definition_filepath": definition_filepath,
}
# Create environment in the project
environment_obj = self.environment.create(input_dict)
log_filepath = os.path.join(self.project.home, "task.log")
# Build environment in the project
_ = self.environment.build(environment_obj.id)
# Run environment in the project
_, run_id, _ = \
self.environment.run(environment_obj.id, run_options, log_filepath)
# Stop the running environment
return_code = self.environment.stop(run_id)
assert return_code
# teardown
self.environment.delete(environment_obj.id)
class TestTaskModule():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.project_controller = ProjectController()
_ = self.project_controller.init("test", "test description")
self.input_dict = {
"id": "test",
"model_id": "my_model",
"session_id": "my_session",
"command": "python test.py"
}
def teardown_method(self):
pass
def test_task_entity_instantiate(self):
core_task_entity = CoreTask(self.input_dict)
task_entity = Task(core_task_entity)
for k, v in self.input_dict.items():
assert getattr(task_entity, k) == v
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_run(self):
# 1) Run task with no commit or code available (cannot save states before), string command
# 2) Run task with simple python file, no environment definition, string command (auto generate env)
# 3) Run task with simple python file and environment definition, string command
# 4) Run task with simple python file and environment definition, list command
# 5) Run task with simple python file and environment definition passed as a list
# 6) Run task with simple python file and environment definition path present in project environment directory
# folder in project
# 1) Test out option 1)
failed = False
try:
_ = run(command="test")
except CommitFailed:
failed = True
assert failed
# Create a basic task and run it with string command
# (fails w/ no environment)
test_mem_limit = "4g"
test_filepath = os.path.join(self.temp_dir, "script.py")
with open(test_filepath, "wb") as f:
f.write(to_bytes("import os\n"))
f.write(to_bytes("import sys\n"))
f.write(to_bytes("print('hello')\n"))
f.write(to_bytes("print(' accuracy: 0.56 ')\n"))
# 2) Test out option 2
task_obj_0 = run(command="python script.py", mem_limit=test_mem_limit)
assert isinstance(task_obj_0, Task)
assert task_obj_0.id
assert 'hello' in task_obj_0.logs
assert task_obj_0.results == {"accuracy": "0.56"}
# Add environment definition
test_filepath = os.path.join(self.temp_dir, "Dockerfile")
with open(test_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# 3) Test out option 3
task_obj_1 = run(command="python script.py",
env=test_filepath,
mem_limit=test_mem_limit)
assert isinstance(task_obj_1, Task)
assert task_obj_1.id
assert 'hello' in task_obj_1.logs
assert task_obj_1.results == {"accuracy": "0.56"}
# 4) Test out option 4
task_obj_2 = run(command=["python", "script.py"],
env=test_filepath,
mem_limit=test_mem_limit)
assert isinstance(task_obj_2, Task)
assert task_obj_2.id
assert 'hello' in task_obj_2.logs
assert task_obj_2.results == {"accuracy": "0.56"}
# 5) Test out option 5
task_obj_3 = run(command=["python", "script.py"],
env=[test_filepath + ">Dockerfile"],
mem_limit=test_mem_limit)
assert isinstance(task_obj_3, Task)
assert task_obj_3.id
assert 'hello' in task_obj_3.logs
assert task_obj_3.results == {"accuracy": "0.56"}
# 6) Test out option 6
os.remove(test_filepath)
test_filepath = os.path.join(
self.project_controller.file_driver.environment_directory,
"Dockerfile")
with open(test_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
task_obj_4 = run(command=["python", "script.py"])
assert isinstance(task_obj_4, Task)
assert task_obj_4.id
assert 'hello' in task_obj_4.logs
assert task_obj_4.results == {"accuracy": "0.56"}
def test_ls(self):
# check project is not initialized if wrong home
Config().set_home(os.path.join("does", "not", "exist"))
failed = False
try:
ls()
except InvalidProjectPath:
failed = True
assert failed
# check session does not exist if wrong session
Config().set_home(self.temp_dir)
failed = False
try:
ls(session_id="does_not_exist")
except SessionDoesNotExist:
failed = True
assert failed
# run a task with default params
self.__setup()
# list all tasks with no filters
task_list_1 = ls()
assert task_list_1
assert len(list(task_list_1)) == 1
assert isinstance(task_list_1[0], Task)
# run another task with default params
self.__setup(command="test")
# list all tasks with no filters (works when more than 1 task)
task_list_2 = ls()
assert task_list_2
assert len(list(task_list_2)) == 2
assert isinstance(task_list_2[0], Task)
assert isinstance(task_list_2[1], Task)
# list tasks with specific filter
task_list_3 = ls(filter="script.py")
assert task_list_3
assert len(list(task_list_3)) == 1
assert isinstance(task_list_3[0], Task)
# list snapshots with filter of none
task_list_4 = ls(filter="random")
assert len(list(task_list_4)) == 0
def __setup(self, command="python script.py"):
# Create a basic task and run it with string command
test_filepath = os.path.join(self.temp_dir, "script.py")
with open(test_filepath, "wb") as f:
f.write(to_bytes("import os\n"))
f.write(to_bytes("import shutil\n"))
f.write(to_bytes("print('hello')\n"))
test_filepath = os.path.join(self.temp_dir, "Dockerfile")
with open(test_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
return run(command=command, env=test_filepath)
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_entity_status(self):
core_task_entity = CoreTask(self.input_dict)
task_entity = Task(core_task_entity)
# Test failure because entity has not been created by controller
failed = False
try:
task_entity.status
except DoesNotExist:
failed = True
assert failed
# Test success
task_entity = self.__setup()
result = task_entity.status
assert result
assert isinstance(result, to_unicode)
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_entity_start_time(self):
core_task_entity = CoreTask(self.input_dict)
task_entity = Task(core_task_entity)
# Test failure because entity has not been created by controller
failed = False
try:
task_entity.start_time
except DoesNotExist:
failed = True
assert failed
# Test success
task_entity = self.__setup()
result = task_entity.start_time
assert result
assert isinstance(result, datetime.datetime)
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_entity_end_time(self):
core_task_entity = CoreTask(self.input_dict)
task_entity = Task(core_task_entity)
# Test failure because entity has not been created by controller
failed = False
try:
task_entity.end_time
except DoesNotExist:
failed = True
assert failed
# Test success
task_entity = self.__setup()
result = task_entity.end_time
assert result
assert isinstance(result, datetime.datetime)
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_entity_duration(self):
core_task_entity = CoreTask(self.input_dict)
task_entity = Task(core_task_entity)
# Test failure because entity has not been created by controller
failed = False
try:
task_entity.duration
except DoesNotExist:
failed = True
assert failed
# Test success
task_entity = self.__setup()
result = task_entity.duration
assert result
assert isinstance(result, float)
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_entity_logs(self):
core_task_entity = CoreTask(self.input_dict)
task_entity = Task(core_task_entity)
# Test failure because entity has not been created by controller
failed = False
try:
task_entity.logs
except DoesNotExist:
failed = True
assert failed
# Test success
task_entity = self.__setup()
result = task_entity.logs
assert result
assert isinstance(result, to_unicode)
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_entity_results(self):
core_task_entity = CoreTask(self.input_dict)
task_entity = Task(core_task_entity)
# Test failure because entity has not been created by controller
failed = False
try:
task_entity.results
except DoesNotExist:
failed = True
assert failed
# Test success
task_entity = self.__setup()
result = task_entity.results
assert result == None
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_entity_files(self):
core_task_entity = CoreTask(self.input_dict)
task_entity = Task(core_task_entity)
# Test failure because entity has not been created by controller
failed = False
try:
task_entity.files
except DoesNotExist:
failed = True
assert failed
# Test success
task_entity = self.__setup()
result = task_entity.files
assert len(result) == 1
assert isinstance(result[0], TextIOWrapper)
assert result[0].mode == "r"
assert result[0].name
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_entity_get_files(self):
core_task_entity = CoreTask(self.input_dict)
task_entity = Task(core_task_entity)
# Test failure because entity has not been created by controller
failed = False
try:
task_entity.get_files()
except DoesNotExist:
failed = True
assert failed
task_entity = self.__setup()
result = task_entity.get_files()
assert len(result) == 1
assert isinstance(result[0], TextIOWrapper)
assert result[0].mode == "r"
assert result[0].name
result = task_entity.get_files(mode="a")
assert len(result) == 1
assert isinstance(result[0], TextIOWrapper)
assert result[0].mode == "a"
assert result[0].name
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_task_entity_str(self):
task_entity = self.__setup()
for k in self.input_dict:
if k != "model_id":
assert str(task_entity.__dict__[k]) in str(task_entity)
class TestCodeController():
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.project_controller = ProjectController()
self.code_controller = CodeController()
def teardown_method(self):
pass
def test_current_code(self):
self.project_controller.init("test4", "test description")
# Create test file
definition_filepath = os.path.join(self.code_controller.home,
"test.txt")
with open(definition_filepath, "wb") as f:
f.write(to_bytes(str("test")))
# Test failing with unstaged changes
failed = False
try:
self.code_controller.current_code()
except UnstagedChanges:
failed = True
assert failed
# Test passing with something to commit
code_obj = self.code_controller.create()
current_code_obj = self.code_controller.current_code()
assert code_obj == current_code_obj
def test_create(self):
self.project_controller.init("test3", "test description")
# Test failing for nothing to commit, no id
result = self.code_controller.create()
assert result
# Test failing for non-existant commit_id
failed = False
try:
self.code_controller.create(commit_id="random")
except CommitDoesNotExist:
failed = True
assert failed
# Create test file
definition_filepath = os.path.join(self.code_controller.home,
"test.txt")
with open(definition_filepath, "wb") as f:
f.write(to_bytes(str("test")))
# Test passing with something to commit
code_obj = self.code_controller.create()
assert code_obj
assert code_obj.id
assert code_obj.driver_type
# Test should return same code_obj if same commit_id
code_obj_2 = self.code_controller.create()
assert code_obj_2 == code_obj
# Test failing with random id given
random_commit_id = "random"
try:
self.code_controller.create(commit_id=random_commit_id)
except CommitDoesNotExist:
assert True
def test_list(self):
self.project_controller.init("test4", "test description")
# Create test file
definition_filepath = os.path.join(self.code_controller.home,
"test.txt")
with open(definition_filepath, "wb") as f:
f.write(to_bytes(str("test")))
# Test passing with something to commit
code_obj_1 = self.code_controller.create()
# Create test file
definition_filepath = os.path.join(self.code_controller.home,
"test2.txt")
with open(definition_filepath, "wb") as f:
f.write(to_bytes(str("test")))
# Test passing with something to commit
code_obj_2 = self.code_controller.create()
# List all code and ensure they exist
result = self.code_controller.list()
assert len(result) == 2 and \
code_obj_1 in result and \
code_obj_2 in result
def test_delete(self):
self.project_controller.init("test5", "test description")
# Create test file
definition_filepath = os.path.join(self.code_controller.home,
"test.txt")
with open(definition_filepath, "wb") as f:
f.write(to_bytes(str("test")))
# Test passing with something to commit
code_obj = self.code_controller.create()
# Delete code in the project
result = self.code_controller.delete(code_obj.id)
# Check if code retrieval throws error
thrown = False
try:
self.code_controller.dal.code.get_by_id(code_obj.id)
except EntityNotFound:
thrown = True
assert result == True and \
thrown == True
def test_exists(self):
# Test failure, not initialized
failed = False
try:
_ = self.code_controller.create()
except:
failed = True
assert failed
# Initialize project and test file
self.project_controller.init("test5", "test description")
definition_filepath = os.path.join(self.code_controller.home,
"test.txt")
with open(definition_filepath, "wb") as f:
f.write(to_bytes(str("test")))
code_obj = self.code_controller.create()
# Check by code id
result = self.code_controller.exists(code_id=code_obj.id)
assert result
# Check by code commit id
result = self.code_controller.exists(code_commit_id=code_obj.commit_id)
assert result
# Test with wrong environment id
result = self.code_controller.exists(code_id='test_wrong_env_id')
assert not result
def test_checkout(self):
# Test code does not exist
failed = False
try:
self.code_controller.checkout("does_not_exist")
except CodeDoesNotExist:
failed = True
assert failed
def __setup(self):
Config().set_home(self.temp_dir)
self.project_controller = ProjectController()
self.project_controller.init("test", "test description")
self.task_controller = TaskController()
self.snapshot_controller = SnapshotController()
def __setup(self):
Config().set_home(self.temp_dir)
self.project_controller = ProjectController()
self.project_controller.init("test", "test description")
self.environment_controller = EnvironmentController()
self.task_controller = TaskController()
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.project_controller = ProjectController()
self.environment_ids = []
class TestFileCollectionController():
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.project_controller = ProjectController()
self.file_collection_controller = FileCollectionController()
def teardown_method(self):
pass
def __setup(self):
# Create the files in the project files directory
dirpath1 = os.path.join(
self.file_collection_controller.file_driver.files_directory,
"dirpath1")
os.makedirs(dirpath1)
filepath1 = os.path.join(
self.file_collection_controller.file_driver.files_directory,
"filepath1")
with open(filepath1, "wb") as _:
pass
return filepath1, dirpath1
def test_current_file_collection(self):
self.project_controller.init("test3", "test description")
_ = self.__setup()
# Test failure because of unstaged changes
failed = False
try:
self.file_collection_controller.current_file_collection()
except UnstagedChanges:
failed = True
assert failed
# Test success with created files
file_collection_obj = self.file_collection_controller.create([])
current_file_collection_obj = self.file_collection_controller.current_file_collection(
)
assert current_file_collection_obj == file_collection_obj
def test_create(self):
self.project_controller.init("test3", "test description")
# Test failure creation of collection if no path given
failed = False
try:
self.file_collection_controller.create()
except TypeError:
failed = True
assert failed
# Test create success with paths
paths = self.__setup()
file_collection_obj = self.file_collection_controller.create(paths)
assert file_collection_obj
assert file_collection_obj.id
assert file_collection_obj.path
assert file_collection_obj.driver_type
assert file_collection_obj.filehash == "74be16979710d4c4e7c6647856088456"
# Test create success without paths (should be the same as previous)
file_collection_obj_1 = self.file_collection_controller.create([])
assert file_collection_obj_1 == file_collection_obj
assert file_collection_obj_1.id == file_collection_obj.id
assert file_collection_obj_1.path == file_collection_obj.path
assert file_collection_obj_1.driver_type == file_collection_obj.driver_type
assert file_collection_obj_1.filehash == file_collection_obj.filehash
# Test create success with paths again (should be same as previous)
file_collection_obj_2 = self.file_collection_controller.create(paths)
assert file_collection_obj_2 == file_collection_obj_1
assert file_collection_obj_2.id == file_collection_obj_1.id
assert file_collection_obj_2.path == file_collection_obj_1.path
assert file_collection_obj_2.driver_type == file_collection_obj_1.driver_type
assert file_collection_obj_2.filehash == file_collection_obj_1.filehash
# Test file collection with empty paths (should be same as previous)
file_collection_obj_3 = self.file_collection_controller.create([])
assert file_collection_obj_3 == file_collection_obj_2
assert file_collection_obj_3.id == file_collection_obj_2.id
assert file_collection_obj_3.path == file_collection_obj_2.path
assert file_collection_obj_3.driver_type == file_collection_obj_2.driver_type
assert file_collection_obj_3.filehash == file_collection_obj_2.filehash
def test_list(self):
self.project_controller.init("test4", "test description")
paths_1 = self.__setup()
filepath2 = os.path.join(self.file_collection_controller.home,
"filepath2")
with open(filepath2, "wb") as f:
f.write(to_bytes("test" + "\n"))
paths_2 = [filepath2]
file_collection_obj_1 = self.file_collection_controller.create(paths_1)
file_collection_obj_2 = self.file_collection_controller.create(paths_2)
# List all code and ensure they exist
result = self.file_collection_controller.list()
assert len(result) == 2 and \
file_collection_obj_1 in result and \
file_collection_obj_2 in result
def test_delete(self):
self.project_controller.init("test5", "test description")
paths = self.__setup()
file_collection_obj = self.file_collection_controller.create(paths)
# Delete code in the project
result = self.file_collection_controller.delete(file_collection_obj.id)
# Check if code retrieval throws error
thrown = False
try:
self.file_collection_controller.dal.file_collection.get_by_id(
file_collection_obj.id)
except EntityNotFound:
thrown = True
assert result == True and \
thrown == True
def test_exists_file(self):
self.project_controller.init("test6", "test description")
paths = self.__setup()
file_collection_obj = self.file_collection_controller.create(paths)
# check for file_collection_id
result = self.file_collection_controller.exists(
file_collection_id=file_collection_obj.id)
assert result
# check for file_hash in file_collection
result = self.file_collection_controller.exists(
file_hash=file_collection_obj.filehash)
assert result
# check for not proper file_collection_id
result = self.file_collection_controller.exists(
file_collection_id="test_file_collection_id")
assert not result
def test_calculate_project_files_hash(self):
self.project_controller.init("test7", "test description")
filepath1, dirpath1 = self.__setup()
# Test if hash is for 1 blank filepath and empty directory
result = self.file_collection_controller._calculate_project_files_hash(
)
assert result == "74be16979710d4c4e7c6647856088456"
def test_has_unstaged_changes(self):
self.project_controller.init("test8", "test description")
# Create the files in the project files directory
paths = self.__setup()
# Test when there are unstaged changes
result = self.file_collection_controller._has_unstaged_changes()
assert result
# Save the file collection
self.file_collection_controller.create(paths)
# Test when there are no unstaged changes
result = self.file_collection_controller._has_unstaged_changes()
assert not result
# Change the file contents
with open(paths[0], "wb") as f:
f.write(to_bytes("hello"))
# Test when there are unstaged changes again
result = self.file_collection_controller._has_unstaged_changes()
assert result
def test_check_unstaged_changes(self):
self.project_controller.init("test9", "test description")
# Create the files in the project files directory
paths = self.__setup()
# Test when there are unstaged changes
failed = False
try:
_ = self.file_collection_controller.check_unstaged_changes()
except UnstagedChanges:
failed = True
assert failed
# Save the file collection
self.file_collection_controller.create(paths)
# Test when there are no unstaged changes
result = self.file_collection_controller.check_unstaged_changes()
assert not result
# Change the file contents
with open(paths[0], "wb") as f:
f.write(to_bytes("hello"))
# Test when there are unstaged changes again
failed = False
try:
_ = self.file_collection_controller.check_unstaged_changes()
except UnstagedChanges:
failed = True
assert failed
# Test when there are no files (should be staged)
os.remove(paths[0])
shutil.rmtree(paths[1])
result = self.file_collection_controller.check_unstaged_changes()
assert not result
def test_checkout(self):
self.project_controller.init("test9", "test description")
# Create the files in the project files directory
paths = self.__setup()
# Create a file collection to checkout to with paths
file_collection_obj = self.file_collection_controller.create(paths)
# Checkout success when there are no unstaged changes
result = self.file_collection_controller.checkout(
file_collection_obj.id)
assert result
current_hash = self.file_collection_controller._calculate_project_files_hash(
)
assert current_hash == "74be16979710d4c4e7c6647856088456"
assert file_collection_obj.filehash == current_hash
# Check the filenames as well because the hash does not take this into account
assert os.path.isfile(paths[0])
# Change file contents to make it unstaged
with open(paths[0], "wb") as f:
f.write(to_bytes("hello"))
# Checkout failure when there are unstaged changes
failed = False
try:
_ = self.file_collection_controller.checkout(
file_collection_obj.id)
except UnstagedChanges:
failed = True
assert failed
# Create a new file collection with paths
file_collection_obj_1 = self.file_collection_controller.create(paths)
# Checkout success back when there are no unstaged changes
result = self.file_collection_controller.checkout(
file_collection_obj.id)
assert result
current_hash = self.file_collection_controller._calculate_project_files_hash(
)
assert current_hash == "74be16979710d4c4e7c6647856088456"
assert file_collection_obj.filehash == current_hash
assert file_collection_obj_1.filehash != current_hash
# Check the filenames as well because the hash does not take this into account
assert os.path.isfile(paths[0])
class TestSessionController():
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
def teardown_method(self):
shutil.rmtree(self.temp_dir)
def __setup(self):
Config().set_home(self.temp_dir)
self.project_controller = ProjectController()
self.project_controller.init("test", "test description")
self.session_controller = SessionController()
def test_init_fail_project_not_init(self):
Config().set_home(self.temp_dir)
failed = False
try:
SessionController()
except ProjectNotInitialized:
failed = True
assert failed
def test_init_fail_invalid_path(self):
test_home = "some_random_dir"
Config().set_home(test_home)
failed = False
try:
SessionController()
except InvalidProjectPath:
failed = True
assert failed
def test_find_default_session(self):
self.__setup()
sessions = self.session_controller.list()
has_default = False
for s in sessions:
if s.name == 'default':
has_default = True
break
assert has_default
def test_create_session(self):
self.__setup()
test_sess = self.session_controller.create({"name": "test1"})
assert test_sess.id
assert test_sess.model_id == self.project_controller.model.id
def test_select(self):
self.__setup()
session_obj = self.session_controller.create({"name": "test2"})
# Test success with name
new_current = self.session_controller.select("test2")
assert new_current.current == True
found_old_current = False
current_was_updated = False
for sess in self.session_controller.list():
if sess.current == True and sess.id != new_current.id:
found_old_current = True
if sess.current == True and sess.id == new_current.id:
current_was_updated = True
assert not found_old_current == True
assert current_was_updated == True
# reset to default
_ = self.session_controller.select("default")
# Test success with id
new_current = self.session_controller.select(session_obj.id)
assert new_current.current == True
found_old_current = False
current_was_updated = False
for sess in self.session_controller.list():
if sess.current == True and sess.id != new_current.id:
found_old_current = True
if sess.current == True and sess.id == new_current.id:
current_was_updated = True
assert not found_old_current == True
assert current_was_updated == True
# Test failure no session
failed = False
try:
_ = self.session_controller.select("random_name_or_id")
except SessionDoesNotExist:
failed = True
assert failed
def test_get_current(self):
self.__setup()
current_sess = self.session_controller.get_current()
assert current_sess
assert current_sess.current == True
def test_list_session_sort(self):
self.__setup()
# Sort ascending
sessions = self.session_controller.list(
sort_key='created_at', sort_order='ascending')
assert sessions[0].created_at <= sessions[-1].created_at
# Sort descending
sessions = self.session_controller.list(
sort_key='created_at', sort_order='descending')
assert sessions[0].created_at >= sessions[-1].created_at
# Wrong order being passed in
failed = False
try:
_ = self.session_controller.list(
sort_key='created_at', sort_order='wrong_order')
except InvalidArgumentType:
failed = True
assert failed
# Wrong key and order being passed in
failed = False
try:
_ = self.session_controller.list(
sort_key='wrong_key', sort_order='wrong_order')
except InvalidArgumentType:
failed = True
assert failed
# wrong key and right order being passed in
expected_sessions = self.session_controller.list(
sort_key='created_at', sort_order='ascending')
sessions = self.session_controller.list(
sort_key='wrong_key', sort_order='ascending')
expected_ids = [item.id for item in expected_sessions]
ids = [item.id for item in sessions]
assert set(expected_ids) == set(ids)
def test_update(self):
self.__setup()
# Test successful update
session_obj = self.session_controller.create({"name": "test3"})
new_name = "new"
updated_session_obj = self.session_controller.update(
session_obj.id, name=new_name)
same_session_obj = self.session_controller.dal.session.findOne({
"name": new_name
})
assert updated_session_obj == same_session_obj
assert updated_session_obj.name == new_name
# Test failure session does not exist
failed = False
try:
self.session_controller.update("random_id")
except SessionDoesNotExist:
failed = True
assert failed
# Test failure try to update default
default_session_obj = self.session_controller.dal.session.findOne({
"name": "default"
})
failed = False
try:
self.session_controller.update(default_session_obj.id)
except InvalidOperation:
failed = True
assert failed
def test_delete(self):
self.__setup()
# Test successful delete
session_obj = self.session_controller.create({"name": "test3"})
_ = self.session_controller.select("test3")
self.session_controller.delete(session_obj.id)
entity_does_not_exist = False
try:
_ = self.session_controller.dal.session.get_by_id(session_obj.id)
except EntityNotFound:
entity_does_not_exist = True
assert entity_does_not_exist
# current session should be "default"
assert self.session_controller.get_current().name == "default"
# Test failure delete default
failed = False
try:
self.session_controller.delete(
self.session_controller.get_current().id)
except InvalidOperation:
failed = True
assert failed
# Test failure does not exist
failed = False
try:
self.session_controller.delete("random_id")
except SessionDoesNotExist:
failed = True
assert failed
def test_delete_by_name(self):
self.__setup()
session_obj = self.session_controller.create({"name": "test3"})
_ = self.session_controller.select("test3")
self.session_controller.delete_by_name("test3")
entity_does_not_exist = False
try:
_ = self.session_controller.dal.session.get_by_id(session_obj.id)
except EntityNotFound:
entity_does_not_exist = True
assert entity_does_not_exist
# current session should be "default"
assert self.session_controller.get_current().name == "default"
# Test failure delete default
failed = False
try:
self.session_controller.delete_by_name(
self.session_controller.get_current().name)
except InvalidOperation:
failed = True
assert failed
# Test failure does not exist
failed = False
try:
self.session_controller.delete_by_name("random_name")
except SessionDoesNotExist:
failed = True
assert failed
class ProjectCommand(BaseCommand):
def __init__(self, cli_helper):
super(ProjectCommand, self).__init__(cli_helper)
self.project_controller = ProjectController()
def init(self, name, description, force):
"""Initialize command
Parameters
----------
name : str
name for the project
description : str
description of the project
force : bool
Boolean to force initialization without prompts
Returns
-------
datmo.core.entity.model.Model
"""
# Check if project already exists
is_new_model = False
if not self.project_controller.model:
is_new_model = True
if is_new_model: # Initialize a new project
self.cli_helper.echo(
__("info", "cli.project.init.create",
{"path": self.project_controller.home}))
if not name:
_, default_name = os.path.split(self.project_controller.home)
if not force:
name = self.cli_helper.prompt(__("prompt",
"cli.project.init.name"),
default=default_name)
else:
name = default_name
if not description:
if not force:
description = self.cli_helper.prompt(
__("prompt", "cli.project.init.description"))
else:
description = ""
try:
success = self.project_controller.init(name, description)
if success:
self.cli_helper.echo(
__("info", "cli.project.init.create.success", {
"name": name,
"path": self.project_controller.home
}))
except Exception:
self.cli_helper.echo(
__("info", "cli.project.init.create.failure", {
"name": name,
"path": self.project_controller.home
}))
return None
else: # Update the current project
self.cli_helper.echo(
__(
"info", "cli.project.init.update", {
"name": self.project_controller.model.name,
"path": self.project_controller.home
}))
# Prompt for the name and description and add default if not given
if not name and not force:
name = self.cli_helper.prompt(
__("prompt", "cli.project.init.name"),
default=self.project_controller.model.name)
elif force:
name = self.project_controller.model.name
if not description and not force:
description = self.cli_helper.prompt(
__("prompt", "cli.project.init.description"),
default=self.project_controller.model.description)
elif force:
description = self.project_controller.model.description
# Update the project with the values given
try:
success = self.project_controller.init(name, description)
if success:
self.cli_helper.echo(
__("info", "cli.project.init.update.success", {
"name": name,
"path": self.project_controller.home
}))
except Exception:
self.cli_helper.echo(
__("info", "cli.project.init.update.failure", {
"name": name,
"path": self.project_controller.home
}))
return None
self.cli_helper.echo("")
# Print out simple project meta data
for k, v in self.project_controller.model.to_dictionary().items():
if k != "config":
self.cli_helper.echo(str(k) + ": " + str(v))
# Ask question if the user would like to setup environment
environment_setup = self.cli_helper.prompt_bool(
__("prompt",
"cli.project.environment.setup")) if not force else False
if environment_setup:
# TODO: remove business logic from here and create common helper
# Setting up the environment definition file
self.environment_controller = EnvironmentController()
environment_types = self.environment_controller.get_environment_types(
)
environment_type = self.cli_helper.prompt_available_options(
environment_types, option_type="type")
available_environment_frameworks = self.environment_controller.get_supported_frameworks(
environment_type)
environment_framework = self.cli_helper.prompt_available_options(
available_environment_frameworks, option_type="framework")
available_environment_languages = self.environment_controller.get_supported_languages(
environment_type, environment_framework)
environment_language = self.cli_helper.prompt_available_options(
available_environment_languages, option_type="language")
options = {
"environment_type": environment_type,
"environment_framework": environment_framework,
"environment_language": environment_language
}
environment_obj = self.environment_controller.setup(
options=options)
self.cli_helper.echo(
__("info", "cli.environment.setup.success",
(environment_obj.name, environment_obj.id)))
else:
self.cli_helper.echo(
"there was no environment setup. you can get information"
" here: https://datmo.readthedocs.io/en/latest/env-setup.html")
return self.project_controller.model
def version(self):
return self.cli_helper.echo("datmo version: %s" % __version__)
def configure(self):
"""
Configure datmo installation
"""
# General setup
setup_remote_bool = self.cli_helper.prompt_bool(
"Would you like to setup your remote credentials? [yN]")
if setup_remote_bool:
datmo_api_key = None
while not datmo_api_key:
datmo_api_key = self.cli_helper.prompt(
"Enter API key for datmo")
# Initialize remote API to get master ip address
remote_api = RemoteAPI(datmo_api_key)
response = remote_api.get_deployment_info()
master_system_info = response['body']['master_system_info']
master_server_ip = str(master_system_info.get('datmo_master_ip')) if isinstance(master_system_info, dict)\
else None
# Create a config file
self.datmo_config = JSONStore(
os.path.join(os.path.expanduser("~"), ".datmo", "config"))
config = dict()
if master_server_ip and datmo_api_key:
config["MASTER_SERVER_IP"] = master_server_ip
config["DATMO_API_KEY"] = datmo_api_key
self.datmo_config.to_file(config)
else:
self.cli_helper.echo(
"Datmo API key could not be saved. Please try again")
# Setup project specific things
if self.project_controller.model:
pass
else:
self.cli_helper.echo(
"No datmo project found. Skipping configuration for project.")
@Helper.notify_no_project_found
def status(self):
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
self.project_controller.status()
# Print out simple project meta data
for k, v in status_dict.items():
if k != "config":
self.cli_helper.echo(str(k) + ": " + str(v))
self.cli_helper.echo("")
# Print out any unstaged changes else print out the latest snapshot state of the repository
if not unstaged_code and not unstaged_environment and not unstaged_files:
self.cli_helper.echo(
"all changes have been saved, no unstaged changes")
self.cli_helper.echo("")
self.cli_helper.echo("current snapshot state of the repository: ")
if current_snapshot:
self.cli_helper.echo(current_snapshot)
else:
# Print out the unstaged components if unstaged
self.cli_helper.echo("unstaged changes since latest snapshot:")
if unstaged_code:
self.cli_helper.echo("code has been changed")
if unstaged_environment:
self.cli_helper.echo("environment has been changed")
if unstaged_files:
self.cli_helper.echo("files have been changed")
# Print out info for the latest snapshot (the most recent first, and state if autogenerated or by user)
if latest_snapshot_user_generated and not latest_snapshot_auto_generated:
self.cli_helper.echo("latest snapshot generated by the user: "******"no snapshot autogenerated by datmo")
elif latest_snapshot_auto_generated and not latest_snapshot_user_generated:
self.cli_helper.echo("latest snapshot autogenerated by datmo: ")
self.cli_helper.echo(latest_snapshot_auto_generated)
self.cli_helper.echo("no snapshot generated by the user")
elif not latest_snapshot_user_generated and not latest_snapshot_auto_generated:
self.cli_helper.echo("no snapshots created yet")
elif latest_snapshot_user_generated.created_at > latest_snapshot_auto_generated.created_at:
self.cli_helper.echo("latest snapshot generated by the user: "******"latest snapshot autogenerated by datmo: ")
self.cli_helper.echo(latest_snapshot_auto_generated)
elif latest_snapshot_user_generated.created_at < latest_snapshot_auto_generated.created_at:
self.cli_helper.echo("latest snapshot autogenerated by datmo: ")
self.cli_helper.echo(latest_snapshot_auto_generated)
self.cli_helper.echo("latest snapshot generated by the user: "******"prompt", "cli.project.cleanup.confirm"))
# Cleanup datmo project if user specifies
if response:
name = self.project_controller.model.name if self.project_controller.model.name else ""
path = self.project_controller.home if self.project_controller.home else ""
self.cli_helper.echo(
__("info", "cli.project.cleanup", {
"name": name,
"path": path
}))
try:
success = self.project_controller.cleanup()
if success:
self.cli_helper.echo(
__("info", "cli.project.cleanup.success", {
"name": name,
"path": path
}))
return success
except Exception:
self.cli_helper.echo(
__("info", "cli.project.cleanup.failure", {
"name": name,
"path": path
}))
return False
def dashboard(self):
if not self.project_controller.is_initialized:
self.cli_helper.echo(
"Please initialize datmo before using this command")
return False
dir_path = os.path.dirname(os.path.abspath(__file__))
os.chdir(os.path.join(dir_path, "../../dashboard"))
app.run(host='0.0.0.0')
return True
def __setup(self):
self.project = ProjectController(self.temp_dir)
self.project.init("test", "test description")
self.environment = EnvironmentController(self.temp_dir)
self.task = TaskController(self.temp_dir)
class ProjectCommand(BaseCommand):
# NOTE: dal_driver is not passed into the project because it is created
# first by ProjectController and then passed down to all other Controllers
def __init__(self, home, cli_helper):
super(ProjectCommand, self).__init__(home, cli_helper)
self.project_controller = ProjectController(home=home)
def init(self, name, description):
"""Initialize command
Parameters
----------
name : str
name for the project
description : str
description of the project
Returns
-------
datmo.core.entity.model.Model
"""
# Check if project already exists
is_new_model = False
if not self.project_controller.model:
is_new_model = True
if is_new_model: # Initialize a new project
self.cli_helper.echo(
__("info", "cli.project.init.create", {
"name": name,
"path": self.home
}))
if not name:
name = self.cli_helper.prompt(
__("prompt", "cli.project.init.name"))
if not description:
description = self.cli_helper.prompt(
__("prompt", "cli.project.init.description"))
try:
success = self.project_controller.init(name, description)
if success:
self.cli_helper.echo(
__("info", "cli.project.init.create.success", {
"name": name,
"path": self.home
}))
except:
self.cli_helper.echo(
__("info", "cli.project.init.create.failure", {
"name": name,
"path": self.home
}))
return None
else: # Update the current project
self.cli_helper.echo(
__("info", "cli.project.init.update", {
"name": self.project_controller.model.name,
"path": self.home
}))
if not name:
name = self.cli_helper.prompt(
__("prompt", "cli.project.init.name"))
if not description:
description = self.cli_helper.prompt(
__("prompt", "cli.project.init.description"))
# Update project parameter if given parameter is not falsy and different
if not name or name == self.project_controller.model.name:
name = self.project_controller.model.name
if not description or description == self.project_controller.model.description:
description = self.project_controller.model.description
# Update the project with the values given
try:
success = self.project_controller.init(name, description)
if success:
self.cli_helper.echo(
__("info", "cli.project.init.update.success", {
"name": name,
"path": self.home
}))
except:
self.cli_helper.echo(
__("info", "cli.project.init.update.failure", {
"name": name,
"path": self.home
}))
return None
self.cli_helper.echo("")
# Print out simple project meta data
for k, v in self.project_controller.model.to_dictionary().items():
if k != "config":
self.cli_helper.echo(str(k) + ": " + str(v))
return self.project_controller.model
def version(self):
return self.cli_helper.echo("datmo version: %s" % __version__)
def status(self):
status_dict, latest_snapshot, ascending_unstaged_tasks = self.project_controller.status(
)
# Print out simple project meta data
for k, v in status_dict.items():
if k != "config":
self.cli_helper.echo(str(k) + ": " + str(v))
self.cli_helper.echo("")
# Print out project config meta data
self.cli_helper.echo("config: ")
self.cli_helper.echo(status_dict['config'])
self.cli_helper.echo("")
# Print out latest snapshot info
self.cli_helper.echo("latest snapshot id: ")
if latest_snapshot:
self.cli_helper.echo(latest_snapshot.id)
else:
self.cli_helper.echo("no snapshots created yet")
self.cli_helper.echo("")
# Print out unstaged tasks
self.cli_helper.echo("unstaged task ids:")
if ascending_unstaged_tasks:
for task in ascending_unstaged_tasks:
self.cli_helper.echo(task.id)
else:
self.cli_helper.echo("no unstaged tasks")
return status_dict, latest_snapshot, ascending_unstaged_tasks
def cleanup(self):
# Prompt user to ensure they would like to remove all datmo information
response = self.cli_helper.prompt_bool(
__("prompt", "cli.project.cleanup.confirm"))
# Cleanup datmo project if user specifies
if response:
self.cli_helper.echo(
__(
"info", "cli.project.cleanup", {
"name": self.project_controller.model.name,
"path": self.project_controller.home
}))
try:
success = self.project_controller.cleanup()
if success:
self.cli_helper.echo(
__(
"info", "cli.project.cleanup.success", {
"name": self.project_controller.model.name,
"path": self.project_controller.home
}))
return success
except Exception:
self.cli_helper.echo(
__(
"info", "cli.project.cleanup.failure", {
"name": self.project_controller.model.name,
"path": self.project_controller.home
}))
return False
def __init__(self, cli_helper):
super(ProjectCommand, self).__init__(cli_helper)
self.project_controller = ProjectController()
class TestEnvironmentController():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.project_controller = ProjectController()
def teardown_method(self):
pass
def __setup(self):
self.project_controller.init("test_setup", "test description")
self.environment_controller = EnvironmentController()
with open(os.path.join(self.temp_dir, "test.txt"), "wb") as f:
f.write(to_bytes("hello"))
self.random_filepath = os.path.join(
self.environment_controller.file_driver.environment_directory,
"test")
with open(self.random_filepath, "wb") as f:
f.write(to_bytes("cool"))
self.definition_filepath = os.path.join(
self.environment_controller.file_driver.environment_directory,
"Dockerfile")
with open(self.definition_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
def test_init_fail_project_not_init(self):
Config().set_home(self.temp_dir)
failed = False
try:
EnvironmentController()
except ProjectNotInitialized:
failed = True
assert failed
def test_get_supported_environments(self):
self.__setup()
result = self.environment_controller.get_supported_environments()
assert result
def test_setup(self):
self.project_controller.init("test_setup", "test description")
self.environment_controller = EnvironmentController()
# Test success setup once (no files present)
options = {"name": "xgboost:cpu"}
result = self.environment_controller.setup(options=options)
output_definition_filepath = os.path.join(
self.environment_controller.file_driver.environment_directory,
"Dockerfile")
assert isinstance(result, Environment)
assert result.name == options['name']
assert result.description == "supported base environment created by datmo"
assert os.path.isfile(output_definition_filepath)
assert "FROM datmo/xgboost:cpu" in open(output_definition_filepath,
"r").read()
# Test success setup again (files present, but staged)
options = {"name": "xgboost:cpu"}
result = self.environment_controller.setup(options=options)
output_definition_filepath = os.path.join(
self.environment_controller.file_driver.environment_directory,
"Dockerfile")
assert isinstance(result, Environment)
assert result.name == options['name']
assert result.description == "supported base environment created by datmo"
assert os.path.isfile(output_definition_filepath)
assert "FROM datmo/xgboost:cpu" in open(output_definition_filepath,
"r").read()
# Test failure in downstream function (e.g. bad inputs, no name given)
failed = False
try:
self.environment_controller.setup(options={})
except EnvironmentDoesNotExist:
failed = True
assert failed
# Change environment file
with open(output_definition_filepath, "wb") as f:
f.write(to_bytes("new content"))
# Test failure setup (unstaged changes)
failed = False
try:
self.environment_controller.setup(options=options)
except UnstagedChanges:
failed = True
assert failed
def test_create(self):
# 0) Test SUCCESS create when definition path exists in project environment directory (no input, no root) -- with hardware file
# 1) Test SUCCESS create when definition path exists in project environment directory (no input, no root)
# 5) Test SUCCESS when definition path exists in project environment directory and passed from input dict (takes input)
# 2) Test SUCCESS create when definition path exists in root project folder (no input, no project environment dir)
# 3) Test SUCCESS create when definition path is passed into input dict (takes input, no project environment dir)
# 4) Test SUCCESS create when definition path is passed into input dict along with expected filename to be saved
# 6) Test FAIL when passing same filepath with same filename into input dict
self.__setup()
input_dict_0 = {"name": "test", "description": "test description"}
# 0) Test option 0 (cannot test hash because hardware is machine-dependent)
environment_obj_0 = self.environment_controller.create(input_dict_0)
assert environment_obj_0
assert isinstance(environment_obj_0, Environment)
assert environment_obj_0.id
assert environment_obj_0.driver_type == "docker"
assert environment_obj_0.file_collection_id
assert environment_obj_0.definition_filename
assert environment_obj_0.hardware_info
assert environment_obj_0.unique_hash
assert environment_obj_0.name == "test"
assert environment_obj_0.description == "test description"
# Get file collection path
file_collection_obj = self.environment_controller.dal.file_collection. \
get_by_id(environment_obj_0.file_collection_id)
file_collection_dir = self.environment_controller.file_driver. \
get_collection_path(file_collection_obj.filehash)
assert os.path.isfile(os.path.join(file_collection_dir, "test"))
assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile"))
output = open(os.path.join(file_collection_dir, "Dockerfile"),
"r").read()
print(repr(output))
assert os.path.isfile(
os.path.join(file_collection_dir, "datmoDockerfile"))
output = open(os.path.join(file_collection_dir, "datmoDockerfile"),
"r").read()
print(repr(output))
assert os.path.isfile(
os.path.join(file_collection_dir, "hardware_info"))
output = open(os.path.join(file_collection_dir, "hardware_info"),
"r").read()
print(repr(output))
# 1) Test option 1
environment_obj_0 = self.environment_controller.create(
input_dict_0, save_hardware_file=False)
assert environment_obj_0
assert isinstance(environment_obj_0, Environment)
assert environment_obj_0.id
assert environment_obj_0.driver_type == "docker"
assert environment_obj_0.file_collection_id
assert environment_obj_0.definition_filename
assert environment_obj_0.hardware_info
# Get file collection path
file_collection_obj = self.environment_controller.dal.file_collection. \
get_by_id(environment_obj_0.file_collection_id)
file_collection_dir = self.environment_controller.file_driver. \
get_collection_path(file_collection_obj.filehash)
assert os.path.isfile(os.path.join(file_collection_dir, "test"))
assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile"))
output = open(os.path.join(file_collection_dir, "Dockerfile"),
"r").read()
print(repr(output))
assert os.path.isfile(
os.path.join(file_collection_dir, "datmoDockerfile"))
output = open(os.path.join(file_collection_dir, "datmoDockerfile"),
"r").read()
print(repr(output))
assert environment_obj_0.unique_hash == "c309ae4f58163693a91816988d9dc88b"
assert environment_obj_0.name == "test"
assert environment_obj_0.description == "test description"
# Files ["test", "Dockerfile", "datmoDockerfile"]
# 5) Test option 5
input_dict_1 = {
"name": "test",
"description": "test description",
"paths": [self.definition_filepath],
}
environment_obj = self.environment_controller.create(
input_dict_1, save_hardware_file=False)
assert environment_obj
assert isinstance(environment_obj, Environment)
assert environment_obj.id
assert environment_obj.driver_type == "docker"
assert environment_obj.file_collection_id
assert environment_obj.definition_filename
assert environment_obj.hardware_info
# Get file collection path
file_collection_obj = self.environment_controller.dal.file_collection. \
get_by_id(environment_obj.file_collection_id)
file_collection_dir = self.environment_controller.file_driver. \
get_collection_path(file_collection_obj.filehash)
assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile"))
output = open(os.path.join(file_collection_dir, "Dockerfile"),
"r").read()
print(repr(output))
assert os.path.isfile(
os.path.join(file_collection_dir, "datmoDockerfile"))
output = open(os.path.join(file_collection_dir, "datmoDockerfile"),
"r").read()
print(repr(output))
assert environment_obj.unique_hash == "6e06d7c4d77cb6ae69e7e0efa883ef4b"
assert environment_obj.name == "test"
assert environment_obj.description == "test description"
# Files ["Dockerfile", "datmoDockerfile"]
# remove the project environment directory
shutil.rmtree(
self.environment_controller.file_driver.environment_directory)
# Create environment definition in root directory
home_definition_filepath = os.path.join(
self.environment_controller.home, "Dockerfile")
with open(home_definition_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# 2) Test option 2
environment_obj_1 = self.environment_controller.create(
input_dict_0, save_hardware_file=False)
assert environment_obj_1
assert isinstance(environment_obj_1, Environment)
assert environment_obj_1.id
assert environment_obj_1.driver_type == "docker"
assert environment_obj_1.file_collection_id
assert environment_obj_1.definition_filename
assert environment_obj_1.hardware_info
assert environment_obj_1.unique_hash == file_collection_obj.filehash
# Get file collection path
file_collection_obj = self.environment_controller.dal.file_collection. \
get_by_id(environment_obj_1.file_collection_id)
file_collection_dir = self.environment_controller.file_driver. \
get_collection_path(file_collection_obj.filehash)
assert environment_obj_1.name == "test"
assert environment_obj_1.description == "test description"
assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile"))
assert os.path.isfile(
os.path.join(file_collection_dir, "datmoDockerfile"))
assert environment_obj_1.unique_hash == "6e06d7c4d77cb6ae69e7e0efa883ef4b"
# 3) Test option 3
input_dict_2 = {
"name": "test",
"description": "test description",
"paths": [home_definition_filepath],
}
# Create environment in the project
environment_obj_2 = self.environment_controller.create(
input_dict_2, save_hardware_file=False)
assert environment_obj_2
assert isinstance(environment_obj_2, Environment)
assert environment_obj_2.id
assert environment_obj_2.driver_type == "docker"
assert environment_obj_2.file_collection_id
assert environment_obj_2.definition_filename
assert environment_obj_2.hardware_info
assert environment_obj_2.unique_hash == file_collection_obj.filehash
# Get file collection path
file_collection_obj = self.environment_controller.dal.file_collection. \
get_by_id(environment_obj_2.file_collection_id)
file_collection_dir = self.environment_controller.file_driver. \
get_collection_path(file_collection_obj.filehash)
assert environment_obj_2.name == "test"
assert environment_obj_2.description == "test description"
assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile"))
assert os.path.isfile(
os.path.join(file_collection_dir, "datmoDockerfile"))
assert environment_obj_2.unique_hash == "6e06d7c4d77cb6ae69e7e0efa883ef4b"
# 4) Test option 4
input_dict_3 = {
"paths": [home_definition_filepath + ">Dockerfile"],
}
# Create environment in the project
environment_obj_3 = self.environment_controller.create(
input_dict_3, save_hardware_file=False)
assert environment_obj_3
assert isinstance(environment_obj_3, Environment)
assert environment_obj_3.id
assert environment_obj_3.driver_type == "docker"
assert environment_obj_3.file_collection_id
assert environment_obj_3.definition_filename
assert environment_obj_3.hardware_info
assert environment_obj_3.unique_hash == file_collection_obj.filehash
# Get file collection path
file_collection_obj = self.environment_controller.dal.file_collection. \
get_by_id(environment_obj_3.file_collection_id)
file_collection_dir = self.environment_controller.file_driver. \
get_collection_path(file_collection_obj.filehash)
assert environment_obj_3.name == "test"
assert environment_obj_3.description == "test description"
assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile"))
assert os.path.isfile(
os.path.join(file_collection_dir, "datmoDockerfile"))
assert environment_obj_3.unique_hash == "6e06d7c4d77cb6ae69e7e0efa883ef4b"
# 6) Test option 6
definition_filepath = os.path.join(self.environment_controller.home,
"Dockerfile")
input_dict = {
"paths": [
definition_filepath + ">Dockerfile",
definition_filepath + ">Dockerfile"
],
}
# Create environment in the project
failed = False
try:
_ = self.environment_controller.create(input_dict,
save_hardware_file=False)
except FileAlreadyExistsError:
failed = True
assert failed
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_build(self):
# 1) Test build when no environment given
# 2) Test build when definition path exists and given
# 3) Test build when NO file exists and definition path exists
# 4) Test build when file exists and definition path exists
# 5) Test build when file exists but NO definition path exists
self.project_controller.init("test5", "test description")
self.environment_controller = EnvironmentController()
# 1) Test option 1
failed = False
try:
_ = self.environment_controller.build("does_not_exist")
except EnvironmentDoesNotExist:
failed = True
assert failed
# Create environment definition
definition_filepath = os.path.join(self.environment_controller.home,
"Dockerfile")
random_text = str(uuid.uuid1())
with open(definition_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine" + "\n"))
f.write(to_bytes(str("RUN echo " + random_text)))
input_dict = {
"paths": [definition_filepath],
}
# 2) Test option 2
# Create environment in the project
environment_obj_1 = self.environment_controller.create(input_dict)
result = self.environment_controller.build(environment_obj_1.id)
assert result
# 3) Test option 3
# Create environment in the project
environment_obj_2 = self.environment_controller.create({})
result = self.environment_controller.build(environment_obj_2.id)
assert result
# Create script to test
test_filepath = os.path.join(self.environment_controller.home,
"script.py")
with open(test_filepath, "wb") as f:
f.write(to_bytes("import numpy\n"))
f.write(to_bytes("import sklearn\n"))
f.write(to_bytes("print('hello')\n"))
# 4) Test option 4
environment_obj_3 = self.environment_controller.create({})
result = self.environment_controller.build(environment_obj_3.id)
assert result
# test 2), 3), and 4) will result in the same environment
assert environment_obj_1.id == environment_obj_2.id
assert environment_obj_2.id == environment_obj_3.id
# Test for building dockerfile when there exists not
os.remove(definition_filepath)
# 5) Test option 5
# Create environment definition in project environment directory
definition_filepath = os.path.join(
self.environment_controller.file_driver.environment_directory,
"Dockerfile")
random_text = str(uuid.uuid1())
with open(definition_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine" + "\n"))
f.write(to_bytes(str("RUN echo " + random_text)))
environment_obj_4 = self.environment_controller.create({})
result = self.environment_controller.build(environment_obj_4.id)
assert result
# teardown
self.environment_controller.delete(environment_obj_1.id)
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_run(self):
# Test run simple command with simple Dockerfile
self.project_controller.init("test5", "test description")
self.environment_controller = EnvironmentController()
# 0) Test option 0
# Create environment definition in project environment directory
definition_filepath = os.path.join(
self.environment_controller.file_driver.environment_directory,
"Dockerfile")
random_text = str(uuid.uuid1())
with open(definition_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine" + "\n"))
f.write(to_bytes(str("RUN echo " + random_text)))
random_name = ''.join([
random.choice(string.ascii_letters + string.digits)
for _ in range(32)
])
run_options = {
"command": ["sh", "-c", "echo yo"],
"ports": ["8888:8888"],
"name": random_name,
"volumes": None,
"detach": True,
"stdin_open": False,
"mem_limit": "4g",
"tty": False,
"api": False
}
# Create environment in the project
environment_obj = self.environment_controller.create({})
log_filepath = os.path.join(self.project_controller.home, "task.log")
# Build environment in the project
_ = self.environment_controller.build(environment_obj.id)
# Run environment in the project
return_code, run_id, logs = \
self.environment_controller.run(environment_obj.id, run_options, log_filepath)
assert return_code == 0
assert run_id
assert logs
# teardown
self.environment_controller.delete(environment_obj.id)
shutil.rmtree(
self.environment_controller.file_driver.environment_directory)
# 1) Test option 1
# Create environment definition
definition_filepath = os.path.join(self.environment_controller.home,
"Dockerfile")
random_text = str(uuid.uuid1())
with open(definition_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine" + "\n"))
f.write(to_bytes(str("RUN echo " + random_text)))
random_name = ''.join([
random.choice(string.ascii_letters + string.digits)
for _ in range(32)
])
run_options = {
"command": ["sh", "-c", "echo yo"],
"ports": ["8888:8888"],
"name": random_name,
"volumes": None,
"mem_limit": "4g",
"detach": True,
"stdin_open": False,
"tty": False,
"api": False
}
input_dict = {
"paths": [definition_filepath],
}
# Create environment in the project
environment_obj = self.environment_controller.create(input_dict)
log_filepath = os.path.join(self.project_controller.home, "task.log")
# Build environment in the project
_ = self.environment_controller.build(environment_obj.id)
# Run environment in the project
return_code, run_id, logs = \
self.environment_controller.run(environment_obj.id, run_options, log_filepath)
assert return_code == 0
assert run_id
assert logs
# teardown
self.environment_controller.delete(environment_obj.id)
# 2) Test option 2
os.remove(definition_filepath)
# Create script to test
test_filepath = os.path.join(self.environment_controller.home,
"script.py")
with open(test_filepath, "wb") as f:
f.write(to_bytes("import os\n"))
f.write(to_bytes("import sys\n"))
f.write(to_bytes("print('hello')\n"))
# Create environment in the project
environment_obj = self.environment_controller.create({})
self.environment_controller.build(environment_obj.id)
random_name = ''.join([
random.choice(string.ascii_letters + string.digits)
for _ in range(32)
])
run_options = {
"command": ["sh", "-c", "echo yo"],
"ports": ["8888:8888"],
"name": random_name,
"volumes": {
self.environment_controller.home: {
'bind': '/home/',
'mode': 'rw'
}
},
"mem_limit": "4g",
"detach": False,
"stdin_open": False,
"tty": False,
"api": False
}
# Run environment in the project
return_code, run_id, logs = \
self.environment_controller.run(environment_obj.id, run_options, log_filepath)
assert return_code == 0
assert run_id
assert logs
# teardown
self.environment_controller.delete(environment_obj.id)
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_interactive_run(self):
# 1) Test run interactive terminal in environment
# 2) Test run jupyter notebook in environment
# Create environment definition
self.project_controller.init("test6", "test description")
self.environment_controller = EnvironmentController()
definition_filepath = os.path.join(self.environment_controller.home,
"Dockerfile")
random_text = str(uuid.uuid1())
with open(definition_filepath, "wb") as f:
f.write(to_bytes("FROM datmo/xgboost:cpu" + "\n"))
f.write(to_bytes(str("RUN echo " + random_text)))
input_dict = {
"paths": [definition_filepath],
}
# Create environment in the project
environment_obj = self.environment_controller.create(input_dict)
# 1) Test option 1
@timeout_decorator.timeout(10, use_signals=False)
def timed_run(container_name, timed_run):
run_options = {
"command": [],
"ports": ["8888:8888"],
"name": container_name,
"volumes": None,
"mem_limit": "4g",
"detach": True,
"stdin_open": True,
"tty": True,
"api": False
}
log_filepath = os.path.join(self.project_controller.home,
"task.log")
# Build environment in the project
_ = self.environment_controller.build(environment_obj.id)
# Run environment in the project
self.environment_controller.run(environment_obj.id, run_options,
log_filepath)
return timed_run
container_name = str(uuid.uuid1())
timed_run_result = False
try:
timed_run_result = timed_run(container_name, timed_run_result)
except timeout_decorator.timeout_decorator.TimeoutError:
timed_run_result = True
assert timed_run_result
# teardown
self.environment_controller.delete(environment_obj.id)
# 2) Test option 2
environment_obj = self.environment_controller.create(input_dict)
@timeout_decorator.timeout(10, use_signals=False)
def timed_run(container_name, timed_run):
run_options = {
"command": ["jupyter", "notebook"],
"ports": ["8888:8888"],
"name": container_name,
"volumes": None,
"mem_limit": "4g",
"detach": True,
"stdin_open": False,
"tty": False,
"api": False
}
log_filepath = os.path.join(self.project_controller.home,
"task.log")
# Build environment in the project
_ = self.environment_controller.build(environment_obj.id)
# Run environment in the project
self.environment_controller.run(environment_obj.id, run_options,
log_filepath)
return timed_run
container_name = str(uuid.uuid1())
timed_run_result = False
try:
timed_run_result = timed_run(container_name, timed_run_result)
except timeout_decorator.timeout_decorator.TimeoutError:
timed_run_result = True
assert timed_run_result
# Stop the running environment
# self.environment_controller.stop(container_name)
# teardown
self.environment_controller.delete(environment_obj.id)
def test_list(self):
self.project_controller.init("test4", "test description")
self.environment_controller = EnvironmentController()
# Create environment definition for object 1
definition_path_1 = os.path.join(self.environment_controller.home,
"Dockerfile")
with open(definition_path_1, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
input_dict_1 = {
"paths": [definition_path_1],
}
# Create environment in the project
environment_obj_1 = self.environment_controller.create(input_dict_1)
# Create environment definition for object 2
definition_path_2 = os.path.join(self.environment_controller.home,
"Dockerfile2")
with open(definition_path_2, "wb") as f:
f.write(to_bytes("FROM python:3.4-alpine"))
input_dict_2 = {
"paths": [definition_path_2 + ">Dockerfile"],
}
# Create second environment in the project
environment_obj_2 = self.environment_controller.create(input_dict_2)
# List all environments and ensure they exist
result = self.environment_controller.list()
assert len(result) == 2 and \
environment_obj_1 in result and \
environment_obj_2 in result
def test_update(self):
self.project_controller.init("test5", "test description")
self.environment_controller = EnvironmentController()
# Create environment definition
definition_filepath = os.path.join(self.environment_controller.home,
"Dockerfile")
with open(definition_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
input_dict = {
"paths": [definition_filepath],
}
# Create environment in the project
environment_obj = self.environment_controller.create(input_dict)
# Test success update
new_name = "test name"
new_description = "test description"
result = self.environment_controller.update(
environment_obj.id, name=new_name, description=new_description)
assert result
assert isinstance(result, Environment)
assert result.name == new_name
assert result.description == new_description
# Test failed update
failed = False
try:
self.environment_controller.update("random_id",
name=new_name,
description=new_description)
except EnvironmentDoesNotExist:
failed = True
assert failed
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_delete(self):
self.project_controller.init("test5", "test description")
self.environment_controller = EnvironmentController()
# Create environment definition
definition_filepath = os.path.join(self.environment_controller.home,
"Dockerfile")
with open(definition_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
input_dict = {
"paths": [definition_filepath],
}
# Create environment in the project
environment_obj = self.environment_controller.create(input_dict)
# Delete environment in the project
result = self.environment_controller.delete(environment_obj.id)
# Check if environment retrieval throws error
thrown = False
try:
self.environment_controller.dal.environment.get_by_id(
environment_obj.id)
except EntityNotFound:
thrown = True
assert result == True and \
thrown == True
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_stop_failure(self):
# 1) Test failure with RequiredArgumentMissing
# 2) Test failure with TooManyArgumentsFound
self.project_controller.init("test5", "test description")
self.environment_controller = EnvironmentController()
# 1) Test option 1
failed = False
try:
self.environment_controller.stop()
except RequiredArgumentMissing:
failed = True
assert failed
# 2) Test option 2
failed = False
try:
self.environment_controller.stop(run_id="hello",
match_string="there")
except TooManyArgumentsFound:
failed = True
assert failed
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_stop_success(self):
# TODO: test more run options
# 1) Test run_id input to stop
# 2) Test match_string input to stop
# 3) Test all input to stop
self.project_controller.init("test5", "test description")
self.environment_controller = EnvironmentController()
# Create environment definition
definition_filepath = os.path.join(self.environment_controller.home,
"Dockerfile")
with open(definition_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
run_options = {
"command": ["sh", "-c", "echo yo"],
"ports": ["8888:8888"],
"name": "datmo-task-" + self.environment_controller.model.id +
"-" + "test",
"volumes": None,
"mem_limit": "4g",
"detach": False,
"stdin_open": False,
"tty": False,
"api": False
}
# Create environment definition
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
random_text = str(uuid.uuid1())
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine" + "\n"))
f.write(to_bytes(str("RUN echo " + random_text)))
input_dict = {
"paths": [definition_filepath],
}
# Create environment in the project
environment_obj = self.environment_controller.create(input_dict)
log_filepath = os.path.join(self.project_controller.home, "task.log")
# Build environment in the project
_ = self.environment_controller.build(environment_obj.id)
# 1) Test option 1
_, run_id, _ = \
self.environment_controller.run(environment_obj.id, run_options, log_filepath)
return_code = self.environment_controller.stop(run_id=run_id)
assert return_code
# 2) Test option 2
_, _, _ = \
self.environment_controller.run(environment_obj.id, run_options, log_filepath)
return_code = self.environment_controller.stop(
match_string="datmo-task-" + self.environment_controller.model.id)
assert return_code
# 3) Test option 3
_, _, _ = \
self.environment_controller.run(environment_obj.id, run_options, log_filepath)
run_options_2 = {
"command": ["sh", "-c", "echo yo"],
"ports": ["8888:8888"],
"name": "datmo-task-" + self.environment_controller.model.id +
"-" + "test2",
"volumes": None,
"mem_limit": "4g",
"detach": False,
"stdin_open": False,
"tty": False,
"api": False
}
_, _, _ = \
self.environment_controller.run(environment_obj.id, run_options_2, log_filepath)
return_code = self.environment_controller.stop(all=True)
assert return_code
# teardown
self.environment_controller.delete(environment_obj.id)
def test_exists_env(self):
# Test failure, not initialized
failed = False
try:
_ = self.environment_controller.create({})
except:
failed = True
assert failed
# Setup
self.__setup()
environment_obj = self.environment_controller.create({})
# Check by environment id
result = self.environment_controller.exists(
environment_id=environment_obj.id)
assert result
# Check by unique hash
result = self.environment_controller.exists(
environment_unique_hash=environment_obj.unique_hash)
assert result
# Test with wrong environment id
result = self.environment_controller.exists(
environment_id='test_wrong_env_id')
assert not result
def test_calculate_project_environment_hash(self):
# Setup
self.__setup()
# Test hashing the default (with hardware info)
result = self.environment_controller._calculate_project_environment_hash(
)
assert result
# Test hashing the default Dockerfile
result = self.environment_controller._calculate_project_environment_hash(
save_hardware_file=False)
assert result == "c309ae4f58163693a91816988d9dc88b"
# Test if hash is the same as that of create
environment_obj = self.environment_controller.create(
{}, save_hardware_file=False)
result = self.environment_controller._calculate_project_environment_hash(
save_hardware_file=False)
assert result == "c309ae4f58163693a91816988d9dc88b"
assert result == environment_obj.unique_hash
# Test if the hash is the same if the same file is passed in as an input
input_dict = {
"paths": [self.definition_filepath, self.random_filepath]
}
environment_obj_1 = self.environment_controller.create(
input_dict, save_hardware_file=False)
result = self.environment_controller._calculate_project_environment_hash(
save_hardware_file=False)
assert result == "c309ae4f58163693a91816988d9dc88b"
assert result == environment_obj_1.unique_hash
def test_has_unstaged_changes(self):
# Setup
self.__setup()
_ = self.environment_controller.create({})
# Check for no unstaged changes
result = self.environment_controller._has_unstaged_changes()
assert not result
# Make a change to the file (update python version)
with open(
os.path.join(
self.environment_controller.file_driver.
environment_directory, "Dockerfile"), "wb") as f:
f.write(to_bytes("FROM python:3.6-alpine"))
# Check again, should have unstaged changes
result = self.environment_controller._has_unstaged_changes()
assert result
def test_check_unstaged_changes(self):
# Setup
self.__setup()
obj = self.environment_controller.create({})
# 1) After commiting the changes
# Check for no unstaged changes because already committed
result = self.environment_controller.check_unstaged_changes()
assert not result
# Add a new file
with open(
os.path.join(
self.environment_controller.file_driver.
environment_directory, "test2"), "wb") as f:
f.write(to_bytes("cool"))
# 2) Not commiting the changes, should error and raise UnstagedChanges
failed = False
try:
self.environment_controller.check_unstaged_changes()
except UnstagedChanges:
failed = True
assert failed
# Remove new file
os.remove(
os.path.join(
self.environment_controller.file_driver.environment_directory,
"test2"))
# 3) Files are the same as before but no new commit, should have no unstaged changes
result = self.environment_controller.check_unstaged_changes()
assert not result
# 4) Remove another file, now it is different and should have unstaged changes
os.remove(
os.path.join(
self.environment_controller.file_driver.environment_directory,
"test"))
failed = False
try:
self.environment_controller.check_unstaged_changes()
except UnstagedChanges:
failed = True
assert failed
# 5) Remove the rest of the files, now it is empty and should return as already staged
os.remove(
os.path.join(
self.environment_controller.file_driver.environment_directory,
"Dockerfile"))
result = self.environment_controller.check_unstaged_changes()
assert not result
def test_checkout(self):
# Setup and create all environment files
self.__setup()
# Create environment to checkout to with defaults
environment_obj = self.environment_controller.create({})
# Checkout success with there are no unstaged changes
result = self.environment_controller.checkout(environment_obj.id)
assert result
current_hash = self.environment_controller._calculate_project_environment_hash(
)
assert environment_obj.unique_hash == current_hash
# Check the filenames as well because the hash does not take this into account
assert os.path.isfile(
os.path.join(
self.environment_controller.file_driver.environment_directory,
"test"))
assert os.path.isfile(
os.path.join(
self.environment_controller.file_driver.environment_directory,
"Dockerfile"))
assert not os.path.isfile(
os.path.join(
self.environment_controller.file_driver.environment_directory,
"datmoDockerfile"))
assert not os.path.isfile(
os.path.join(
self.environment_controller.file_driver.environment_directory,
"hardware_info"))
# Change file contents to make it unstaged
with open(self.definition_filepath, "wb") as f:
f.write(to_bytes("new content"))
# Checkout failure with unstaged changes
failed = False
try:
_ = self.environment_controller.checkout(environment_obj.id)
except UnstagedChanges:
failed = True
assert failed
# Create new environment to checkout to with defaults (no hardware)
environment_obj_1 = self.environment_controller.create(
{}, save_hardware_file=False)
# Checkout success with there are no unstaged changes
result = self.environment_controller.checkout(environment_obj.id)
assert result
current_hash = self.environment_controller._calculate_project_environment_hash(
)
assert environment_obj.unique_hash == current_hash
assert environment_obj_1.unique_hash != current_hash
# Check the filenames as well because the hash does not take this into account
assert os.path.isfile(
os.path.join(
self.environment_controller.file_driver.environment_directory,
"test"))
assert os.path.isfile(
os.path.join(
self.environment_controller.file_driver.environment_directory,
"Dockerfile"))
assert not os.path.isfile(
os.path.join(
self.environment_controller.file_driver.environment_directory,
"datmoDockerfile"))
assert not os.path.isfile(
os.path.join(
self.environment_controller.file_driver.environment_directory,
"hardware_info"))
class TestProjectController():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.project_controller = ProjectController()
self.environment_ids = []
def teardown_method(self):
if not check_docker_inactive(test_datmo_dir):
self.project_controller = ProjectController()
if self.project_controller.is_initialized:
self.environment_controller = EnvironmentController()
for env_id in list(set(self.environment_ids)):
if not self.environment_controller.delete(env_id):
raise Exception
def test_init_failure_none(self):
# Test failed case
failed = False
try:
self.project_controller.init(None, None)
except ValidationFailed:
failed = True
assert failed
def test_init_failure_empty_str(self):
# Test failed case
failed = False
try:
self.project_controller.init("", "")
except ValidationFailed:
failed = True
assert failed
assert not self.project_controller.code_driver.is_initialized
assert not self.project_controller.file_driver.is_initialized
def test_init_failure_git_code_driver(self):
# Create a HEAD.lock file in .git to make GitCodeDriver.init() fail
if self.project_controller.code_driver.type == "git":
git_dir = os.path.join(
self.project_controller.code_driver.filepath, ".git")
os.makedirs(git_dir)
with open(os.path.join(git_dir, "HEAD.lock"), "a+") as f:
f.write(to_bytes("test"))
failed = False
try:
self.project_controller.init("test1", "test description")
except Exception:
failed = True
assert failed
assert not self.project_controller.code_driver.is_initialized
assert not self.project_controller.file_driver.is_initialized
def test_init_success(self):
result = self.project_controller.init("test1", "test description")
# Tested with is_initialized
assert self.project_controller.model.name == "test1"
assert self.project_controller.model.description == "test description"
assert result and self.project_controller.is_initialized
# Changeable by user, not tested in is_initialized
assert self.project_controller.current_session.name == "default"
# TODO: Test lower level functions (DAL, JSONStore, etc for interruptions)
# def test_init_with_interruption(self):
# # Reinitializing after timed interruption during init
# @timeout_decorator.timeout(0.001, use_signals=False)
# def timed_init_with_interruption():
# result = self.project_controller.init("test1", "test description")
# return result
#
# failed = False
# try:
# timed_init_with_interruption()
# except timeout_decorator.timeout_decorator.TimeoutError:
# failed = True
# # Tested with is_initialized
# assert failed
#
# # Reperforming init after a wait of 2 seconds
# time.sleep(2)
# result = self.project_controller.init("test2", "test description")
# # Tested with is_initialized
# assert self.project_controller.model.name == "test2"
# assert self.project_controller.model.description == "test description"
# assert result and self.project_controller.is_initialized
#
# # Changeable by user, not tested in is_initialized
# assert self.project_controller.current_session.name == "default"
def test_init_reinit_failure_empty_str(self):
_ = self.project_controller.init("test1", "test description")
failed = True
try:
self.project_controller.init("", "")
except Exception:
failed = True
assert failed
assert self.project_controller.model.name == "test1"
assert self.project_controller.model.description == "test description"
assert self.project_controller.code_driver.is_initialized
assert self.project_controller.file_driver.is_initialized
def test_init_reinit_success(self):
_ = self.project_controller.init("test1", "test description")
# Test out functionality for re-initialize project
result = self.project_controller.init("anything", "else")
assert self.project_controller.model.name == "anything"
assert self.project_controller.model.description == "else"
assert result == True
def test_cleanup_no_environment(self):
self.project_controller.init("test2", "test description")
result = self.project_controller.cleanup()
assert not self.project_controller.code_driver.is_initialized
assert not self.project_controller.file_driver.is_initialized
# Ensure that containers built with this image do not exist
# assert not self.project_controller.environment_driver.list_containers(filters={
# "ancestor": image_id
# })
assert result == True
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_cleanup_with_environment(self):
self.project_controller.init("test2", "test description")
result = self.project_controller.cleanup()
assert not self.project_controller.code_driver.is_initialized
assert not self.project_controller.file_driver.is_initialized
assert not self.project_controller.environment_driver.list_images(
"datmo-test2")
# Ensure that containers built with this image do not exist
# assert not self.project_controller.environment_driver.list_containers(filters={
# "ancestor": image_id
# })
assert result == True
def test_status_basic(self):
self.project_controller.init("test3", "test description")
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
self.project_controller.status()
assert status_dict
assert isinstance(status_dict, dict)
assert status_dict['name'] == "test3"
assert status_dict['description'] == "test description"
assert isinstance(status_dict['config'], dict)
assert not current_snapshot
assert not latest_snapshot_user_generated
assert not latest_snapshot_auto_generated
assert unstaged_code # no files, but unstaged because blank commit id has not yet been created (no initial snapshot)
assert not unstaged_environment
assert not unstaged_files
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_status_snapshot_task(self):
self.project_controller.init("test4", "test description")
self.snapshot_controller = SnapshotController()
self.task_controller = TaskController()
# Create files to add
self.snapshot_controller.file_driver.create("dirpath1", directory=True)
self.snapshot_controller.file_driver.create("dirpath2", directory=True)
self.snapshot_controller.file_driver.create("filepath1")
# Create environment definition
env_def_path = os.path.join(self.snapshot_controller.home,
"Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
environment_paths = [env_def_path]
# Create config
config_filepath = os.path.join(self.snapshot_controller.home,
"config.json")
with open(config_filepath, "wb") as f:
f.write(to_bytes(str("{}")))
# Create stats
stats_filepath = os.path.join(self.snapshot_controller.home,
"stats.json")
with open(stats_filepath, "wb") as f:
f.write(to_bytes(str("{}")))
input_dict = {
"message":
"my test snapshot",
"paths": [
os.path.join(self.snapshot_controller.home, "dirpath1"),
os.path.join(self.snapshot_controller.home, "dirpath2"),
os.path.join(self.snapshot_controller.home, "filepath1")
],
"environment_paths":
environment_paths,
"config_filename":
config_filepath,
"stats_filename":
stats_filepath,
}
# Create snapshot in the project, then wait, and try status
first_snapshot = self.snapshot_controller.create(input_dict)
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
self.project_controller.status()
assert status_dict
assert isinstance(status_dict, dict)
assert status_dict['name'] == "test4"
assert status_dict['description'] == "test description"
assert isinstance(status_dict['config'], dict)
assert not current_snapshot # snapshot was created from other environments and files (so user is not on any current snapshot)
assert isinstance(latest_snapshot_user_generated, Snapshot)
assert latest_snapshot_user_generated == first_snapshot
assert not latest_snapshot_auto_generated
assert not unstaged_code
assert not unstaged_environment
assert not unstaged_files
# Create and run a task and test if task is shown
first_task = self.task_controller.create()
# Create task_dict
task_command = ["sh", "-c", "echo accuracy:0.45"]
task_dict = {"command_list": task_command}
updated_first_task = self.task_controller.run(
first_task.id, task_dict=task_dict)
before_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_first_task.before_snapshot_id)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_first_task.after_snapshot_id)
before_environment_obj = self.task_controller.dal.environment.get_by_id(
before_snapshot_obj.environment_id)
after_environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
assert before_environment_obj == after_environment_obj
self.environment_ids.append(after_environment_obj.id)
status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
self.project_controller.status()
assert status_dict
assert isinstance(status_dict, dict)
assert status_dict['name'] == "test4"
assert status_dict['description'] == "test description"
assert isinstance(status_dict['config'], dict)
assert isinstance(current_snapshot, Snapshot)
assert isinstance(latest_snapshot_user_generated, Snapshot)
assert latest_snapshot_user_generated == first_snapshot
assert isinstance(latest_snapshot_auto_generated, Snapshot)
# current snapshot is the before snapshot for the run
assert current_snapshot == before_snapshot_obj
assert current_snapshot != latest_snapshot_auto_generated
assert current_snapshot != latest_snapshot_user_generated
# latest autogenerated snapshot is the after snapshot id
assert latest_snapshot_auto_generated == after_snapshot_obj
assert latest_snapshot_auto_generated != latest_snapshot_user_generated
# user generated snapshot is not associated with any before or after snapshot
assert latest_snapshot_user_generated != before_snapshot_obj
assert latest_snapshot_user_generated != after_snapshot_obj
assert not unstaged_code
assert not unstaged_environment
assert not unstaged_files
class TestSnapshotController():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
Config().set_home(self.temp_dir)
self.environment_ids = []
def teardown_method(self):
if not check_docker_inactive(test_datmo_dir,
Config().datmo_directory_name):
self.__setup()
self.environment_controller = EnvironmentController()
for env_id in list(set(self.environment_ids)):
if not self.environment_controller.delete(env_id):
raise Exception
def __setup(self):
Config().set_home(self.temp_dir)
self.project_controller = ProjectController()
self.project_controller.init("test", "test description")
self.task_controller = TaskController()
self.snapshot_controller = SnapshotController()
def test_init_fail_project_not_init(self):
Config().set_home(self.temp_dir)
failed = False
try:
SnapshotController()
except ProjectNotInitialized:
failed = True
assert failed
def test_init_fail_invalid_path(self):
test_home = "some_random_dir"
Config().set_home(test_home)
failed = False
try:
SnapshotController()
except InvalidProjectPath:
failed = True
assert failed
def test_current_snapshot(self):
self.__setup()
# Test failure for unstaged changes
failed = False
try:
self.snapshot_controller.current_snapshot()
except UnstagedChanges:
failed = True
assert failed
# Test success after snapshot created
snapshot_obj = self.__default_create()
current_snapshot_obj = self.snapshot_controller.current_snapshot()
assert current_snapshot_obj == snapshot_obj
def test_create_fail_no_message(self):
self.__setup()
# Test no message
failed = False
try:
self.snapshot_controller.create({})
except RequiredArgumentMissing:
failed = True
assert failed
def test_create_success_no_code(self):
self.__setup()
# Test default values for snapshot, fail due to code
result = self.snapshot_controller.create(
{"message": "my test snapshot"})
assert result
def test_create_success_no_code_environment(self):
self.__setup()
# Create environment definition
env_def_path = os.path.join(
self.project_controller.environment_driver.
environment_directory_path, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# test must pass when there is file present in root project folder
result = self.snapshot_controller.create(
{"message": "my test snapshot"})
assert result
def test_create_success_no_code_environment_files(self):
self.__setup()
# Create environment definition
env_def_path = os.path.join(
self.project_controller.environment_driver.
environment_directory_path, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
test_file = os.path.join(
self.project_controller.file_driver.files_directory, "test.txt")
with open(test_file, "wb") as f:
f.write(to_bytes(str("hello")))
# test must pass when there is file present in root project folder
result = self.snapshot_controller.create(
{"message": "my test snapshot"})
assert result
def test_create_no_environment_detected_in_file(self):
self.__setup()
# Test default values for snapshot, fail due to no environment from file
self.snapshot_controller.file_driver.create("filepath1")
snapshot_obj_0 = self.snapshot_controller.create(
{"message": "my test snapshot"})
assert isinstance(snapshot_obj_0, Snapshot)
assert snapshot_obj_0.code_id
assert snapshot_obj_0.environment_id
assert snapshot_obj_0.file_collection_id
assert snapshot_obj_0.config == {}
assert snapshot_obj_0.stats == {}
def test_create_success_default_detected_in_file(self):
self.__setup()
# Test default values for snapshot when there is no environment
test_filepath = os.path.join(self.snapshot_controller.home,
"script.py")
with open(test_filepath, "wb") as f:
f.write(to_bytes("import os\n"))
f.write(to_bytes("import sys\n"))
f.write(to_bytes("print('hello')\n"))
snapshot_obj_1 = self.snapshot_controller.create(
{"message": "my test snapshot"})
assert isinstance(snapshot_obj_1, Snapshot)
assert snapshot_obj_1.code_id
assert snapshot_obj_1.environment_id
assert snapshot_obj_1.file_collection_id
assert snapshot_obj_1.config == {}
assert snapshot_obj_1.stats == {}
def test_create_success_default_env_def(self):
self.__setup()
# Create environment definition
env_def_path = os.path.join(
self.project_controller.environment_driver.
environment_directory_path, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# Creating a file in project folder
test_filepath = os.path.join(self.snapshot_controller.home,
"script.py")
with open(test_filepath, "wb") as f:
f.write(to_bytes("import numpy\n"))
f.write(to_bytes("import sklearn\n"))
f.write(to_bytes("print('hello')\n"))
# Test default values for snapshot, success
snapshot_obj = self.snapshot_controller.create(
{"message": "my test snapshot"})
assert isinstance(snapshot_obj, Snapshot)
assert snapshot_obj.code_id
assert snapshot_obj.environment_id
assert snapshot_obj.file_collection_id
assert snapshot_obj.config == {}
assert snapshot_obj.stats == {}
def test_create_success_with_environment(self):
self.__setup()
# Create environment definition
env_def_path = os.path.join(
self.project_controller.environment_driver.
environment_directory_path, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# creating a file in project folder
test_filepath = os.path.join(self.snapshot_controller.home,
"script.py")
with open(test_filepath, "wb") as f:
f.write(to_bytes("import numpy\n"))
f.write(to_bytes("import sklearn\n"))
f.write(to_bytes("print('hello')\n"))
# Test default values for snapshot, success
snapshot_obj = self.snapshot_controller.create(
{"message": "my test snapshot"})
assert isinstance(snapshot_obj, Snapshot)
assert snapshot_obj.code_id
assert snapshot_obj.environment_id
assert snapshot_obj.file_collection_id
assert snapshot_obj.config == {}
assert snapshot_obj.stats == {}
def test_create_success_env_paths(self):
self.__setup()
# Create environment definition
random_dir = os.path.join(self.snapshot_controller.home, "random_dir")
os.makedirs(random_dir)
env_def_path = os.path.join(random_dir, "randomDockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
environment_paths = [env_def_path + ">Dockerfile"]
# Test default values for snapshot, success
snapshot_obj = self.snapshot_controller.create({
"message":
"my test snapshot",
"environment_paths":
environment_paths
})
assert isinstance(snapshot_obj, Snapshot)
assert snapshot_obj.code_id
assert snapshot_obj.environment_id
assert snapshot_obj.file_collection_id
assert snapshot_obj.config == {}
assert snapshot_obj.stats == {}
def test_create_success_default_env_def_duplicate(self):
self.__setup()
# Test 2 snapshots with same parameters
# Create environment definition
env_def_path = os.path.join(
self.project_controller.environment_driver.
environment_directory_path, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# Creating a file in project folder
test_filepath = os.path.join(self.snapshot_controller.home,
"script.py")
with open(test_filepath, "wb") as f:
f.write(to_bytes("import numpy\n"))
f.write(to_bytes("import sklearn\n"))
f.write(to_bytes("print('hello')\n"))
snapshot_obj = self.snapshot_controller.create(
{"message": "my test snapshot"})
snapshot_obj_1 = self.snapshot_controller.create(
{"message": "my test snapshot"})
# Should return the same object back
assert snapshot_obj_1.id == snapshot_obj.id
assert snapshot_obj_1.code_id == snapshot_obj.code_id
assert snapshot_obj_1.environment_id == \
snapshot_obj.environment_id
assert snapshot_obj_1.file_collection_id == \
snapshot_obj.file_collection_id
assert snapshot_obj_1.config == \
snapshot_obj.config
assert snapshot_obj_1.stats == \
snapshot_obj.stats
def test_create_success_given_files_env_def_config_file_stats_file(self):
self.__setup()
# Create environment definition
env_def_path = os.path.join(
self.project_controller.environment_driver.
environment_directory_path, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# Creating a file in project folder
test_filepath = os.path.join(self.snapshot_controller.home,
"script.py")
with open(test_filepath, "wb") as f:
f.write(to_bytes("import numpy\n"))
f.write(to_bytes("import sklearn\n"))
f.write(to_bytes("print('hello')\n"))
snapshot_obj = self.snapshot_controller.create(
{"message": "my test snapshot"})
# Create files to add
_, files_directory_name = os.path.split(
self.project_controller.file_driver.files_directory)
files_directory_relative_path = os.path.join(
self.project_controller.file_driver.datmo_directory_name,
files_directory_name)
self.snapshot_controller.file_driver.create(os.path.join(
files_directory_relative_path, "dirpath1"),
directory=True)
self.snapshot_controller.file_driver.create(os.path.join(
files_directory_relative_path, "dirpath2"),
directory=True)
self.snapshot_controller.file_driver.create(
os.path.join(files_directory_relative_path, "filepath1"))
# Create config
config_filepath = os.path.join(self.snapshot_controller.home,
"config.json")
with open(config_filepath, "wb") as f:
f.write(to_bytes(str('{"foo":"bar"}')))
# Create stats
stats_filepath = os.path.join(self.snapshot_controller.home,
"stats.json")
with open(stats_filepath, "wb") as f:
f.write(to_bytes(str('{"foo":"bar"}')))
input_dict = {
"message": "my test snapshot",
"config_filepath": config_filepath,
"stats_filepath": stats_filepath,
}
# Create snapshot in the project
snapshot_obj_4 = self.snapshot_controller.create(input_dict)
assert snapshot_obj_4 != snapshot_obj
assert snapshot_obj_4.code_id != snapshot_obj.code_id
assert snapshot_obj_4.environment_id == \
snapshot_obj.environment_id
assert snapshot_obj_4.file_collection_id != \
snapshot_obj.file_collection_id
assert snapshot_obj_4.config == {"foo": "bar"}
assert snapshot_obj_4.stats == {"foo": "bar"}
def test_create_success_given_files_env_def_different_config_stats(self):
self.__setup()
# Create environment definition
env_def_path = os.path.join(
self.project_controller.environment_driver.
environment_directory_path, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# Creating a file in project folder
test_filepath = os.path.join(self.snapshot_controller.home,
"script.py")
with open(test_filepath, "wb") as f:
f.write(to_bytes("import numpy\n"))
f.write(to_bytes("import sklearn\n"))
f.write(to_bytes("print('hello')\n"))
snapshot_obj = self.snapshot_controller.create(
{"message": "my test snapshot"})
# Create files to add
_, files_directory_name = os.path.split(
self.project_controller.file_driver.files_directory)
files_directory_relative_path = os.path.join(
self.project_controller.file_driver.datmo_directory_name,
files_directory_name)
self.snapshot_controller.file_driver.create(os.path.join(
files_directory_relative_path, "dirpath1"),
directory=True)
self.snapshot_controller.file_driver.create(os.path.join(
files_directory_relative_path, "dirpath2"),
directory=True)
self.snapshot_controller.file_driver.create(
os.path.join(files_directory_relative_path, "filepath1"))
# Create config
config_filepath = os.path.join(self.snapshot_controller.home,
"config.json")
with open(config_filepath, "wb") as f:
f.write(to_bytes(str('{"foo":"bar"}')))
# Create stats
stats_filepath = os.path.join(self.snapshot_controller.home,
"stats.json")
with open(stats_filepath, "wb") as f:
f.write(to_bytes(str('{"foo":"bar"}')))
# Test different config and stats inputs
input_dict = {
"message": "my test snapshot",
"config_filename": "different_name",
"stats_filename": "different_name",
}
# Create snapshot in the project
snapshot_obj_1 = self.snapshot_controller.create(input_dict)
assert snapshot_obj_1 != snapshot_obj
assert snapshot_obj_1.config == {}
assert snapshot_obj_1.stats == {}
def test_create_success_given_files_env_def_direct_config_stats(self):
self.__setup()
# Create environment definition
env_def_path = os.path.join(
self.project_controller.environment_driver.
environment_directory_path, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# Create files to add
_, files_directory_name = os.path.split(
self.project_controller.file_driver.files_directory)
files_directory_relative_path = os.path.join(
self.project_controller.file_driver.datmo_directory_name,
files_directory_name)
self.snapshot_controller.file_driver.create(os.path.join(
files_directory_relative_path, "dirpath1"),
directory=True)
self.snapshot_controller.file_driver.create(os.path.join(
files_directory_relative_path, "dirpath2"),
directory=True)
self.snapshot_controller.file_driver.create(
os.path.join(files_directory_relative_path, "filepath1"))
# Creating a file in project folder
test_filepath = os.path.join(self.snapshot_controller.home,
"script.py")
with open(test_filepath, "wb") as f:
f.write(to_bytes("import numpy\n"))
f.write(to_bytes("import sklearn\n"))
f.write(to_bytes("print('hello')\n"))
# Test different config and stats inputs
input_dict = {
"message": "my test snapshot",
"config": {
"foo": "bar"
},
"stats": {
"foo": "bar"
},
}
# Create snapshot in the project
snapshot_obj_6 = self.snapshot_controller.create(input_dict)
assert snapshot_obj_6.config == {"foo": "bar"}
assert snapshot_obj_6.stats == {"foo": "bar"}
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_create_from_task(self):
self.__setup()
# 0) Test if fails with TaskNotComplete error
# 1) Test if success with empty task files, results
# 2) Test if success with task files, results, and message
# 3) Test if success with message, label, config and stats
# 4) Test if success with updated stats from after_snapshot_id and task_results
# Create task in the project
task_obj = self.task_controller.create()
# 0) Test option 0
failed = False
try:
_ = self.snapshot_controller.create_from_task(
message="my test snapshot", task_id=task_obj.id)
except TaskNotComplete:
failed = True
assert failed
# 1) Test option 1
# Create task_dict
task_command = ["sh", "-c", "echo test"]
task_dict = {"command_list": task_command}
# Create environment definition
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
updated_task_obj = self.task_controller.run(task_obj.id,
task_dict=task_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
snapshot_obj = self.snapshot_controller.create_from_task(
message="my test snapshot", task_id=updated_task_obj.id)
assert isinstance(snapshot_obj, Snapshot)
assert snapshot_obj.id == updated_task_obj.after_snapshot_id
assert snapshot_obj.message == "my test snapshot"
assert snapshot_obj.stats == updated_task_obj.results
assert snapshot_obj.visible == True
# Create new task and corresponding dict
task_obj = self.task_controller.create()
task_command = ["sh", "-c", "echo accuracy:0.45"]
task_dict = {"command_list": task_command}
# Create environment definition
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# Test the default values
updated_task_obj = self.task_controller.run(task_obj.id,
task_dict=task_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
# 2) Test option 2
snapshot_obj = self.snapshot_controller.create_from_task(
message="my test snapshot", task_id=updated_task_obj.id)
assert isinstance(snapshot_obj, Snapshot)
assert snapshot_obj.id == updated_task_obj.after_snapshot_id
assert snapshot_obj.message == "my test snapshot"
assert snapshot_obj.stats == updated_task_obj.results
assert snapshot_obj.visible == True
# 3) Test option 3
test_config = {"algo": "regression"}
test_stats = {"accuracy": 0.9}
snapshot_obj = self.snapshot_controller.create_from_task(
message="my test snapshot",
task_id=updated_task_obj.id,
label="best",
config=test_config,
stats=test_stats)
assert isinstance(snapshot_obj, Snapshot)
assert snapshot_obj.id == updated_task_obj.after_snapshot_id
assert snapshot_obj.message == "my test snapshot"
assert snapshot_obj.label == "best"
assert snapshot_obj.config == test_config
assert snapshot_obj.stats == test_stats
assert snapshot_obj.visible == True
# 4) Test option 4
test_config = {"algo": "regression"}
test_stats = {"new_key": 0.9}
task_obj_2 = self.task_controller.create()
updated_task_obj_2 = self.task_controller.run(task_obj_2.id,
task_dict=task_dict,
snapshot_dict={
"config":
test_config,
"stats": test_stats
})
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj_2.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
snapshot_obj = self.snapshot_controller.create_from_task(
message="my test snapshot",
task_id=updated_task_obj_2.id,
label="best")
updated_stats_dict = {}
updated_stats_dict.update(test_stats)
updated_stats_dict.update(updated_task_obj.results)
assert isinstance(snapshot_obj, Snapshot)
assert snapshot_obj.id == updated_task_obj_2.after_snapshot_id
assert snapshot_obj.message == "my test snapshot"
assert snapshot_obj.label == "best"
assert snapshot_obj.stats == updated_stats_dict
assert snapshot_obj.visible == True
def __default_create(self):
# Create files to add
_, files_directory_name = os.path.split(
self.project_controller.file_driver.files_directory)
files_directory_relative_path = os.path.join(
self.project_controller.file_driver.datmo_directory_name,
files_directory_name)
self.snapshot_controller.file_driver.create(os.path.join(
files_directory_relative_path, "dirpath1"),
directory=True)
self.snapshot_controller.file_driver.create(os.path.join(
files_directory_relative_path, "dirpath2"),
directory=True)
self.snapshot_controller.file_driver.create(
os.path.join(files_directory_relative_path, "filepath1"))
self.snapshot_controller.file_driver.create("filepath2")
with open(os.path.join(self.snapshot_controller.home, "filepath2"),
"wb") as f:
f.write(to_bytes(str("import sys\n")))
# Create environment_driver definition
env_def_path = os.path.join(
self.project_controller.environment_driver.
environment_directory_path, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# Create config
config_filepath = os.path.join(self.snapshot_controller.home,
"config.json")
with open(config_filepath, "wb") as f:
f.write(to_bytes(str("{}")))
# Create stats
stats_filepath = os.path.join(self.snapshot_controller.home,
"stats.json")
with open(stats_filepath, "wb") as f:
f.write(to_bytes(str("{}")))
input_dict = {
"message": "my test snapshot",
"config_filename": config_filepath,
"stats_filename": stats_filepath,
}
# Create snapshot in the project
return self.snapshot_controller.create(input_dict)
def test_check_unstaged_changes(self):
self.__setup()
# Check unstaged changes
failed = False
try:
self.snapshot_controller.check_unstaged_changes()
except UnstagedChanges:
failed = True
assert failed
# Check no unstaged changes
_ = self.__default_create()
result = self.snapshot_controller.check_unstaged_changes()
assert result == False
def test_checkout(self):
self.__setup()
# Create snapshot
snapshot_obj_1 = self.__default_create()
# Create duplicate snapshot in project
self.snapshot_controller.file_driver.create("test")
snapshot_obj_2 = self.__default_create()
assert snapshot_obj_2 != snapshot_obj_1
# Checkout to snapshot 1 using snapshot id
result = self.snapshot_controller.checkout(snapshot_obj_1.id)
# TODO: Check for which snapshot we are on
assert result == True
def test_list(self):
self.__setup()
# Create file to add to snapshot
test_filepath_1 = os.path.join(self.snapshot_controller.home,
"test.txt")
with open(test_filepath_1, "wb") as f:
f.write(to_bytes(str("test")))
# Create snapshot in the project
snapshot_obj_1 = self.__default_create()
# Create file to add to second snapshot
test_filepath_2 = os.path.join(self.snapshot_controller.home,
"test2.txt")
with open(test_filepath_2, "wb") as f:
f.write(to_bytes(str("test2")))
# Create second snapshot in the project
snapshot_obj_2 = self.__default_create()
# List all snapshots and ensure they exist
result = self.snapshot_controller.list()
assert len(result) == 2 and \
snapshot_obj_1 in result and \
snapshot_obj_2 in result
# List all tasks regardless of filters in ascending
result = self.snapshot_controller.list(sort_key='created_at',
sort_order='ascending')
assert len(result) == 2 and \
snapshot_obj_1 in result and \
snapshot_obj_2 in result
assert result[0].created_at <= result[-1].created_at
# List all tasks regardless of filters in descending
result = self.snapshot_controller.list(sort_key='created_at',
sort_order='descending')
assert len(result) == 2 and \
snapshot_obj_1 in result and \
snapshot_obj_2 in result
assert result[0].created_at >= result[-1].created_at
# Wrong order being passed in
failed = False
try:
_ = self.snapshot_controller.list(sort_key='created_at',
sort_order='wrong_order')
except InvalidArgumentType:
failed = True
assert failed
# Wrong key and order being passed in
failed = False
try:
_ = self.snapshot_controller.list(sort_key='wrong_key',
sort_order='wrong_order')
except InvalidArgumentType:
failed = True
assert failed
# wrong key and right order being passed in
expected_result = self.snapshot_controller.list(sort_key='created_at',
sort_order='ascending')
result = self.snapshot_controller.list(sort_key='wrong_key',
sort_order='ascending')
expected_ids = [item.id for item in expected_result]
ids = [item.id for item in result]
assert set(expected_ids) == set(ids)
# List snapshots with visible filter
result = self.snapshot_controller.list(visible=False)
assert len(result) == 0
result = self.snapshot_controller.list(visible=True)
assert len(result) == 2 and \
snapshot_obj_1 in result and \
snapshot_obj_2 in result
def test_update(self):
self.__setup()
test_config = {"config_foo": "bar"}
test_stats = {"stats_foo": "bar"}
test_message = 'test_message'
test_label = 'test_label'
# Updating all config, stats, message and label
# Create snapshot in the project
snapshot_obj = self.__default_create()
# Update snapshot in the project
self.snapshot_controller.update(snapshot_obj.id,
config=test_config,
stats=test_stats,
message=test_message,
label=test_label)
# Get the updated snapshot obj
updated_snapshot_obj = self.snapshot_controller.dal.snapshot.get_by_id(
snapshot_obj.id)
assert updated_snapshot_obj.config == test_config
assert updated_snapshot_obj.stats == test_stats
assert updated_snapshot_obj.message == test_message
assert updated_snapshot_obj.label == test_label
# Updating config, stats
# Create snapshot in the project
snapshot_obj = self.__default_create()
# Update snapshot in the project
self.snapshot_controller.update(snapshot_obj.id,
config=test_config,
stats=test_stats)
# Get the updated snapshot obj
updated_snapshot_obj = self.snapshot_controller.dal.snapshot.get_by_id(
snapshot_obj.id)
assert updated_snapshot_obj.config == test_config
assert updated_snapshot_obj.stats == test_stats
# Updating both message and label
# Create snapshot in the project
snapshot_obj = self.__default_create()
# Update snapshot in the project
self.snapshot_controller.update(snapshot_obj.id,
message=test_message,
label=test_label)
# Get the updated snapshot obj
updated_snapshot_obj = self.snapshot_controller.dal.snapshot.get_by_id(
snapshot_obj.id)
assert updated_snapshot_obj.message == test_message
assert updated_snapshot_obj.label == test_label
# Updating only message
# Create snapshot in the project
snapshot_obj_1 = self.__default_create()
# Update snapshot in the project
self.snapshot_controller.update(snapshot_obj_1.id,
message=test_message)
# Get the updated snapshot obj
updated_snapshot_obj_1 = self.snapshot_controller.dal.snapshot.get_by_id(
snapshot_obj_1.id)
assert updated_snapshot_obj_1.message == test_message
# Updating only label
# Create snapshot in the project
snapshot_obj_2 = self.__default_create()
# Update snapshot in the project
self.snapshot_controller.update(snapshot_obj_2.id, label=test_label)
# Get the updated snapshot obj
updated_snapshot_obj_2 = self.snapshot_controller.dal.snapshot.get_by_id(
snapshot_obj_2.id)
assert updated_snapshot_obj_2.label == test_label
def test_get(self):
self.__setup()
# Test failure for no snapshot
failed = False
try:
self.snapshot_controller.get("random")
except DoesNotExist:
failed = True
assert failed
# Test success for snapshot
snapshot_obj = self.__default_create()
snapshot_obj_returned = self.snapshot_controller.get(snapshot_obj.id)
assert snapshot_obj == snapshot_obj_returned
def test_get_files(self):
self.__setup()
# Test failure case
failed = False
try:
self.snapshot_controller.get_files("random")
except DoesNotExist:
failed = True
assert failed
# Test success case
snapshot_obj = self.__default_create()
result = self.snapshot_controller.get_files(snapshot_obj.id)
file_collection_obj = self.task_controller.dal.file_collection.get_by_id(
snapshot_obj.file_collection_id)
file_names = [item.name for item in result]
assert len(result) == 1
for item in result:
assert isinstance(item, TextIOWrapper)
assert item.mode == "r"
assert os.path.join(self.task_controller.home, ".datmo", "collections",
file_collection_obj.filehash,
"filepath1") in file_names
result = self.snapshot_controller.get_files(snapshot_obj.id, mode="a")
assert len(result) == 1
for item in result:
assert isinstance(item, TextIOWrapper)
assert item.mode == "a"
assert os.path.join(self.task_controller.home, ".datmo", "collections",
file_collection_obj.filehash,
"filepath1") in file_names
def test_delete(self):
self.__setup()
# Create snapshot in the project
snapshot_obj = self.__default_create()
# Delete snapshot in the project
result = self.snapshot_controller.delete(snapshot_obj.id)
# Check if snapshot retrieval throws error
thrown = False
try:
self.snapshot_controller.dal.snapshot.get_by_id(snapshot_obj.id)
except EntityNotFound:
thrown = True
assert result == True and \
thrown == True
class TestSnapshotController():
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.project = ProjectController(self.temp_dir)
self.project.init("test", "test description")
self.task = TaskController(self.temp_dir)
self.snapshot = SnapshotController(self.temp_dir)
def teardown_method(self):
pass
def test_create_fail_no_message(self):
# Test no message
failed = False
try:
self.snapshot.create({})
except RequiredArgumentMissing:
failed = True
assert failed
def test_create_fail_no_code(self):
# Test default values for snapshot, fail due to code
failed = False
try:
self.snapshot.create({"message": "my test snapshot"})
except GitCommitDoesNotExist:
failed = True
assert failed
def test_create_fail_no_environment_with_language(self):
# Test default values for snapshot, fail due to environment with other than default
self.snapshot.file_driver.create("filepath1")
failed = False
try:
self.snapshot.create({
"message": "my test snapshot",
"language": "java"
})
except EnvironmentDoesNotExist:
failed = True
assert failed
def test_create_fail_no_environment_detected_in_file(self):
# Test default values for snapshot, fail due to no environment from file
self.snapshot.file_driver.create("filepath1")
failed = False
try:
self.snapshot.create({
"message": "my test snapshot",
})
except EnvironmentDoesNotExist:
failed = True
assert failed
def test_create_success_default_detected_in_file(self):
# Test default values for snapshot when there is no environment
test_filepath = os.path.join(self.snapshot.home, "script.py")
with open(test_filepath, "w") as f:
f.write(to_unicode("import numpy\n"))
f.write(to_unicode("import sklearn\n"))
f.write(to_unicode("print('hello')\n"))
snapshot_obj_1 = self.snapshot.create({"message": "my test snapshot"})
assert snapshot_obj_1
assert snapshot_obj_1.code_id
assert snapshot_obj_1.environment_id
assert snapshot_obj_1.file_collection_id
assert snapshot_obj_1.config == {}
assert snapshot_obj_1.stats == {}
def test_create_success_default_env_def(self):
# Create environment definition
env_def_path = os.path.join(self.snapshot.home, "Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
# Test default values for snapshot, success
snapshot_obj = self.snapshot.create({"message": "my test snapshot"})
assert snapshot_obj
assert snapshot_obj.code_id
assert snapshot_obj.environment_id
assert snapshot_obj.file_collection_id
assert snapshot_obj.config == {}
assert snapshot_obj.stats == {}
def test_create_success_default_env_def_duplicate(self):
# Test 2 snapshots with same parameters
# Create environment definition
env_def_path = os.path.join(self.snapshot.home, "Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
snapshot_obj = self.snapshot.create({"message": "my test snapshot"})
snapshot_obj_1 = self.snapshot.create({"message": "my test snapshot"})
# Should return the same object back
assert snapshot_obj_1 == snapshot_obj
assert snapshot_obj_1.code_id == snapshot_obj.code_id
assert snapshot_obj_1.environment_id == \
snapshot_obj.environment_id
assert snapshot_obj_1.file_collection_id == \
snapshot_obj.file_collection_id
assert snapshot_obj_1.config == \
snapshot_obj.config
assert snapshot_obj_1.stats == \
snapshot_obj.stats
def test_create_success_given_files_env_def_config_file_stats_file(self):
# Create environment definition
env_def_path = os.path.join(self.snapshot.home, "Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
snapshot_obj = self.snapshot.create({"message": "my test snapshot"})
# Create files to add
self.snapshot.file_driver.create("dirpath1", directory=True)
self.snapshot.file_driver.create("dirpath2", directory=True)
self.snapshot.file_driver.create("filepath1")
# Create config
config_filepath = os.path.join(self.snapshot.home, "config.json")
with open(config_filepath, "w") as f:
f.write(to_unicode(str('{"foo":"bar"}')))
# Create stats
stats_filepath = os.path.join(self.snapshot.home, "stats.json")
with open(stats_filepath, "w") as f:
f.write(to_unicode(str('{"foo":"bar"}')))
input_dict = {
"message":
"my test snapshot",
"filepaths": [
os.path.join(self.snapshot.home, "dirpath1"),
os.path.join(self.snapshot.home, "dirpath2"),
os.path.join(self.snapshot.home, "filepath1")
],
"environment_definition_filepath":
env_def_path,
"config_filepath":
config_filepath,
"stats_filepath":
stats_filepath,
}
# Create snapshot in the project
snapshot_obj_4 = self.snapshot.create(input_dict)
assert snapshot_obj_4 != snapshot_obj
assert snapshot_obj_4.code_id != snapshot_obj.code_id
assert snapshot_obj_4.environment_id == \
snapshot_obj.environment_id
assert snapshot_obj_4.file_collection_id != \
snapshot_obj.file_collection_id
assert snapshot_obj_4.config == {"foo": "bar"}
assert snapshot_obj_4.stats == {"foo": "bar"}
def test_create_success_given_files_env_def_different_config_stats(self):
# Create environment definition
env_def_path = os.path.join(self.snapshot.home, "Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
snapshot_obj = self.snapshot.create({"message": "my test snapshot"})
# Create files to add
self.snapshot.file_driver.create("dirpath1", directory=True)
self.snapshot.file_driver.create("dirpath2", directory=True)
self.snapshot.file_driver.create("filepath1")
# Create config
config_filepath = os.path.join(self.snapshot.home, "config.json")
with open(config_filepath, "w") as f:
f.write(to_unicode(str('{"foo":"bar"}')))
# Create stats
stats_filepath = os.path.join(self.snapshot.home, "stats.json")
with open(stats_filepath, "w") as f:
f.write(to_unicode(str('{"foo":"bar"}')))
# Test different config and stats inputs
input_dict = {
"message":
"my test snapshot",
"filepaths": [
os.path.join(self.snapshot.home, "dirpath1"),
os.path.join(self.snapshot.home, "dirpath2"),
os.path.join(self.snapshot.home, "filepath1")
],
"environment_definition_filepath":
env_def_path,
"config_filename":
"different_name",
"stats_filename":
"different_name",
}
# Create snapshot in the project
snapshot_obj_1 = self.snapshot.create(input_dict)
assert snapshot_obj_1 != snapshot_obj
assert snapshot_obj_1.config == {}
assert snapshot_obj_1.stats == {}
def test_create_success_given_files_env_def_direct_config_stats(self):
# Create environment definition
env_def_path = os.path.join(self.snapshot.home, "Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
# Create files to add
self.snapshot.file_driver.create("dirpath1", directory=True)
self.snapshot.file_driver.create("dirpath2", directory=True)
self.snapshot.file_driver.create("filepath1")
# Test different config and stats inputs
input_dict = {
"message":
"my test snapshot",
"filepaths": [
os.path.join(self.snapshot.home, "dirpath1"),
os.path.join(self.snapshot.home, "dirpath2"),
os.path.join(self.snapshot.home, "filepath1")
],
"environment_definition_filepath":
env_def_path,
"config": {
"foo": "bar"
},
"stats": {
"foo": "bar"
},
}
# Create snapshot in the project
snapshot_obj_6 = self.snapshot.create(input_dict)
assert snapshot_obj_6.config == {"foo": "bar"}
assert snapshot_obj_6.stats == {"foo": "bar"}
def test_create_from_task(self):
# 1) Test if fails with TaskNotComplete error
# 2) Test if success with task files, results, and message
# Setup task
task_command = ["sh", "-c", "echo accuracy:0.45"]
input_dict = {"command": task_command}
# Create task in the project
task_obj = self.task.create(input_dict)
# 1) Test option 1
failed = False
try:
_ = self.snapshot.create_from_task(message="my test snapshot",
task_id=task_obj.id)
except TaskNotComplete:
failed = True
assert failed
# Create environment definition
env_def_path = os.path.join(self.project.home, "Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
# Test the default values
updated_task_obj = self.task.run(task_obj.id)
# 2) Test option 2
snapshot_obj = self.snapshot.create_from_task(
message="my test snapshot", task_id=updated_task_obj.id)
assert snapshot_obj.id == updated_task_obj.after_snapshot_id
assert snapshot_obj.message == "my test snapshot"
assert snapshot_obj.stats == updated_task_obj.results
assert snapshot_obj.visible == True
def __default_create(self):
# Create files to add
self.snapshot.file_driver.create("dirpath1", directory=True)
self.snapshot.file_driver.create("dirpath2", directory=True)
self.snapshot.file_driver.create("filepath1")
# Create environment_driver definition
env_def_path = os.path.join(self.snapshot.home, "Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
# Create config
config_filepath = os.path.join(self.snapshot.home, "config.json")
with open(config_filepath, "w") as f:
f.write(to_unicode(str("{}")))
# Create stats
stats_filepath = os.path.join(self.snapshot.home, "stats.json")
with open(stats_filepath, "w") as f:
f.write(to_unicode(str("{}")))
input_dict = {
"message":
"my test snapshot",
"filepaths": [
os.path.join(self.snapshot.home, "dirpath1"),
os.path.join(self.snapshot.home, "dirpath2"),
os.path.join(self.snapshot.home, "filepath1")
],
"environment_definition_filepath":
env_def_path,
"config_filename":
config_filepath,
"stats_filename":
stats_filepath,
}
# Create snapshot in the project
return self.snapshot.create(input_dict)
def test_checkout(self):
# Create snapshot
snapshot_obj_1 = self.__default_create()
code_obj_1 = self.snapshot.dal.code.get_by_id(snapshot_obj_1.code_id)
# Create duplicate snapshot in project
self.snapshot.file_driver.create("test")
snapshot_obj_2 = self.__default_create()
assert snapshot_obj_2 != snapshot_obj_1
# Checkout to snapshot 1 using snapshot id
result = self.snapshot.checkout(snapshot_obj_1.id)
# Snapshot directory in user directory
snapshot_obj_1_path = os.path.join(self.snapshot.home,
"datmo_snapshots",
snapshot_obj_1.id)
assert result == True and \
self.snapshot.code_driver.latest_commit() == code_obj_1.commit_id and \
os.path.isdir(snapshot_obj_1_path)
def test_list(self):
# Check for error if incorrect session given
failed = False
try:
self.snapshot.list(session_id="does_not_exist")
except SessionDoesNotExistException:
failed = True
assert failed
# Create file to add to snapshot
test_filepath_1 = os.path.join(self.snapshot.home, "test.txt")
with open(test_filepath_1, "w") as f:
f.write(to_unicode(str("test")))
# Create snapshot in the project
snapshot_obj_1 = self.__default_create()
# Create file to add to second snapshot
test_filepath_2 = os.path.join(self.snapshot.home, "test2.txt")
with open(test_filepath_2, "w") as f:
f.write(to_unicode(str("test2")))
# Create second snapshot in the project
snapshot_obj_2 = self.__default_create()
# List all snapshots and ensure they exist
result = self.snapshot.list()
assert len(result) == 2 and \
snapshot_obj_1 in result and \
snapshot_obj_2 in result
# List all snapshots with session filter
result = self.snapshot.list(session_id=self.project.current_session.id)
assert len(result) == 2 and \
snapshot_obj_1 in result and \
snapshot_obj_2 in result
# List snapshots with visible filter
result = self.snapshot.list(visible=False)
assert len(result) == 0
result = self.snapshot.list(visible=True)
assert len(result) == 2 and \
snapshot_obj_1 in result and \
snapshot_obj_2 in result
def test_delete(self):
# Create snapshot in the project
snapshot_obj = self.__default_create()
# Delete snapshot in the project
result = self.snapshot.delete(snapshot_obj.id)
# Check if snapshot retrieval throws error
thrown = False
try:
self.snapshot.dal.snapshot.get_by_id(snapshot_obj.id)
except EntityNotFound:
thrown = True
assert result == True and \
thrown == True
def __setup(self):
Config().set_home(self.temp_dir)
self.project_controller = ProjectController()
self.project_controller.init("test", "test description")
self.session_controller = SessionController()
class TestTaskController():
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system() == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.project = ProjectController(self.temp_dir)
self.project.init("test", "test description")
self.environment = EnvironmentController(self.temp_dir)
self.task = TaskController(self.temp_dir)
def teardown_method(self):
pass
def test_create(self):
task_command = ["sh", "-c", "echo accuracy:0.45"]
task_gpu = False
input_dict = {
"command": task_command,
"gpu": task_gpu
}
# Create task in the project
task_obj = self.task.create(input_dict)
assert task_obj
assert task_obj.command == task_command
assert task_obj.gpu == task_gpu
def test_run_helper(self):
# TODO: Try out more options (see below)
# Create environment_driver id
env_def_path = os.path.join(self.project.home,
"Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
environment_obj = self.environment.create({
"definition_filepath": env_def_path
})
# Set log filepath
log_filepath = os.path.join(self.task.home,
"test.log")
# create volume to mount
temp_test_dirpath = os.path.join(self.temp_dir, "temp")
os.makedirs(temp_test_dirpath)
# Test option set 1
random_name = ''.join([random.choice(string.ascii_letters + string.digits)
for _ in range(32)])
options_dict = {
"command": ["sh", "-c", "echo accuracy:0.45"],
"ports": ["8888:8888"],
"gpu": False,
"name": random_name,
"volumes": {
temp_test_dirpath: {
'bind': '/task/',
'mode': 'rw'
}
},
"detach": False,
"stdin_open": True,
"tty": False,
"api": False
}
return_code, run_id, logs = \
self.task._run_helper(environment_obj.id,
options_dict, log_filepath)
assert return_code == 0
assert run_id and \
self.task.environment_driver.get_container(run_id)
assert logs and \
os.path.exists(log_filepath)
self.task.environment_driver.stop_remove_containers_by_term(term=random_name)
# Test option set 2
random_name_2 = ''.join([random.choice(string.ascii_letters + string.digits)
for _ in range(32)])
options_dict = {
"command": ["sh", "-c", "echo accuracy:0.45"],
"ports": ["8888:8888"],
"gpu": False,
"name": random_name_2 ,
"volumes": {
temp_test_dirpath: {
'bind': '/task/',
'mode': 'rw'
}
},
"detach": False,
"stdin_open": True,
"tty": False,
"api": True
}
return_code, run_id, logs = \
self.task._run_helper(environment_obj.id,
options_dict, log_filepath)
assert return_code == 0
assert run_id and \
self.task.environment_driver.get_container(run_id)
assert logs and \
os.path.exists(log_filepath)
self.task.environment_driver.stop_remove_containers_by_term(term=random_name_2)
def test_parse_logs_for_results(self):
test_logs = """
this is a log
accuracy is good
accuracy : 0.94
this did not work
validation : 0.32
model_type : logistic regression
"""
result = self.task._parse_logs_for_results(test_logs)
assert isinstance(result, dict)
assert result['accuracy'] == "0.94"
assert result['validation'] == "0.32"
assert result['model_type'] == "logistic regression"
def test_run(self):
# 1) Test success case with default values and env def file
# 2) Test failure case if running same task (conflicting containers)
# 3) Test failure case if running same task with snapshot_dict (conflicting containers)
# 4) Test success case with snapshot_dict
# 5) Test success case with saved file during task run
# TODO: look into log filepath randomness, sometimes logs are not written
task_command = ["sh", "-c", "echo accuracy:0.45"]
input_dict = {
"command": task_command
}
# Create task in the project
task_obj = self.task.create(input_dict)
# Create environment definition
env_def_path = os.path.join(self.project.home,
"Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
# 1) Test option 1
updated_task_obj = self.task.run(task_obj.id)
assert task_obj.id == updated_task_obj.id
assert updated_task_obj.before_snapshot_id
assert updated_task_obj.ports == None
assert updated_task_obj.gpu == False
assert updated_task_obj.interactive == False
assert updated_task_obj.task_dirpath
assert updated_task_obj.log_filepath
assert updated_task_obj.start_time
assert updated_task_obj.after_snapshot_id
assert updated_task_obj.run_id
assert updated_task_obj.logs
assert "accuracy" in updated_task_obj.logs
assert updated_task_obj.results
assert updated_task_obj.results == {"accuracy": "0.45"}
assert updated_task_obj.status == "SUCCESS"
assert updated_task_obj.end_time
assert updated_task_obj.duration
# 2) Test option 2
failed = False
try:
self.task.run(task_obj.id)
except TaskRunException:
failed = True
assert failed
# 3) Test option 3
# Create files to add
self.project.file_driver.create("dirpath1", directory=True)
self.project.file_driver.create("dirpath2", directory=True)
self.project.file_driver.create("filepath1")
# Snapshot dictionary
snapshot_dict = {
"filepaths": [os.path.join(self.project.home, "dirpath1"),
os.path.join(self.project.home, "dirpath2"),
os.path.join(self.project.home, "filepath1")],
}
# Run a basic task in the project
failed = False
try:
self.task.run(task_obj.id,
snapshot_dict=snapshot_dict)
except TaskRunException:
failed = True
assert failed
# Test when the specific task id is already RUNNING
# Create task in the project
task_obj_1 = self.task.create(input_dict)
self.task.dal.task.update({"id": task_obj_1.id, "status": "RUNNING"})
# Create environment_driver definition
env_def_path = os.path.join(self.project.home,
"Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
failed = False
try:
self.task.run(task_obj_1.id)
except TaskRunException:
failed = True
assert failed
# 4) Test option 4
# Create a new task in the project
task_obj_2 = self.task.create(input_dict)
# Run another task in the project
updated_task_obj_2 = self.task.run(task_obj_2.id,
snapshot_dict=snapshot_dict)
assert task_obj_2.id == updated_task_obj_2.id
assert updated_task_obj_2.before_snapshot_id
assert updated_task_obj_2.ports == None
assert updated_task_obj_2.gpu == False
assert updated_task_obj_2.interactive == False
assert updated_task_obj_2.task_dirpath
assert updated_task_obj_2.log_filepath
assert updated_task_obj_2.start_time
assert updated_task_obj_2.after_snapshot_id
assert updated_task_obj_2.run_id
assert updated_task_obj_2.logs
assert "accuracy" in updated_task_obj_2.logs
assert updated_task_obj_2.results
assert updated_task_obj_2.results == {"accuracy": "0.45"}
assert updated_task_obj_2.status == "SUCCESS"
assert updated_task_obj_2.end_time
assert updated_task_obj_2.duration
# 5) Test option 5
# Create a basic script
# (fails w/ no environment)
test_filepath = os.path.join(self.temp_dir, "script.py")
with open(test_filepath, "w") as f:
f.write(to_unicode("import os\n"))
f.write(to_unicode("import numpy\n"))
f.write(to_unicode("import sklearn\n"))
f.write(to_unicode("print('hello')\n"))
f.write(to_unicode("print(' accuracy: 0.56 ')\n"))
f.write(to_unicode("with open(os.path.join('/task', 'new_file.txt'), 'a') as f:\n"))
f.write(to_unicode(" f.write('my test file')\n"))
task_command = ["python", "script.py"]
input_dict = {
"command": task_command
}
# Create task in the project
task_obj_2 = self.task.create(input_dict)
# Create environment definition
env_def_path = os.path.join(self.project.home,
"Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
updated_task_obj_2 = self.task.run(task_obj_2.id)
assert updated_task_obj_2.before_snapshot_id
assert updated_task_obj_2.ports == None
assert updated_task_obj_2.gpu == False
assert updated_task_obj_2.interactive == False
assert updated_task_obj_2.task_dirpath
assert updated_task_obj_2.log_filepath
assert updated_task_obj_2.start_time
assert updated_task_obj_2.after_snapshot_id
assert updated_task_obj_2.run_id
assert updated_task_obj_2.logs
assert "accuracy" in updated_task_obj_2.logs
assert updated_task_obj_2.results
assert updated_task_obj_2.results == {"accuracy": "0.56"}
assert updated_task_obj_2.status == "SUCCESS"
assert updated_task_obj_2.end_time
assert updated_task_obj_2.duration
# test if after snapshot has the file written
after_snapshot_obj = self.task.dal.snapshot.get_by_id(
updated_task_obj_2.after_snapshot_id
)
file_collection_obj = self.task.dal.file_collection.get_by_id(
after_snapshot_obj.file_collection_id
)
files_absolute_path = os.path.join(self.task.home, file_collection_obj.path)
assert os.path.isfile(os.path.join(files_absolute_path, "task.log"))
assert os.path.isfile(os.path.join(files_absolute_path, "new_file.txt"))
def test_list(self):
task_command = ["sh", "-c", "echo accuracy:0.45"]
input_dict = {
"command": task_command
}
# Create tasks in the project
task_obj_1 = self.task.create(input_dict)
task_obj_2 = self.task.create(input_dict)
# List all tasks regardless of filters
result = self.task.list()
assert len(result) == 2 and \
task_obj_1 in result and \
task_obj_2 in result
# List all tasks and filter by session
result = self.task.list(session_id=
self.project.current_session.id)
assert len(result) == 2 and \
task_obj_1 in result and \
task_obj_2 in result
def test_get_files(self):
task_command = ["sh", "-c", "echo accuracy:0.45"]
input_dict = {
"command": task_command
}
# Create task in the project
task_obj = self.task.create(input_dict)
# Create environment definition
env_def_path = os.path.join(self.project.home,
"Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
# Create file to add
self.project.file_driver.create("dirpath1", directory=True)
self.project.file_driver.create(os.path.join("dirpath1", "filepath1"))
# Snapshot dictionary
snapshot_dict = {
"filepaths": [os.path.join(self.project.home, "dirpath1", "filepath1")],
}
# Test the default values
updated_task_obj = self.task.run(task_obj.id,
snapshot_dict=snapshot_dict)
# TODO: Test case for during run and before_snapshot run
# Get files for the task after run is complete (default)
result = self.task.get_files(updated_task_obj.id)
after_snapshot_obj = self.task.dal.snapshot.get_by_id(
updated_task_obj.after_snapshot_id
)
file_collection_obj = self.task.dal.file_collection.get_by_id(
after_snapshot_obj.file_collection_id
)
assert len(result) == 2
assert isinstance(result[0], TextIOWrapper)
assert result[0].name == os.path.join(self.task.home, ".datmo",
"collections",
file_collection_obj.filehash,
"task.log")
assert result[0].mode == "r"
assert isinstance(result[1], TextIOWrapper)
assert result[1].name == os.path.join(self.task.home, ".datmo",
"collections",
file_collection_obj.filehash,
"filepath1")
assert result[1].mode == "r"
# Get files for the task after run is complete for different mode
result = self.task.get_files(updated_task_obj.id, mode="a")
assert len(result) == 2
assert isinstance(result[0], TextIOWrapper)
assert result[0].name == os.path.join(self.task.home, ".datmo",
"collections",
file_collection_obj.filehash,
"task.log")
assert result[0].mode == "a"
assert isinstance(result[1], TextIOWrapper)
assert result[1].name == os.path.join(self.task.home, ".datmo",
"collections",
file_collection_obj.filehash,
"filepath1")
assert result[1].mode == "a"
def test_delete(self):
task_command = ["sh", "-c", "echo accuracy:0.45"]
input_dict = {
"command": task_command
}
# Create tasks in the project
task_obj = self.task.create(input_dict)
# Delete task from the project
result = self.task.delete(task_obj.id)
# Check if task retrieval throws error
thrown = False
try:
self.task.dal.snapshot.get_by_id(task_obj.id)
except EntityNotFound:
thrown = True
assert result == True and \
thrown == True
def test_stop(self):
task_command = ["sh", "-c", "echo accuracy:0.45"]
input_dict = {
"command": task_command
}
# Create task in the project
task_obj = self.task.create(input_dict)
# Create environment driver definition
env_def_path = os.path.join(self.project.home,
"Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
# Test the default values
updated_task_obj = self.task.run(task_obj.id)
# Stop the task
task_id = updated_task_obj.id
result = self.task.stop(task_id)
# Check if task stop throws error when wrong task id is given
thrown = False
try:
self.task.dal.snapshot.get_by_id(task_obj.id)
except EntityNotFound:
thrown = True
assert result == True and \
thrown == True
class TestSnapshotController():
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.project = ProjectController(self.temp_dir)
self.project.init("test", "test description")
self.snapshot = SnapshotController(self.temp_dir)
# Create environment_driver definition
self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(self.env_def_path, "w") as f:
f.write(str("FROM datmo/xgboost:cpu"))
# Create config
self.config_filepath = os.path.join(self.snapshot.home, "config.json")
with open(self.config_filepath, "w") as f:
f.write(str('{"foo":1}'))
# Create stats
self.stats_filepath = os.path.join(self.snapshot.home, "stats.json")
with open(self.stats_filepath, "w") as f:
f.write(str('{"bar":1}'))
# Create test file
self.filepath = os.path.join(self.snapshot.home, "file.txt")
with open(self.filepath, "w") as f:
f.write(str("test"))
def teardown_method(self):
pass
def test_code_setup_with_code_id(self):
val = 1
incoming_data = {"code_id": val}
final_data = {}
self.snapshot._code_setup(incoming_data, final_data)
assert final_data['code_id'] == val
def test_code_setup_with_commit_id(self):
val = "f38a8ace"
incoming_data = {"commit_id": val}
final_data = {}
self.snapshot._code_setup(incoming_data, final_data)
assert final_data['code_id']
def test_code_setup_with_none(self):
incoming_data = {}
final_data = {}
self.snapshot._code_setup(incoming_data, final_data)
assert final_data['code_id']
def test_env_setup_with_none(self):
incoming_data = {}
final_data = {}
self.snapshot._env_setup(incoming_data, final_data)
assert final_data['environment_id']
def test_file_setup_with_none(self):
incoming_data = {}
final_data = {}
self.snapshot._file_setup(incoming_data, final_data)
assert final_data['file_collection_id']
def test_file_setup_with_filepaths(self):
incoming_data = {"filepaths": [self.filepath]}
final_data = {}
self.snapshot._file_setup(incoming_data, final_data)
assert final_data['file_collection_id']
def test_config_setup_with_json(self):
incoming_data = {"config": {"foo": 1}}
final_data = {}
self.snapshot._config_setup(incoming_data, final_data)
assert final_data['config']["foo"] == 1
def test_config_setup_with_filepath(self):
incoming_data = {"config_filepath": self.config_filepath}
final_data = {}
self.snapshot._config_setup(incoming_data, final_data)
assert final_data['config']["foo"] == 1
def test_config_setup_with_filename(self):
incoming_data = {"config_filename": "config.json"}
final_data = {}
self.snapshot._file_setup(incoming_data, final_data)
self.snapshot._config_setup(incoming_data, final_data)
assert final_data['config']["foo"] == 1
def test_config_setup_with_empty_and_file(self):
incoming_data = {}
final_data = {}
self.snapshot._file_setup(incoming_data, final_data)
self.snapshot._config_setup(incoming_data, final_data)
assert final_data["config"]["foo"] == 1
def test_config_setup_with_empty_no_file(self):
os.remove(os.path.join(self.snapshot.home, "config.json"))
incoming_data = {}
final_data = {}
self.snapshot._file_setup(incoming_data, final_data)
self.snapshot._config_setup(incoming_data, final_data)
assert final_data["config"] == {}
def test_stats_setup_with_json(self):
incoming_data = {"stats": {"bar": 1}}
final_data = {}
self.snapshot._stats_setup(incoming_data, final_data)
assert final_data['stats']["bar"] == 1
def test_stats_setup_with_filepath(self):
incoming_data = {"stats_filepath": self.stats_filepath}
final_data = {}
self.snapshot._stats_setup(incoming_data, final_data)
assert final_data['stats']["bar"] == 1
def test_stats_setup_with_filename(self):
incoming_data = {"stats_filename": "stats.json"}
final_data = {}
self.snapshot._file_setup(incoming_data, final_data)
self.snapshot._stats_setup(incoming_data, final_data)
assert final_data['stats']["bar"] == 1
def test_stats_setup_with_empty_with_file(self):
incoming_data = {}
final_data = {}
self.snapshot._file_setup(incoming_data, final_data)
self.snapshot._stats_setup(incoming_data, final_data)
assert final_data['stats']["bar"] == 1
def test_stats_setup_with_empty_no_file(self):
os.remove(os.path.join(self.snapshot.home, "stats.json"))
incoming_data = {}
final_data = {}
self.snapshot._file_setup(incoming_data, final_data)
self.snapshot._stats_setup(incoming_data, final_data)
assert final_data["stats"] == {}
