class TestTaskController():
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system() == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.project = ProjectController(self.temp_dir)
self.project.init("test", "test description")
self.environment = EnvironmentController(self.temp_dir)
self.task = TaskController(self.temp_dir)
def teardown_method(self):
pass
def test_create(self):
task_command = ["sh", "-c", "echo accuracy:0.45"]
task_gpu = False
input_dict = {
"command": task_command,
"gpu": task_gpu
}
# Create task in the project
task_obj = self.task.create(input_dict)
assert task_obj
assert task_obj.command == task_command
assert task_obj.gpu == task_gpu
def test_run_helper(self):
# TODO: Try out more options (see below)
# Create environment_driver id
env_def_path = os.path.join(self.project.home,
"Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
environment_obj = self.environment.create({
"definition_filepath": env_def_path
})
# Set log filepath
log_filepath = os.path.join(self.task.home,
"test.log")
# create volume to mount
temp_test_dirpath = os.path.join(self.temp_dir, "temp")
os.makedirs(temp_test_dirpath)
# Test option set 1
random_name = ''.join([random.choice(string.ascii_letters + string.digits)
for _ in range(32)])
options_dict = {
"command": ["sh", "-c", "echo accuracy:0.45"],
"ports": ["8888:8888"],
"gpu": False,
"name": random_name,
"volumes": {
temp_test_dirpath: {
'bind': '/task/',
'mode': 'rw'
}
},
"detach": False,
"stdin_open": True,
"tty": False,
"api": False
}
return_code, run_id, logs = \
self.task._run_helper(environment_obj.id,
options_dict, log_filepath)
assert return_code == 0
assert run_id and \
self.task.environment_driver.get_container(run_id)
assert logs and \
os.path.exists(log_filepath)
self.task.environment_driver.stop_remove_containers_by_term(term=random_name)
# Test option set 2
random_name_2 = ''.join([random.choice(string.ascii_letters + string.digits)
for _ in range(32)])
options_dict = {
"command": ["sh", "-c", "echo accuracy:0.45"],
"ports": ["8888:8888"],
"gpu": False,
"name": random_name_2 ,
"volumes": {
temp_test_dirpath: {
'bind': '/task/',
'mode': 'rw'
}
},
"detach": False,
"stdin_open": True,
"tty": False,
"api": True
}
return_code, run_id, logs = \
self.task._run_helper(environment_obj.id,
options_dict, log_filepath)
assert return_code == 0
assert run_id and \
self.task.environment_driver.get_container(run_id)
assert logs and \
os.path.exists(log_filepath)
self.task.environment_driver.stop_remove_containers_by_term(term=random_name_2)
def test_parse_logs_for_results(self):
test_logs = """
this is a log
accuracy is good
accuracy : 0.94
this did not work
validation : 0.32
model_type : logistic regression
"""
result = self.task._parse_logs_for_results(test_logs)
assert isinstance(result, dict)
assert result['accuracy'] == "0.94"
assert result['validation'] == "0.32"
assert result['model_type'] == "logistic regression"
def test_run(self):
# 1) Test success case with default values and env def file
# 2) Test failure case if running same task (conflicting containers)
# 3) Test failure case if running same task with snapshot_dict (conflicting containers)
# 4) Test success case with snapshot_dict
# 5) Test success case with saved file during task run
# TODO: look into log filepath randomness, sometimes logs are not written
task_command = ["sh", "-c", "echo accuracy:0.45"]
input_dict = {
"command": task_command
}
# Create task in the project
task_obj = self.task.create(input_dict)
# Create environment definition
env_def_path = os.path.join(self.project.home,
"Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
# 1) Test option 1
updated_task_obj = self.task.run(task_obj.id)
assert task_obj.id == updated_task_obj.id
assert updated_task_obj.before_snapshot_id
assert updated_task_obj.ports == None
assert updated_task_obj.gpu == False
assert updated_task_obj.interactive == False
assert updated_task_obj.task_dirpath
assert updated_task_obj.log_filepath
assert updated_task_obj.start_time
assert updated_task_obj.after_snapshot_id
assert updated_task_obj.run_id
assert updated_task_obj.logs
assert "accuracy" in updated_task_obj.logs
assert updated_task_obj.results
assert updated_task_obj.results == {"accuracy": "0.45"}
assert updated_task_obj.status == "SUCCESS"
assert updated_task_obj.end_time
assert updated_task_obj.duration
# 2) Test option 2
failed = False
try:
self.task.run(task_obj.id)
except TaskRunException:
failed = True
assert failed
# 3) Test option 3
# Create files to add
self.project.file_driver.create("dirpath1", directory=True)
self.project.file_driver.create("dirpath2", directory=True)
self.project.file_driver.create("filepath1")
# Snapshot dictionary
snapshot_dict = {
"filepaths": [os.path.join(self.project.home, "dirpath1"),
os.path.join(self.project.home, "dirpath2"),
os.path.join(self.project.home, "filepath1")],
}
# Run a basic task in the project
failed = False
try:
self.task.run(task_obj.id,
snapshot_dict=snapshot_dict)
except TaskRunException:
failed = True
assert failed
# Test when the specific task id is already RUNNING
# Create task in the project
task_obj_1 = self.task.create(input_dict)
self.task.dal.task.update({"id": task_obj_1.id, "status": "RUNNING"})
# Create environment_driver definition
env_def_path = os.path.join(self.project.home,
"Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
failed = False
try:
self.task.run(task_obj_1.id)
except TaskRunException:
failed = True
assert failed
# 4) Test option 4
# Create a new task in the project
task_obj_2 = self.task.create(input_dict)
# Run another task in the project
updated_task_obj_2 = self.task.run(task_obj_2.id,
snapshot_dict=snapshot_dict)
assert task_obj_2.id == updated_task_obj_2.id
assert updated_task_obj_2.before_snapshot_id
assert updated_task_obj_2.ports == None
assert updated_task_obj_2.gpu == False
assert updated_task_obj_2.interactive == False
assert updated_task_obj_2.task_dirpath
assert updated_task_obj_2.log_filepath
assert updated_task_obj_2.start_time
assert updated_task_obj_2.after_snapshot_id
assert updated_task_obj_2.run_id
assert updated_task_obj_2.logs
assert "accuracy" in updated_task_obj_2.logs
assert updated_task_obj_2.results
assert updated_task_obj_2.results == {"accuracy": "0.45"}
assert updated_task_obj_2.status == "SUCCESS"
assert updated_task_obj_2.end_time
assert updated_task_obj_2.duration
# 5) Test option 5
# Create a basic script
# (fails w/ no environment)
test_filepath = os.path.join(self.temp_dir, "script.py")
with open(test_filepath, "w") as f:
f.write(to_unicode("import os\n"))
f.write(to_unicode("import numpy\n"))
f.write(to_unicode("import sklearn\n"))
f.write(to_unicode("print('hello')\n"))
f.write(to_unicode("print(' accuracy: 0.56 ')\n"))
f.write(to_unicode("with open(os.path.join('/task', 'new_file.txt'), 'a') as f:\n"))
f.write(to_unicode(" f.write('my test file')\n"))
task_command = ["python", "script.py"]
input_dict = {
"command": task_command
}
# Create task in the project
task_obj_2 = self.task.create(input_dict)
# Create environment definition
env_def_path = os.path.join(self.project.home,
"Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
updated_task_obj_2 = self.task.run(task_obj_2.id)
assert updated_task_obj_2.before_snapshot_id
assert updated_task_obj_2.ports == None
assert updated_task_obj_2.gpu == False
assert updated_task_obj_2.interactive == False
assert updated_task_obj_2.task_dirpath
assert updated_task_obj_2.log_filepath
assert updated_task_obj_2.start_time
assert updated_task_obj_2.after_snapshot_id
assert updated_task_obj_2.run_id
assert updated_task_obj_2.logs
assert "accuracy" in updated_task_obj_2.logs
assert updated_task_obj_2.results
assert updated_task_obj_2.results == {"accuracy": "0.56"}
assert updated_task_obj_2.status == "SUCCESS"
assert updated_task_obj_2.end_time
assert updated_task_obj_2.duration
# test if after snapshot has the file written
after_snapshot_obj = self.task.dal.snapshot.get_by_id(
updated_task_obj_2.after_snapshot_id
)
file_collection_obj = self.task.dal.file_collection.get_by_id(
after_snapshot_obj.file_collection_id
)
files_absolute_path = os.path.join(self.task.home, file_collection_obj.path)
assert os.path.isfile(os.path.join(files_absolute_path, "task.log"))
assert os.path.isfile(os.path.join(files_absolute_path, "new_file.txt"))
def test_list(self):
task_command = ["sh", "-c", "echo accuracy:0.45"]
input_dict = {
"command": task_command
}
# Create tasks in the project
task_obj_1 = self.task.create(input_dict)
task_obj_2 = self.task.create(input_dict)
# List all tasks regardless of filters
result = self.task.list()
assert len(result) == 2 and \
task_obj_1 in result and \
task_obj_2 in result
# List all tasks and filter by session
result = self.task.list(session_id=
self.project.current_session.id)
assert len(result) == 2 and \
task_obj_1 in result and \
task_obj_2 in result
def test_get_files(self):
task_command = ["sh", "-c", "echo accuracy:0.45"]
input_dict = {
"command": task_command
}
# Create task in the project
task_obj = self.task.create(input_dict)
# Create environment definition
env_def_path = os.path.join(self.project.home,
"Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
# Create file to add
self.project.file_driver.create("dirpath1", directory=True)
self.project.file_driver.create(os.path.join("dirpath1", "filepath1"))
# Snapshot dictionary
snapshot_dict = {
"filepaths": [os.path.join(self.project.home, "dirpath1", "filepath1")],
}
# Test the default values
updated_task_obj = self.task.run(task_obj.id,
snapshot_dict=snapshot_dict)
# TODO: Test case for during run and before_snapshot run
# Get files for the task after run is complete (default)
result = self.task.get_files(updated_task_obj.id)
after_snapshot_obj = self.task.dal.snapshot.get_by_id(
updated_task_obj.after_snapshot_id
)
file_collection_obj = self.task.dal.file_collection.get_by_id(
after_snapshot_obj.file_collection_id
)
assert len(result) == 2
assert isinstance(result[0], TextIOWrapper)
assert result[0].name == os.path.join(self.task.home, ".datmo",
"collections",
file_collection_obj.filehash,
"task.log")
assert result[0].mode == "r"
assert isinstance(result[1], TextIOWrapper)
assert result[1].name == os.path.join(self.task.home, ".datmo",
"collections",
file_collection_obj.filehash,
"filepath1")
assert result[1].mode == "r"
# Get files for the task after run is complete for different mode
result = self.task.get_files(updated_task_obj.id, mode="a")
assert len(result) == 2
assert isinstance(result[0], TextIOWrapper)
assert result[0].name == os.path.join(self.task.home, ".datmo",
"collections",
file_collection_obj.filehash,
"task.log")
assert result[0].mode == "a"
assert isinstance(result[1], TextIOWrapper)
assert result[1].name == os.path.join(self.task.home, ".datmo",
"collections",
file_collection_obj.filehash,
"filepath1")
assert result[1].mode == "a"
def test_delete(self):
task_command = ["sh", "-c", "echo accuracy:0.45"]
input_dict = {
"command": task_command
}
# Create tasks in the project
task_obj = self.task.create(input_dict)
# Delete task from the project
result = self.task.delete(task_obj.id)
# Check if task retrieval throws error
thrown = False
try:
self.task.dal.snapshot.get_by_id(task_obj.id)
except EntityNotFound:
thrown = True
assert result == True and \
thrown == True
def test_stop(self):
task_command = ["sh", "-c", "echo accuracy:0.45"]
input_dict = {
"command": task_command
}
# Create task in the project
task_obj = self.task.create(input_dict)
# Create environment driver definition
env_def_path = os.path.join(self.project.home,
"Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
# Test the default values
updated_task_obj = self.task.run(task_obj.id)
# Stop the task
task_id = updated_task_obj.id
result = self.task.stop(task_id)
# Check if task stop throws error when wrong task id is given
thrown = False
try:
self.task.dal.snapshot.get_by_id(task_obj.id)
except EntityNotFound:
thrown = True
assert result == True and \
thrown == True
class TestTaskController():
def setup_method(self):
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.environment_ids = []
def teardown_method(self):
if not check_docker_inactive(test_datmo_dir,
Config().datmo_directory_name):
self.__setup()
self.environment_controller = EnvironmentController()
for env_id in list(set(self.environment_ids)):
if not self.environment_controller.delete(env_id):
raise Exception
def __setup(self):
Config().set_home(self.temp_dir)
self.project_controller = ProjectController()
self.project_controller.init("test", "test description")
self.environment_controller = EnvironmentController()
self.task_controller = TaskController()
def test_init_fail_project_not_init(self):
Config().set_home(self.temp_dir)
failed = False
try:
TaskController()
except ProjectNotInitialized:
failed = True
assert failed
def test_init_fail_invalid_path(self):
test_home = "some_random_dir"
Config().set_home(test_home)
failed = False
try:
TaskController()
except InvalidProjectPath:
failed = True
assert failed
def test_create(self):
self.__setup()
# Create task in the project
task_obj = self.task_controller.create()
assert isinstance(task_obj, Task)
assert task_obj.created_at
assert task_obj.updated_at
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_run_helper(self):
self.__setup()
# TODO: Try out more options (see below)
# Create environment_driver id
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
paths = [env_def_path]
environment_obj = self.environment_controller.create({"paths": paths})
self.environment_ids.append(environment_obj.id)
# Set log filepath
log_filepath = os.path.join(self.task_controller.home, "test.log")
# create volume to mount
temp_test_dirpath = os.path.join(self.temp_dir, "temp")
os.makedirs(temp_test_dirpath)
# Test option set 1
random_name = str(uuid.uuid1())
options_dict = {
"command": ["sh", "-c", "echo accuracy:0.45"],
"ports": ["8888:8888"],
"name": random_name,
"volumes": {
temp_test_dirpath: {
'bind': '/task/',
'mode': 'rw'
}
},
"mem_limit": "4g",
"detach": False,
"stdin_open": False,
"tty": False,
"api": False,
"interactive": False
}
return_code, run_id, logs = \
self.task_controller._run_helper(environment_obj.id,
options_dict, log_filepath)
assert return_code == 0
assert run_id and \
self.task_controller.environment_driver.get_container(run_id)
assert logs and \
os.path.exists(log_filepath)
self.task_controller.environment_driver.stop_remove_containers_by_term(
term=random_name)
# Test option set 2
random_name_2 = str(uuid.uuid1())
options_dict = {
"command": ["sh", "-c", "echo accuracy:0.45"],
"ports": ["8888:8888"],
"name": random_name_2,
"volumes": {
temp_test_dirpath: {
'bind': '/task/',
'mode': 'rw'
}
},
"mem_limit": "4g",
"detach": True,
"stdin_open": False,
"tty": False,
"api": True,
"interactive": False
}
return_code, run_id, logs = \
self.task_controller._run_helper(environment_obj.id,
options_dict, log_filepath)
assert return_code == 0
assert run_id and \
self.task_controller.environment_driver.get_container(run_id)
assert logs and \
os.path.exists(log_filepath)
self.task_controller.environment_driver.stop_remove_containers_by_term(
term=random_name_2)
def test_parse_logs_for_results(self):
self.__setup()
test_logs = """
this is a log
accuracy is good
accuracy : 0.94
this did not work
validation : 0.32
model_type : logistic regression
"""
result = self.task_controller._parse_logs_for_results(test_logs)
assert isinstance(result, dict)
assert result['accuracy'] == "0.94"
assert result['validation'] == "0.32"
assert result['model_type'] == "logistic regression"
test_logs = """test"""
result = self.task_controller._parse_logs_for_results(test_logs)
assert result is None
def test_update_environment_run_options(self):
self.__setup()
environment_run_option = {
"command": ["python", "script.py"],
"volumes": {
os.path.join(self.temp_dir, "/temp_task"): {
'bind': '/task/',
'mode': 'rw'
},
self.temp_dir: {
'bind': '/home/',
'mode': 'rw'
}
}
}
# create data file
data_dirpath = os.path.join(self.temp_dir, "data")
data_file_dirpath = os.path.join(self.temp_dir, "data_folder")
data_filepath = os.path.join(data_file_dirpath, "data_file.txt")
os.mkdir(data_dirpath)
os.mkdir(data_file_dirpath)
with open(data_filepath, "wb") as f:
f.write(to_bytes("data file"))
data_file_path_map = [(data_filepath, "data_file.txt")]
data_directory_path_map = [(data_dirpath, "data_directory")]
environment_run_option = self.task_controller._update_environment_run_options(
environment_run_option, data_file_path_map,
data_directory_path_map)
assert environment_run_option["volumes"][data_file_dirpath] == {
'bind': '/data/',
'mode': 'rw'
}
assert environment_run_option["volumes"][data_dirpath] == {
'bind': '/data/data_directory',
'mode': 'rw'
}
# Error by passing directory which does not exist
data_dirpath = os.path.join(self.temp_dir, "data_dne")
data_filepath = os.path.join(self.temp_dir, "data_dne",
"data_file.txt")
data_file_path_map = [(data_filepath, "data_file.txt")]
data_directory_path_map = [(data_dirpath, "data_directory")]
failed = False
try:
self.task_controller._update_environment_run_options(
environment_run_option, data_file_path_map,
data_directory_path_map)
except TaskRunError:
failed = True
assert failed
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_run(self):
self.__setup()
# 0) Test failure case without command and without interactive
# 1) Test success case with default values and env def file
# 2) Test failure case if running same task (conflicting containers)
# 3) Test failure case if running same task with snapshot_dict (conflicting containers)
# 4) Test success case with snapshot_dict
# 5) Test success case with saved file during task run
# 6) Test success case with data file path being passed
# 7) Test success case with data directory path being passed
# TODO: look into log filepath randomness, sometimes logs are not written
# Create task in the project
task_obj = self.task_controller.create()
# Create environment definition
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# 0) Test option 0
failed = False
try:
self.task_controller.run(task_obj.id)
except RequiredArgumentMissing:
failed = True
assert failed
failed = False
try:
self.task_controller.run(
task_obj.id,
task_dict={
"command": None,
"interactive": False,
"ports": None
})
except RequiredArgumentMissing:
failed = True
assert failed
# Create task_dict
task_command = ["sh", "-c", "echo accuracy:0.45"]
task_dict = {"command_list": task_command}
# 1) Test option 1
updated_task_obj = self.task_controller.run(
task_obj.id, task_dict=task_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
assert isinstance(updated_task_obj, Task)
assert task_obj.id == updated_task_obj.id
assert updated_task_obj.before_snapshot_id
assert updated_task_obj.ports == None
assert updated_task_obj.interactive == False
assert updated_task_obj.task_dirpath
assert updated_task_obj.log_filepath
assert updated_task_obj.start_time
assert updated_task_obj.after_snapshot_id
assert updated_task_obj.run_id
assert updated_task_obj.logs
assert "accuracy" in updated_task_obj.logs
assert updated_task_obj.results
assert updated_task_obj.results == {"accuracy": "0.45"}
assert after_snapshot_obj.stats == {"accuracy": "0.45"}
assert updated_task_obj.status == "SUCCESS"
assert updated_task_obj.end_time
assert updated_task_obj.duration
self.task_controller.stop(task_obj.id)
# 2) Test option 2
failed = False
try:
self.task_controller.run(task_obj.id)
except TaskRunError:
failed = True
assert failed
# 3) Test option 3
# Create files to add
self.project_controller.file_driver.create("dirpath1", directory=True)
self.project_controller.file_driver.create("dirpath2", directory=True)
self.project_controller.file_driver.create("filepath1")
# Snapshot dictionary
snapshot_dict = {
"paths": [
os.path.join(self.project_controller.home, "dirpath1"),
os.path.join(self.project_controller.home, "dirpath2"),
os.path.join(self.project_controller.home, "filepath1")
],
}
# Run a basic task in the project
failed = False
try:
self.task_controller.run(task_obj.id, snapshot_dict=snapshot_dict)
except TaskRunError:
failed = True
assert failed
# Test when the specific task id is already RUNNING
# Create task in the project
task_obj_1 = self.task_controller.create()
self.task_controller.dal.task.update({
"id": task_obj_1.id,
"status": "RUNNING"
})
# Create environment_driver definition
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
failed = False
try:
self.task_controller.run(task_obj_1.id, task_dict=task_dict)
except TaskRunError:
failed = True
assert failed
# 4) Test option 4
# Create a new task in the project
task_obj_2 = self.task_controller.create()
# Run another task in the project
updated_task_obj_2 = self.task_controller.run(
task_obj_2.id, task_dict=task_dict, snapshot_dict=snapshot_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj_2.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
assert isinstance(updated_task_obj_2, Task)
assert task_obj_2.id == updated_task_obj_2.id
assert updated_task_obj_2.before_snapshot_id
assert updated_task_obj_2.ports == None
assert updated_task_obj_2.interactive == False
assert updated_task_obj_2.task_dirpath
assert updated_task_obj_2.log_filepath
assert updated_task_obj_2.start_time
assert updated_task_obj_2.after_snapshot_id
assert updated_task_obj_2.run_id
assert updated_task_obj_2.logs
assert "accuracy" in updated_task_obj_2.logs
assert updated_task_obj_2.results
assert updated_task_obj_2.results == {"accuracy": "0.45"}
assert updated_task_obj_2.status == "SUCCESS"
assert updated_task_obj_2.end_time
assert updated_task_obj_2.duration
self.task_controller.stop(task_obj_2.id)
# 5) Test option 5
# Create a basic script
test_filepath = os.path.join(self.temp_dir, "script.py")
with open(test_filepath, "wb") as f:
f.write(to_bytes("import os\n"))
f.write(to_bytes("import shutil\n"))
f.write(to_bytes("print('hello')\n"))
f.write(to_bytes("print(' accuracy: 0.56 ')\n"))
f.write(
to_bytes(
"with open(os.path.join('/task', 'new_file.txt'), 'a') as f:\n"
))
f.write(to_bytes(" f.write('my test file')\n"))
# Create task in the project
task_obj_2 = self.task_controller.create()
# Create task_dict
task_command = ["python", "script.py"]
task_dict = {"command_list": task_command}
# Create environment definition
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
updated_task_obj_2 = self.task_controller.run(
task_obj_2.id, task_dict=task_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj_2.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
assert isinstance(updated_task_obj_2, Task)
assert updated_task_obj_2.before_snapshot_id
assert updated_task_obj_2.ports == None
assert updated_task_obj_2.interactive == False
assert updated_task_obj_2.task_dirpath
assert updated_task_obj_2.log_filepath
assert updated_task_obj_2.start_time
assert updated_task_obj_2.after_snapshot_id
assert updated_task_obj_2.run_id
assert updated_task_obj_2.logs
assert "accuracy" in updated_task_obj_2.logs
assert updated_task_obj_2.results
assert updated_task_obj_2.results == {"accuracy": "0.56"}
assert updated_task_obj_2.status == "SUCCESS"
assert updated_task_obj_2.end_time
assert updated_task_obj_2.duration
self.task_controller.stop(task_obj_2.id)
# test if after snapshot has the file written
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj_2.after_snapshot_id)
file_collection_obj = self.task_controller.dal.file_collection.get_by_id(
after_snapshot_obj.file_collection_id)
files_absolute_path = os.path.join(self.task_controller.home,
file_collection_obj.path)
assert os.path.isfile(os.path.join(files_absolute_path, "task.log"))
assert os.path.isfile(
os.path.join(files_absolute_path, "new_file.txt"))
# 6) Test Option 6
self.project_controller.file_driver.create("dirpath1", directory=True)
self.project_controller.file_driver.create(
os.path.join("dirpath1", "file.txt"))
with open(
os.path.join(self.project_controller.home, "dirpath1",
"file.txt"), "wb") as f:
f.write(to_bytes('my initial line\n'))
test_filename = "script.py"
test_filepath = os.path.join(self.temp_dir, test_filename)
with open(test_filepath, "wb") as f:
f.write(to_bytes("import os\n"))
f.write(to_bytes("print('hello')\n"))
f.write(to_bytes("import shutil\n"))
f.write(
to_bytes(
"with open(os.path.join('/data', 'file.txt'), 'a') as f:\n"
))
f.write(to_bytes(" f.write('my test file')\n"))
# Create task in the project
task_obj_3 = self.task_controller.create()
# Create task_dict
task_command = ["python", test_filename]
task_dict = {
"command_list":
task_command,
"data_file_path_map": [(os.path.join(self.project_controller.home,
"dirpath1", "file.txt"),
'file.txt')]
}
# Create environment definition
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
updated_task_obj_3 = self.task_controller.run(
task_obj_3.id, task_dict=task_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj_3.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
assert isinstance(updated_task_obj_3, Task)
assert updated_task_obj_3.before_snapshot_id
assert updated_task_obj_3.ports == None
assert updated_task_obj_3.interactive == False
assert updated_task_obj_3.task_dirpath
assert updated_task_obj_3.log_filepath
assert updated_task_obj_3.start_time
assert updated_task_obj_3.after_snapshot_id
assert updated_task_obj_3.run_id
assert updated_task_obj_3.logs
assert updated_task_obj_3.status == "SUCCESS"
assert updated_task_obj_3.end_time
assert updated_task_obj_3.duration
self.task_controller.stop(task_obj_3.id)
# test if after snapshot has the file written
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj_3.after_snapshot_id)
file_collection_obj = self.task_controller.dal.file_collection.get_by_id(
after_snapshot_obj.file_collection_id)
files_absolute_path = os.path.join(self.task_controller.home,
file_collection_obj.path)
assert os.path.isfile(os.path.join(files_absolute_path, "task.log"))
assert os.path.isfile(
os.path.join(self.project_controller.home, "dirpath1", "file.txt"))
assert "my initial line" in open(
os.path.join(self.project_controller.home, "dirpath1", "file.txt"),
"r").read()
assert "my test file" in open(
os.path.join(self.project_controller.home, "dirpath1", "file.txt"),
"r").read()
# 7) Test Option 7
self.project_controller.file_driver.create("dirpath1", directory=True)
self.project_controller.file_driver.create(
os.path.join("dirpath1", "file.txt"))
with open(
os.path.join(self.project_controller.home, "dirpath1",
"file.txt"), "wb") as f:
f.write(to_bytes('my initial line\n'))
test_filename = "script.py"
test_filepath = os.path.join(self.temp_dir, test_filename)
with open(test_filepath, "wb") as f:
f.write(to_bytes("import os\n"))
f.write(to_bytes("print('hello')\n"))
f.write(to_bytes("import shutil\n"))
f.write(
to_bytes(
"with open(os.path.join('/data', 'dirpath1', 'file.txt'), 'a') as f:\n"
))
f.write(to_bytes(" f.write('my test file')\n"))
# Create task in the project
task_obj_4 = self.task_controller.create()
# Create task_dict
task_command = ["python", test_filename]
task_dict = {
"command_list":
task_command,
"data_directory_path_map": [(os.path.join(
self.project_controller.home, "dirpath1"), 'dirpath1')]
}
# Create environment definition
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
updated_task_obj_4 = self.task_controller.run(
task_obj_4.id, task_dict=task_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj_4.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
assert isinstance(updated_task_obj_4, Task)
assert updated_task_obj_4.before_snapshot_id
assert updated_task_obj_4.ports == None
assert updated_task_obj_4.interactive == False
assert updated_task_obj_4.task_dirpath
assert updated_task_obj_4.log_filepath
assert updated_task_obj_4.start_time
assert updated_task_obj_4.after_snapshot_id
assert updated_task_obj_4.run_id
assert updated_task_obj_4.logs
assert updated_task_obj_4.status == "SUCCESS"
assert updated_task_obj_4.end_time
assert updated_task_obj_4.duration
self.task_controller.stop(task_obj_4.id)
# test if after snapshot has the file written
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj_4.after_snapshot_id)
file_collection_obj = self.task_controller.dal.file_collection.get_by_id(
after_snapshot_obj.file_collection_id)
files_absolute_path = os.path.join(self.task_controller.home,
file_collection_obj.path)
assert os.path.isfile(os.path.join(files_absolute_path, "task.log"))
assert os.path.isfile(
os.path.join(self.project_controller.home, "dirpath1", "file.txt"))
assert "my initial line" in open(
os.path.join(self.project_controller.home, "dirpath1", "file.txt"),
"r").read()
assert "my test file" in open(
os.path.join(self.project_controller.home, "dirpath1", "file.txt"),
"r").read()
def test_list(self):
self.__setup()
# Create tasks in the project
task_obj_1 = self.task_controller.create()
task_obj_2 = self.task_controller.create()
# List all tasks regardless of filters
result = self.task_controller.list()
assert len(result) == 2 and \
task_obj_1 in result and \
task_obj_2 in result
# List all tasks regardless of filters in ascending
result = self.task_controller.list(
sort_key='created_at', sort_order='ascending')
assert len(result) == 2 and \
task_obj_1 in result and \
task_obj_2 in result
assert result[0].created_at <= result[-1].created_at
# List all tasks regardless of filters in descending
result = self.task_controller.list(
sort_key='created_at', sort_order='descending')
assert len(result) == 2 and \
task_obj_1 in result and \
task_obj_2 in result
assert result[0].created_at >= result[-1].created_at
# Wrong order being passed in
failed = False
try:
_ = self.task_controller.list(
sort_key='created_at', sort_order='wrong_order')
except InvalidArgumentType:
failed = True
assert failed
# Wrong key and order being passed in
failed = False
try:
_ = self.task_controller.list(
sort_key='wrong_key', sort_order='wrong_order')
except InvalidArgumentType:
failed = True
assert failed
# wrong key and right order being passed in
expected_result = self.task_controller.list(
sort_key='created_at', sort_order='ascending')
result = self.task_controller.list(
sort_key='wrong_key', sort_order='ascending')
expected_ids = [item.id for item in expected_result]
ids = [item.id for item in result]
assert set(expected_ids) == set(ids)
# List all tasks
result = self.task_controller.list()
assert len(result) == 2 and \
task_obj_1 in result and \
task_obj_2 in result
def test_get(self):
self.__setup()
# Test failure for no task
failed = False
try:
self.task_controller.get("random")
except DoesNotExist:
failed = True
assert failed
# Test success for task
task_obj = self.task_controller.create()
task_obj_returned = self.task_controller.get(task_obj.id)
assert task_obj == task_obj_returned
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_get_files(self):
self.__setup()
# Test failure case
failed = False
try:
self.task_controller.get_files("random")
except DoesNotExist:
failed = True
assert failed
# Create task in the project
task_obj = self.task_controller.create()
# Create environment definition
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# Create file to add
self.project_controller.file_driver.create("dirpath1", directory=True)
self.project_controller.file_driver.create(
os.path.join("dirpath1", "filepath1"))
# Snapshot dictionary
snapshot_dict = {
"paths": [
os.path.join(self.project_controller.home, "dirpath1",
"filepath1")
],
}
# Create task_dict
task_command = ["sh", "-c", "echo accuracy:0.45"]
task_dict = {"command_list": task_command}
# Test the default values
updated_task_obj = self.task_controller.run(
task_obj.id, task_dict=task_dict, snapshot_dict=snapshot_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
# TODO: Test case for during run and before_snapshot run
# Get files for the task after run is complete (default)
result = self.task_controller.get_files(updated_task_obj.id)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj.after_snapshot_id)
file_collection_obj = self.task_controller.dal.file_collection.get_by_id(
after_snapshot_obj.file_collection_id)
file_names = [item.name for item in result]
assert len(result) == 2
for item in result:
assert isinstance(item, TextIOWrapper)
assert item.mode == "r"
assert os.path.join(self.task_controller.home, ".datmo", "collections",
file_collection_obj.filehash,
"task.log") in file_names
assert os.path.join(self.task_controller.home, ".datmo", "collections",
file_collection_obj.filehash,
"filepath1") in file_names
# Get files for the task after run is complete for different mode
result = self.task_controller.get_files(updated_task_obj.id, mode="a")
assert len(result) == 2
for item in result:
assert isinstance(item, TextIOWrapper)
assert item.mode == "a"
assert os.path.join(self.task_controller.home, ".datmo", "collections",
file_collection_obj.filehash,
"task.log") in file_names
assert os.path.join(self.task_controller.home, ".datmo", "collections",
file_collection_obj.filehash,
"filepath1") in file_names
self.task_controller.stop(task_obj.id)
def test_update(self):
self.__setup()
# Create task in the project
task_obj = self.task_controller.create()
assert isinstance(task_obj, Task)
# Test 1: When no meta data is passed
updated_task_obj = self.task_controller.update(task_obj.id)
assert updated_task_obj.workspace is None
# Test 2: When meta data for workspace is passed
test_workspace = "notebook"
test_command = "python script.py"
updated_task_obj = self.task_controller.update(
task_obj.id, workspace=test_workspace, command=test_command)
assert updated_task_obj.workspace == test_workspace
assert updated_task_obj.command == test_command
assert updated_task_obj.command_list == ["python", "script.py"]
# Test 3: When meta data for workspace is passed
test_interactive = True
updated_task_obj = self.task_controller.update(
task_obj.id, interactive=test_interactive)
assert updated_task_obj.interactive == test_interactive
# Test 4: When meta data for workspace is passed
test_command_list = ["python", "script.py"]
updated_task_obj = self.task_controller.update(
task_obj.id, command_list=test_command_list)
assert updated_task_obj.command_list == ["python", "script.py"]
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_delete(self):
self.__setup()
# Create tasks in the project
task_obj = self.task_controller.create()
# Delete task from the project
result = self.task_controller.delete(task_obj.id)
# Check if task retrieval throws error
thrown = False
try:
self.task_controller.dal.snapshot.get_by_id(task_obj.id)
except EntityNotFound:
thrown = True
assert result == True and \
thrown == True
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_stop_failure(self):
self.__setup()
# 1) Test required arguments not provided
# 2) Test too many arguments found
# 3) Test incorrect task id given
# 1) Test option 1
failed = False
try:
self.task_controller.stop()
except RequiredArgumentMissing:
failed = True
assert failed
# 2) Test option 2
failed = False
try:
self.task_controller.stop(task_id="test_task_id", all=True)
except TooManyArgumentsFound:
failed = True
assert failed
# 3) Test option 3
thrown = False
try:
self.task_controller.stop(task_id="incorrect_task_id")
except DoesNotExist:
thrown = True
assert thrown
@pytest_docker_environment_failed_instantiation(test_datmo_dir)
def test_stop_success(self):
self.__setup()
# 1) Test stop with task_id
# 2) Test stop with all given
# Create task in the project
task_obj = self.task_controller.create()
# Create environment driver definition
env_def_path = os.path.join(self.project_controller.home, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# Create task_dict
task_command = ["sh", "-c", "echo accuracy:0.45"]
task_dict = {"command_list": task_command}
# 1) Test option 1
updated_task_obj = self.task_controller.run(
task_obj.id, task_dict=task_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
task_id = updated_task_obj.id
result = self.task_controller.stop(task_id=task_id)
after_task_obj = self.task_controller.dal.task.get_by_id(task_id)
assert result
assert after_task_obj.status == "STOPPED"
# 2) Test option 2
task_obj_2 = self.task_controller.create()
updated_task_obj = self.task_controller.run(
task_obj_2.id, task_dict=task_dict)
after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
updated_task_obj.after_snapshot_id)
environment_obj = self.task_controller.dal.environment.get_by_id(
after_snapshot_obj.environment_id)
self.environment_ids.append(environment_obj.id)
result = self.task_controller.stop(all=True)
all_task_objs = self.task_controller.dal.task.query({})
assert result
for task_obj in all_task_objs:
assert task_obj.status == "STOPPED"