def test_run(self):
# 1) Run task with no commit or code available (cannot save states before), string command
# 2) Run task with simple python file, no environment definition, string command (auto generate env)
# 3) Run task with simple python file and environment definition, string command
# 4) Run task with simple python file and environment definition, list command
# 1) Test out option 1)
failed = False
try:
_ = run(command="test", home=self.temp_dir)
except GitCommitDoesNotExist:
failed = True
assert failed
# Create a basic task and run it with string command
# (fails w/ no environment)
test_filepath = os.path.join(self.temp_dir, "script.py")
with open(test_filepath, "w") as f:
f.write(to_unicode("import numpy\n"))
f.write(to_unicode("import sklearn\n"))
f.write(to_unicode("print('hello')\n"))
f.write(to_unicode("print(' accuracy: 0.56 ')\n"))
# 2) Test out option 2
task_obj_0 = run(command="python script.py", home=self.temp_dir)
assert isinstance(task_obj_0, Task)
assert task_obj_0.id
assert 'hello' in task_obj_0.logs
assert task_obj_0.results == {"accuracy": "0.56"}
# Add environment definition
test_filepath = os.path.join(self.temp_dir, "Dockerfile")
with open(test_filepath, "w") as f:
f.write(to_unicode("FROM datmo/xgboost:cpu"))
# 3) Test out option 3
task_obj_1 = run(command="python script.py",
env=test_filepath,
home=self.temp_dir)
assert isinstance(task_obj_1, Task)
assert task_obj_1.id
assert 'hello' in task_obj_1.logs
assert task_obj_1.results == {"accuracy": "0.56"}
# 4) Test out option 4
task_obj_2 = run(command=["python", "script.py"],
env=test_filepath,
home=self.temp_dir)
assert isinstance(task_obj_2, Task)
assert task_obj_2.id
assert 'hello' in task_obj_2.logs
assert task_obj_2.results == {"accuracy": "0.56"}
def test_create_from_task_fail_user_inputs(self):
# Setup task
# Create environment definition
env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
task_obj = run("sh -c echo accuracy:0.45")
# Test if failure if user gives environment_id with task_id
failed = False
try:
_ = create(message="my test snapshot",
task_id=task_obj.id,
label="best",
config={"foo": "bar"},
stats={"foo": "bar"},
environment_id="test_id")
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
# Test if failure if user gives filepaths with task_id
failed = False
try:
_ = create(message="my test snapshot",
task_id=task_obj.id,
label="best",
config={"foo": "bar"},
stats={"foo": "bar"},
paths=["mypath"])
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
def __setup(self, command="python script.py"):
# Create a basic task and run it with string command
test_filepath = os.path.join(self.temp_dir, "script.py")
with open(test_filepath, "wb") as f:
f.write(to_bytes("import os\n"))
f.write(to_bytes("import shutil\n"))
f.write(to_bytes("print('hello')\n"))
test_filepath = os.path.join(self.temp_dir, "Dockerfile")
with open(test_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
return run(command=command, env=test_filepath)
def test_create_from_task_fail_user_inputs(self):
# Setup task
# Create environment definition
env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
task_obj = run("sh -c echo accuracy:0.45", home=self.temp_dir)
# Test if failure if user gives other parameters
failed = False
try:
_ = create(message="my test snapshot",
task_id=task_obj.id,
home=self.temp_dir,
label="best",
config={"foo": "bar"},
stats={"foo": "bar"},
commit_id="test_id")
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
_ = create(message="my test snapshot",
task_id=task_obj.id,
home=self.temp_dir,
label="best",
config={"foo": "bar"},
stats={"foo": "bar"},
environment_id="test_id")
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
failed = False
try:
_ = create(message="my test snapshot",
task_id=task_obj.id,
home=self.temp_dir,
label="best",
config={"foo": "bar"},
stats={"foo": "bar"},
filepaths=["mypath"])
except SnapshotCreateFromTaskArgs:
failed = True
assert failed
def test_create_from_task(self):
# 1) Test if success with task files, results, and message
# TODO: test for failure case where tasks is not complete
# Setup task
# Create environment definition
env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(env_def_path, "w") as f:
f.write(to_unicode(str("FROM datmo/xgboost:cpu")))
task_obj = run("sh -c echo accuracy:0.45", home=self.temp_dir)
snapshot_obj = create_from_task(message="my test snapshot",
task_id=task_obj.id,
home=self.temp_dir)
assert isinstance(snapshot_obj, Snapshot)
assert snapshot_obj.message == "my test snapshot"
assert snapshot_obj.stats == task_obj.results
def test_create_from_task(self):
# 1) Test if success with task files, results, and message
# 2) Test if success with user given config and stats
# TODO: test for failure case where tasks is not complete
# Setup task
# Create environment definition
env_def_path = os.path.join(self.temp_dir, "Dockerfile")
with open(env_def_path, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
task_obj = run("sh -c echo accuracy:0.45")
# 1) Test option 1
snapshot_obj = create(message="my test snapshot",
task_id=task_obj.id,
label="best",
config={"foo": "bar"})
assert isinstance(snapshot_obj, Snapshot)
assert snapshot_obj.message == "my test snapshot"
assert snapshot_obj.label == "best"
assert len(snapshot_obj.files) == 1
assert "task.log" in snapshot_obj.files[0].name
assert snapshot_obj.config == {"foo": "bar"}
assert snapshot_obj.stats == task_obj.results
# Test option 2
snapshot_obj_2 = create(message="my test snapshot",
task_id=task_obj.id,
label="best",
config={"foo": "bar"},
stats={"foo": "bar"})
assert isinstance(snapshot_obj, Snapshot)
assert snapshot_obj_2.message == "my test snapshot"
assert snapshot_obj_2.label == "best"
assert len(snapshot_obj.files) == 1
assert "task.log" in snapshot_obj.files[0].name
assert snapshot_obj_2.config == {"foo": "bar"}
assert snapshot_obj_2.stats == {"foo": "bar"}
def test_task_entity_files(self):
input_dict = {
"id": "test",
"model_id": "my_model",
"session_id": "my_session",
"command": "python test.py"
}
core_task_entity = CoreTask(input_dict)
task_entity = Task(core_task_entity, home=self.temp_dir)
# Test failure because entity has not been created by controller
failed = False
try:
task_entity.files()
except EntityNotFound:
failed = True
assert failed
# Create a basic task and run it with string command
test_filepath = os.path.join(self.temp_dir, "script.py")
with open(test_filepath, "w") as f:
f.write(to_unicode("import numpy\n"))
f.write(to_unicode("import sklearn\n"))
f.write(to_unicode("print 'hello'\n"))
test_filepath = os.path.join(self.temp_dir, "Dockerfile")
with open(test_filepath, "w") as f:
f.write(to_unicode("FROM datmo/xgboost:cpu"))
task_entity = run(command="python script.py",
env=test_filepath,
home=self.temp_dir)
result = task_entity.files()
assert len(result) == 1
assert isinstance(result[0], TextIOWrapper)
assert result[0].name
def test_run(self):
# 1) Run task with no commit or code available (cannot save states before), string command
# 2) Run task with simple python file, no environment definition, string command (auto generate env)
# 3) Run task with simple python file and environment definition, string command
# 4) Run task with simple python file and environment definition, list command
# 5) Run task with simple python file and environment definition passed as a list
# 6) Run task with simple python file and environment definition path present in project environment directory
# folder in project
# 1) Test out option 1)
failed = False
try:
_ = run(command="test")
except CommitFailed:
failed = True
assert failed
# Create a basic task and run it with string command
# (fails w/ no environment)
test_mem_limit = "4g"
test_filepath = os.path.join(self.temp_dir, "script.py")
with open(test_filepath, "wb") as f:
f.write(to_bytes("import os\n"))
f.write(to_bytes("import sys\n"))
f.write(to_bytes("print('hello')\n"))
f.write(to_bytes("print(' accuracy: 0.56 ')\n"))
# 2) Test out option 2
task_obj_0 = run(command="python script.py", mem_limit=test_mem_limit)
assert isinstance(task_obj_0, Task)
assert task_obj_0.id
assert 'hello' in task_obj_0.logs
assert task_obj_0.results == {"accuracy": "0.56"}
# Add environment definition
test_filepath = os.path.join(self.temp_dir, "Dockerfile")
with open(test_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
# 3) Test out option 3
task_obj_1 = run(command="python script.py",
env=test_filepath,
mem_limit=test_mem_limit)
assert isinstance(task_obj_1, Task)
assert task_obj_1.id
assert 'hello' in task_obj_1.logs
assert task_obj_1.results == {"accuracy": "0.56"}
# 4) Test out option 4
task_obj_2 = run(command=["python", "script.py"],
env=test_filepath,
mem_limit=test_mem_limit)
assert isinstance(task_obj_2, Task)
assert task_obj_2.id
assert 'hello' in task_obj_2.logs
assert task_obj_2.results == {"accuracy": "0.56"}
# 5) Test out option 5
task_obj_3 = run(command=["python", "script.py"],
env=[test_filepath + ">Dockerfile"],
mem_limit=test_mem_limit)
assert isinstance(task_obj_3, Task)
assert task_obj_3.id
assert 'hello' in task_obj_3.logs
assert task_obj_3.results == {"accuracy": "0.56"}
# 6) Test out option 6
os.remove(test_filepath)
test_filepath = os.path.join(
self.project_controller.file_driver.environment_directory,
"Dockerfile")
with open(test_filepath, "wb") as f:
f.write(to_bytes("FROM python:3.5-alpine"))
task_obj_4 = run(command=["python", "script.py"])
assert isinstance(task_obj_4, Task)
assert task_obj_4.id
assert 'hello' in task_obj_4.logs
assert task_obj_4.results == {"accuracy": "0.56"}