class TaskController(BaseController): """TaskController inherits from BaseController and manages business logic associated with tasks within the project. Parameters ---------- home : str home path of the project Attributes ---------- environment : datmo.core.controller.environment.environment.EnvironmentController used to create environment if new definition file snapshot : datmo.core.controller.snapshot.SnapshotController used to create snapshots before and after tasks Methods ------- create(dictionary) creates a Task object with the permanent parameters _run_helper(environment_id, log_filepath, options) helper for run to start environment and run with the appropriate parameters run(self, id, dictionary=None) runs the task and tracks the run, logs, inputs and outputs list(session_id=None) lists all tasks within the project given filters delete(id) deletes the specified task from the project """ def __init__(self): super(TaskController, self).__init__() self.environment = EnvironmentController() self.snapshot = SnapshotController() self.spinner = Spinner() if not self.is_initialized: raise ProjectNotInitialized( __("error", "controller.task.__init__")) def create(self): """Create Task object Returns ------- Task object entity for Task (datmo.core.entity.task.Task) """ # Validate Inputs create_dict = { "model_id": self.model.id, "session_id": self.current_session.id } try: # Create Task self.spinner.start() task_obj = self.dal.task.create(Task(create_dict)) finally: self.spinner.stop() return task_obj def _run_helper(self, environment_id, options, log_filepath): """Run environment with parameters Parameters ---------- environment_id : str the environment id for definition options : dict can include the following values: command : list ports : list Here are some example ports used for common applications. * 'jupyter notebook' - 8888 * flask API - 5000 * tensorboard - 6006 An example input for the above would be ["8888:8888", "5000:5000", "6006:6006"] which maps the running host port (right) to that of the environment (left) name : str volumes : dict mem_limit : str workspace : str detach : bool stdin_open : bool tty : bool log_filepath : str absolute filepath to the log file Returns ------- return_code : int system return code of the environment that was run run_id : str id of the environment run (different from environment id) logs : str output logs from the run """ # Run container with options provided run_options = { "command": options.get('command', None), "ports": options.get('ports', None), "name": options.get('name', None), "volumes": options.get('volumes', None), "mem_limit": options.get('mem_limit', None), "gpu": options.get('gpu', False), "detach": options.get('detach', False), "stdin_open": options.get('stdin_open', False), "tty": options.get('tty', False), "api": False, } workspace = options.get('workspace', None) self.environment.build(environment_id, workspace) # Run container with environment return_code, run_id, logs = self.environment.run( environment_id, run_options, log_filepath) return return_code, run_id, logs def _parse_logs_for_results(self, logs): """Parse log string to extract results and return dictionary. The format of the log line must be "key:value", whitespace will not matter and if there are more than 2 items found when split on ":", it will not log this as a key/value result Note ---- If the same key is found multiple times in the logs, the last occurring one will be the one that is saved. Parameters ---------- logs : str raw string value of output logs Returns ------- dict or None dictionary to represent results from task """ results = {} for line in logs.split("\n"): split_line = line.split(":") if len(split_line) == 2: results[split_line[0].strip()] = split_line[1].strip() if results == {}: results = None return results def run(self, task_id, snapshot_dict=None, task_dict=None): """Run a task with parameters. If dictionary specified, create a new task with new run parameters. Snapshot objects are created before and after the task to keep track of the state. During the run, you can access task outputs using environment variable DATMO_TASK_DIR or `/task` which points to location for the task files. Create config.json, stats.json and any weights or any file such as graphs and visualizations within that directory for quick access Parameters ---------- task_id : str id for the task you would like to run snapshot_dict : dict set of parameters to create a snapshot (see SnapshotController for details. default is None, which means dictionary with `visible` False will be added to hide auto-generated snapshot) NOTE: `visible` False will always be False regardless of whether the user provides another value for `visible`. task_dict : dict set of parameters to characterize the task run (default is None, which translate to {}, see datmo.core.entity.task.Task for more details on inputs) Returns ------- Task the Task object which completed its run with updated parameters Raises ------ TaskRunError If there is any error in creating files for the task or downstream errors """ # Ensure visible=False is present in the snapshot dictionary if not snapshot_dict: snapshot_dict = {"visible": False} else: snapshot_dict['visible'] = False if not task_dict: task_dict = {} # Obtain Task to run task_obj = self.dal.task.get_by_id(task_id) # Ensure that at least 1 of command, command_list, or interactive is present in task_dict important_task_args = ["command", "command_list", "interactive"] if not task_dict.get('command', task_obj.command) and \ not task_dict.get('command_list', task_obj.command_list) and \ not task_dict.get('interactive', task_obj.interactive): raise RequiredArgumentMissing( __("error", "controller.task.run.arg", " or ".join(important_task_args))) if task_obj.status is None: task_obj.status = "RUNNING" else: raise TaskRunError( __("error", "cli.run.run.already_running", task_obj.id)) # Create Task directory for user during run task_dirpath = os.path.join(".datmo", "tasks", task_obj.id) try: _ = self.file_driver.create(task_dirpath, directory=True) except Exception: raise TaskRunError( __("error", "controller.task.run", task_dirpath)) # Create the before snapshot prior to execution before_snapshot_dict = snapshot_dict.copy() before_snapshot_dict[ 'message'] = "autogenerated snapshot created before task %s is run" % task_obj.id before_snapshot_obj = self.snapshot.create(before_snapshot_dict) # Update the task with pre-execution parameters, prefer list first then look for string command # List command will overwrite a string command if given if task_dict.get('command_list', task_obj.command_list): task_dict['command'] = " ".join( task_dict.get('command_list', task_obj.command_list)) else: if task_dict.get('command', task_obj.command): task_dict['command_list'] = shlex.split( task_dict.get('command', task_obj.command)) elif not task_dict.get('interactive', task_obj.interactive): # If it's not interactive then there is not expected task raise TaskNoCommandGiven() validate("create_task", task_dict) task_obj = self.dal.task.update({ "id": task_obj.id, "before_snapshot_id": task_dict.get('before_snapshot_id', before_snapshot_obj.id), "command": task_dict.get('command', task_obj.command), "command_list": task_dict.get('command_list', task_obj.command_list), "gpu": task_dict.get('gpu', False), "mem_limit": task_dict.get('mem_limit', None), "workspace": task_dict.get('workspace', None), "interactive": task_dict.get('interactive', task_obj.interactive), "detach": task_dict.get('detach', task_obj.detach), "ports": task_dict.get('ports', task_obj.ports), "task_dirpath": task_dict.get('task_dirpath', task_dirpath), "log_filepath": task_dict.get('log_filepath', os.path.join(task_dirpath, "task.log")), "start_time": task_dict.get('start_time', datetime.utcnow()), "status": task_obj.status }) # Copy over files from the before_snapshot file collection to task dir file_collection_obj = \ self.dal.file_collection.get_by_id(before_snapshot_obj.file_collection_id) self.file_driver.copytree( os.path.join(self.home, file_collection_obj.path), os.path.join(self.home, task_obj.task_dirpath)) return_code, run_id, logs = 0, None, None try: # Set the parameters set in the task if task_obj.detach and task_obj.interactive: raise TaskInteractiveDetachError( __("error", "controller.task.run.args.detach.interactive")) environment_run_options = { "command": task_obj.command_list, "ports": [] if task_obj.ports is None else task_obj.ports, "name": "datmo-task-" + self.model.id + "-" + task_obj.id, "volumes": { os.path.join(self.home, task_obj.task_dirpath): { 'bind': '/task/', 'mode': 'rw' }, self.home: { 'bind': '/home/', 'mode': 'rw' } }, "mem_limit": task_obj.mem_limit, "workspace": task_obj.workspace, "gpu": task_obj.gpu, "detach": task_obj.detach, "stdin_open": task_obj.interactive, "tty": task_obj.interactive, "api": False } # Run environment via the helper function return_code, run_id, logs = \ self._run_helper(before_snapshot_obj.environment_id, environment_run_options, os.path.join(self.home, task_obj.log_filepath)) except Exception as e: return_code = 1 logs += "Error running task: %" % e.message finally: # Create the after snapshot after execution is completed with new paths after_snapshot_dict = snapshot_dict.copy() after_snapshot_dict[ 'message'] = "autogenerated snapshot created after task %s is run" % task_obj.id # Add in absolute paths from running task directory absolute_task_dir_path = os.path.join(self.home, task_obj.task_dirpath) absolute_paths = [] for item in os.listdir(absolute_task_dir_path): path = os.path.join(absolute_task_dir_path, item) if os.path.isfile(path) or os.path.isdir(path): absolute_paths.append(path) after_snapshot_dict.update({ "paths": absolute_paths, "environment_id": before_snapshot_obj.environment_id, }) after_snapshot_obj = self.snapshot.create(after_snapshot_dict) # (optional) Remove temporary task directory path # Update the task with post-execution parameters end_time = datetime.utcnow() duration = (end_time - task_obj.start_time).total_seconds() update_task_dict = { "id": task_obj.id, "after_snapshot_id": after_snapshot_obj.id, "logs": logs, "status": "SUCCESS" if return_code == 0 else "FAILED", # "results": task_obj.results, # TODO: update during run "end_time": end_time, "duration": duration } if logs is not None: update_task_dict["results"] = self._parse_logs_for_results( logs) if run_id is not None: update_task_dict["run_id"] = run_id return self.dal.task.update(update_task_dict) def list(self, session_id=None, sort_key=None, sort_order=None): query = {} if session_id: try: self.dal.session.get_by_id(session_id) except EntityNotFound: raise SessionDoesNotExist( __("error", "controller.task.list", session_id)) query['session_id'] = session_id return self.dal.task.query(query, sort_key, sort_order) def get(self, task_id): """Get task object and return Parameters ---------- task_id : str id for the task you would like to get Returns ------- datmo.core.entity.task.Task core task object Raises ------ DoesNotExist task does not exist """ try: return self.dal.task.get_by_id(task_id) except EntityNotFound: raise DoesNotExist() def get_files(self, task_id, mode="r"): """Get list of file objects for task id. It will look in the following areas in the following order 1) look in the after snapshot for file collection 2) look in the running task file collection 3) look in the before snapshot for file collection Parameters ---------- task_id : str id for the task you would like to get file objects for mode : str file open mode (default is "r" to open file for read) Returns ------- list list of python file objects Raises ------ DoesNotExist task object does not exist PathDoesNotExist no file objects exist for the task """ try: task_obj = self.dal.task.get_by_id(task_id) except EntityNotFound: raise DoesNotExist() if task_obj.after_snapshot_id: # perform number 1) and return file list return self.snapshot.get_files( task_obj.after_snapshot_id, mode=mode) elif task_obj.task_dirpath: # perform number 2) and return file list return self.file_driver.get( task_obj.task_dirpath, mode=mode, directory=True) elif task_obj.before_snapshot_id: # perform number 3) and return file list return self.snapshot.get_files( task_obj.before_snapshot_id, mode=mode) else: # Error because the task does not have any files associated with it raise PathDoesNotExist() def delete(self, task_id): if not task_id: raise RequiredArgumentMissing( __("error", "controller.task.delete.arg", "id")) stopped_success = self.stop(task_id) delete_task_success = self.dal.task.delete(task_id) return stopped_success and delete_task_success def stop(self, task_id=None, all=False, status="STOPPED"): """Stop and remove run for the task and update task object statuses Parameters ---------- task_id : str, optional id for the task you would like to stop all : bool, optional if specified, will stop all tasks within project Returns ------- return_code : bool system return code of the stop Raises ------ RequiredArgumentMissing TooManyArgumentsFound """ if task_id is None and all is False: raise RequiredArgumentMissing( __("error", "controller.task.stop.arg.missing", "id")) if task_id and all: raise TooManyArgumentsFound() if task_id: try: task_obj = self.get(task_id) except DoesNotExist: time.sleep(1) task_obj = self.get(task_id) task_match_string = "datmo-task-" + self.model.id + "-" + task_id # Get the environment id associated with the task kwargs = {'match_string': task_match_string} # Get the environment from the task before_snapshot_id = task_obj.before_snapshot_id after_snapshot_id = task_obj.after_snapshot_id if not before_snapshot_id and not after_snapshot_id: # TODO: remove...for now database may not be in sync. no task that has run can have NO before_snapshot_id time.sleep(1) task_obj = self.get(task_id) if after_snapshot_id: after_snapshot_obj = self.snapshot.get(after_snapshot_id) kwargs['environment_id'] = after_snapshot_obj.environment_id if not after_snapshot_id and before_snapshot_id: before_snapshot_obj = self.snapshot.get(before_snapshot_id) kwargs['environment_id'] = before_snapshot_obj.environment_id return_code = self.environment.stop(**kwargs) if all: return_code = self.environment.stop(all=True) # Set stopped task statuses to STOPPED if return success if return_code: if task_id: self.dal.task.update({"id": task_id, "status": status}) if all: task_objs = self.dal.task.query({}) for task_obj in task_objs: self.dal.task.update({"id": task_obj.id, "status": status}) return return_code
class TestEnvironmentController(): def setup_method(self): # provide mountable tmp directory for docker tempfile.tempdir = "/tmp" if not platform.system( ) == "Windows" else None test_datmo_dir = os.environ.get('TEST_DATMO_DIR', tempfile.gettempdir()) self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir) self.project = ProjectController(self.temp_dir) self.environment = EnvironmentController(self.temp_dir) def teardown_method(self): pass def test_create(self): # 0) Test create when unsupported language given # 1) Test create when NO file exists and NO definition path exists # 2) Test create when NO file exists and definition path exists # 3) Test create when definition path exists and given # 4) Test create when file exists and definition path exists # 5) Test create when file exists but NO definition path exists # 6) Test create when definition path exists and given for NEW definition filepath self.project.init("test3", "test description") # 0) Test option 0 try: self.environment.create({"language": "java"}) except EnvironmentDoesNotExist: failed = True assert failed # 1) Test option 1 failed = False try: self.environment.create({}) except EnvironmentDoesNotExist: failed = True assert failed # Create environment definition definition_filepath = os.path.join(self.environment.home, "Dockerfile") with open(definition_filepath, "w") as f: f.write(to_unicode(str("FROM datmo/xgboost:cpu"))) # 2) Test option 2 environment_obj_1 = self.environment.create({}) # Get file collection path file_collection_obj = self.environment.dal.file_collection. \ get_by_id(environment_obj_1.file_collection_id) file_collection_dir = self.environment.file_driver. \ get_collection_path(file_collection_obj.filehash) assert environment_obj_1 assert environment_obj_1.id assert environment_obj_1.driver_type == "docker" assert environment_obj_1.file_collection_id assert environment_obj_1.definition_filename assert environment_obj_1.hardware_info assert environment_obj_1.unique_hash == file_collection_obj.filehash assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "datmoDockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "hardware_info")) # 3) Test option 3 input_dict = { "definition_filepath": definition_filepath, } # Create environment in the project environment_obj_2 = self.environment.create(input_dict) # Get file collection path file_collection_obj = self.environment.dal.file_collection. \ get_by_id(environment_obj_2.file_collection_id) file_collection_dir = self.environment.file_driver. \ get_collection_path(file_collection_obj.filehash) assert environment_obj_2 assert environment_obj_2.id assert environment_obj_2.driver_type == "docker" assert environment_obj_2.file_collection_id assert environment_obj_2.definition_filename assert environment_obj_2.hardware_info assert environment_obj_2.unique_hash == file_collection_obj.filehash assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "datmoDockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "hardware_info")) # Create script to test test_filepath = os.path.join(self.environment.home, "script.py") with open(test_filepath, "w") as f: f.write(to_unicode("import numpy\n")) f.write(to_unicode("import sklearn\n")) f.write(to_unicode("print('hello')\n")) # 4) Test option 4 environment_obj_3 = self.environment.create({}) # Get file collection path file_collection_obj = self.environment.dal.file_collection. \ get_by_id(environment_obj_3.file_collection_id) file_collection_dir = self.environment.file_driver. \ get_collection_path(file_collection_obj.filehash) assert environment_obj_3 assert environment_obj_3.id assert environment_obj_3.driver_type == "docker" assert environment_obj_3.file_collection_id assert environment_obj_3.definition_filename assert environment_obj_3.hardware_info assert environment_obj_3.unique_hash == file_collection_obj.filehash assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "datmoDockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "hardware_info")) # Remove definition filepath os.remove(definition_filepath) assert environment_obj_1.id == environment_obj_2.id assert environment_obj_2.id == environment_obj_3.id # 5) Test option 5 environment_obj_4 = self.environment.create({}) # Get file collection path file_collection_obj = self.environment.dal.file_collection. \ get_by_id(environment_obj_4.file_collection_id) file_collection_dir = self.environment.file_driver. \ get_collection_path(file_collection_obj.filehash) assert environment_obj_4 assert environment_obj_4.id assert environment_obj_4.driver_type == "docker" assert environment_obj_4.file_collection_id assert environment_obj_4.definition_filename assert environment_obj_4.hardware_info assert environment_obj_4.unique_hash == file_collection_obj.filehash assert os.path.isfile( os.path.join(file_collection_dir, "requirements.txt")) assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "datmoDockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "hardware_info")) assert environment_obj_1.id != environment_obj_4.id # 6) Test option 6 # Create environment definition definition_filepath = os.path.join(self.environment.home, "Dockerfile") with open(definition_filepath, "w") as f: f.write(to_unicode(str("FROM cloudgear/ubuntu:14.04"))) input_dict = { "definition_filepath": definition_filepath, } # Create a new environment obj environment_obj_5 = self.environment.create(input_dict) # Get file collection path file_collection_obj = self.environment.dal.file_collection. \ get_by_id(environment_obj_5.file_collection_id) file_collection_dir = self.environment.file_driver. \ get_collection_path(file_collection_obj.filehash) assert environment_obj_5 assert environment_obj_5.id assert environment_obj_5.driver_type == "docker" assert environment_obj_5.file_collection_id assert environment_obj_5.definition_filename assert environment_obj_5.hardware_info assert environment_obj_5.unique_hash == file_collection_obj.filehash assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "datmoDockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "hardware_info")) assert environment_obj_5.id != environment_obj_1.id assert environment_obj_5.id != environment_obj_4.id def test_build(self): # 1) Test build when no environment given # 2) Test build when definition path exists and given # 3) Test build when NO file exists and definition path exists # 4) Test build when file exists and definition path exists # 5) Test build when file exists but NO definition path exists self.project.init("test5", "test description") # 1) Test option 1 failed = False try: _ = self.environment.build("does_not_exist") except EntityNotFound: failed = True assert failed # Create environment definition definition_filepath = os.path.join(self.environment.home, "Dockerfile") with open(definition_filepath, "w") as f: f.write(to_unicode(str("FROM datmo/xgboost:cpu"))) input_dict = { "definition_filepath": definition_filepath, } # 2) Test option 2 # Create environment in the project environment_obj_1 = self.environment.create(input_dict) result = self.environment.build(environment_obj_1.id) assert result # 3) Test option 3 # Create environment in the project environment_obj_2 = self.environment.create({}) result = self.environment.build(environment_obj_2.id) assert result # Create script to test test_filepath = os.path.join(self.environment.home, "script.py") with open(test_filepath, "w") as f: f.write(to_unicode("import numpy\n")) f.write(to_unicode("import sklearn\n")) f.write(to_unicode("print('hello')\n")) # 4) Test option 4 environment_obj_3 = self.environment.create({}) result = self.environment.build(environment_obj_3.id) assert result # test 2), 3), and 4) will result in the same environment assert environment_obj_1.id == environment_obj_2.id assert environment_obj_2.id == environment_obj_3.id # Test for building dockerfile when there exists not os.remove(definition_filepath) # 5) Test option 5 environment_obj_4 = self.environment.create({}) result = self.environment.build(environment_obj_4.id) assert result assert environment_obj_4.id != environment_obj_1.id # teardown self.environment.delete(environment_obj_1.id) self.environment.delete(environment_obj_4.id) def test_run(self): # 1) Test run simple command with simple Dockerfile # 2) Test run script, with autogenerated definition self.project.init("test5", "test description") # 1) Test option 1 # Create environment definition definition_filepath = os.path.join(self.environment.home, "Dockerfile") with open(definition_filepath, "w") as f: f.write(to_unicode(str("FROM datmo/xgboost:cpu"))) run_options = { "command": ["sh", "-c", "echo yo"], "ports": ["8888:8888"], "name": None, "volumes": None, "detach": False, "stdin_open": False, "tty": False, "gpu": False, "api": False } input_dict = { "definition_filepath": definition_filepath, } # Create environment in the project environment_obj = self.environment.create(input_dict) log_filepath = os.path.join(self.project.home, "task.log") # Build environment in the project _ = self.environment.build(environment_obj.id) # Run environment in the project return_code, run_id, logs = \ self.environment.run(environment_obj.id, run_options, log_filepath) assert return_code == 0 assert run_id assert logs # teardown self.environment.delete(environment_obj.id) # 2) Test option 2 os.remove(definition_filepath) # Create script to test test_filepath = os.path.join(self.environment.home, "script.py") with open(test_filepath, "w") as f: f.write(to_unicode("import numpy\n")) f.write(to_unicode("import sklearn\n")) f.write(to_unicode("print('hello')\n")) # Create environment in the project environment_obj = self.environment.create({}) self.environment.build(environment_obj.id) run_options = { "command": ["python", "script.py"], "ports": ["8888:8888"], "name": None, "volumes": { self.environment.home: { 'bind': '/home/', 'mode': 'rw' } }, "detach": False, "stdin_open": False, "tty": False, "gpu": False, "api": False } # Run environment in the project return_code, run_id, logs = \ self.environment.run(environment_obj.id, run_options, log_filepath) assert return_code == 0 assert run_id assert logs # teardown self.environment.delete(environment_obj.id) def test_list(self): self.project.init("test4", "test description") # Create environment definition for object 1 definition_path_1 = os.path.join(self.environment.home, "Dockerfile") with open(definition_path_1, "w") as f: f.write(to_unicode(str("FROM datmo/xgboost:cpu"))) input_dict_1 = { "definition_filepath": definition_path_1, } # Create environment in the project environment_obj_1 = self.environment.create(input_dict_1) # Create environment definition for object 2 definition_path_2 = os.path.join(self.environment.home, "Dockerfile2") with open(definition_path_2, "w") as f: f.write(to_unicode(str("FROM datmo/scikit-opencv"))) input_dict_2 = { "definition_filepath": definition_path_2, } # Create second environment in the project environment_obj_2 = self.environment.create(input_dict_2) # List all environments and ensure they exist result = self.environment.list() assert len(result) == 2 and \ environment_obj_1 in result and \ environment_obj_2 in result def test_delete(self): self.project.init("test5", "test description") # Create environment definition definition_filepath = os.path.join(self.environment.home, "Dockerfile") with open(definition_filepath, "w") as f: f.write(to_unicode(str("FROM datmo/xgboost:cpu"))) input_dict = { "definition_filepath": definition_filepath, } # Create environment in the project environment_obj = self.environment.create(input_dict) # Delete environment in the project result = self.environment.delete(environment_obj.id) # Check if environment retrieval throws error thrown = False try: self.environment.dal.environment.get_by_id(environment_obj.id) except EntityNotFound: thrown = True assert result == True and \ thrown == True def test_stop(self): self.project.init("test5", "test description") # Create environment definition definition_filepath = os.path.join(self.environment.home, "Dockerfile") with open(definition_filepath, "w") as f: f.write(to_unicode(str("FROM datmo/xgboost:cpu"))) run_options = { "command": ["sh", "-c", "echo yo"], "ports": ["8888:8888"], "name": None, "volumes": None, "detach": False, "stdin_open": False, "tty": False, "gpu": False, "api": False } # Create environment_driver definition env_def_path = os.path.join(self.project.home, "Dockerfile") with open(env_def_path, "w") as f: f.write(to_unicode(str("FROM datmo/xgboost:cpu"))) input_dict = { "definition_filepath": definition_filepath, } # Create environment in the project environment_obj = self.environment.create(input_dict) log_filepath = os.path.join(self.project.home, "task.log") # Build environment in the project _ = self.environment.build(environment_obj.id) # Run environment in the project _, run_id, _ = \ self.environment.run(environment_obj.id, run_options, log_filepath) # Stop the running environment return_code = self.environment.stop(run_id) assert return_code # teardown self.environment.delete(environment_obj.id)
class TaskController(BaseController): """TaskController inherits from BaseController and manages business logic associated with tasks within the project. Parameters ---------- home : str home path of the project Attributes ---------- environment : EnvironmentController used to create environment if new definition file snapshot : SnapshotController used to create snapshots before and after tasks Methods ------- create(dictionary) creates a Task object with the permanent parameters _run_helper(environment_id, log_filepath, options) helper for run to start environment and run with the appropriate parameters run(self, id, dictionary=None) runs the task and tracks the run, logs, inputs and outputs list(session_id=None) lists all tasks within the project given filters delete(id) deletes the specified task from the project """ def __init__(self, home): super(TaskController, self).__init__(home) self.environment = EnvironmentController(home) self.snapshot = SnapshotController(home) if not self.is_initialized: raise ProjectNotInitializedException( __("error", "controller.task.__init__")) def create(self, dictionary): """Create Task object Parameters ---------- dictionary : dict command : str full command used Returns ------- Task object entity for Task (datmo.core.entity.task.Task) """ # Validate Inputs create_dict = { "model_id": self.model.id, "session_id": self.current_session.id } ## Required args required_args = ["command"] for required_arg in required_args: # Add in any values that are if required_arg in dictionary and dictionary[ required_arg] is not None: create_dict[required_arg] = dictionary[required_arg] else: raise RequiredArgumentMissing( __("error", "controller.task.create.arg", required_arg)) # Create Task return self.dal.task.create(Task(create_dict)) def _run_helper(self, environment_id, options, log_filepath): """Run environment with parameters Parameters ---------- environment_id : str the environment id for definition options : dict can include the following values: command : list ports : list Here are some example ports used for common applications. * 'jupyter notebook' - 8888 * flask API - 5000 * tensorboard - 6006 An example input for the above would be ["8888:8888", "5000:5000", "6006:6006"] which maps the running host port (right) to that of the environment (left) name : str volumes : dict detach : bool stdin_open : bool tty : bool gpu : bool log_filepath : str absolute filepath to the log file Returns ------- return_code : int system return code of the environment that was run run_id : str id of the environment run (different from environment id) logs : str output logs from the run """ # Run container with options provided run_options = { "command": options.get('command', None), "ports": options.get('ports', None), "name": options.get('name', None), "volumes": options.get('volumes', None), "detach": options.get('detach', False), "stdin_open": options.get('stdin_open', False), "tty": options.get('tty', False), "gpu": options.get('gpu', False), "api": False } # Build image for environment self.environment.build(environment_id) # Run container with environment return_code, run_id, logs = \ self.environment.run(environment_id, run_options, log_filepath) return return_code, run_id, logs def _parse_logs_for_results(self, logs): """Parse log string to extract results and return dictionary. Note ---- If the same key is found multiple times in the logs, the last occurring one will be the one that is saved. Parameters ---------- logs : str raw string value of output logs Returns ------- dict dictionary to represent results from task """ results = {} for line in logs.split("\n"): split_line = line.split(":") if len(split_line) == 2: results[split_line[0].strip()] = split_line[1].strip() return results def run(self, task_id, snapshot_dict=None, task_dict=None): """Run a task with parameters. If dictionary specified, create a new task with new run parameters. Snapshot objects are created before and after the task to keep track of the state. During the run, you can access task outputs using environment variable DATMO_TASK_DIR or `/task` which points to location of datmo_tasks/[task-id]. Create config.json, stats.json and any weights or any file such as graphs and visualizations within that directory for quick access Parameters ---------- task_id : str id for the task you would like to run snapshot_dict : dict set of parameters to create a snapshot (see SnapshotController for details. default is None, which means dictionary with `visible` False will be added to hide auto-generated snapshot) NOTE: `visible` False will always be False regardless of whether the user provides another value for `visible`. task_dict : dict set of parameters to characterize the task run (default is None, which translate to {}, see datmo.core.entity.task.Task for more details on inputs) Returns ------- Task the Task object which completed its run with updated parameters Raises ------ TaskRunException If there is any error in creating files for the task or downstream errors """ # Ensure visible=False is present in the snapshot dictionary if not snapshot_dict: snapshot_dict = {"visible": False} else: snapshot_dict['visible'] = False if not task_dict: task_dict = {} # Obtain Task to run task_obj = self.dal.task.get_by_id(task_id) if task_obj.status == None: task_obj.status = 'RUNNING' else: raise TaskRunException( __("error", "cli.task.run.already_running", task_obj.id)) # Create Task directory for user during run task_dirpath = os.path.join("datmo_tasks", task_obj.id) try: _ = self.file_driver.create(os.path.join("datmo_tasks", task_obj.id), directory=True) except: raise TaskRunException( __("error", "controller.task.run", task_dirpath)) # Create the before snapshot prior to execution before_snapshot_dict = snapshot_dict.copy() before_snapshot_dict[ 'message'] = "autogenerated snapshot created before task %s is run" % task_obj.id before_snapshot_obj = self.snapshot.create(before_snapshot_dict) # Update the task with pre-execution parameters task_obj = self.dal.task.update({ "id": task_obj.id, "before_snapshot_id": task_dict.get('before_snapshot_id', before_snapshot_obj.id), "ports": task_dict.get('ports', task_obj.ports), "gpu": task_dict.get('gpu', task_obj.gpu), "interactive": task_dict.get('interactive', task_obj.interactive), "task_dirpath": task_dict.get('task_dirpath', task_dirpath), "log_filepath": task_dict.get('log_filepath', os.path.join(task_dirpath, "task.log")), "start_time": task_dict.get('start_time', datetime.utcnow()) }) # Copy over files from the before_snapshot file collection to task dir file_collection_obj = \ self.dal.file_collection.get_by_id(before_snapshot_obj.file_collection_id) self.file_driver.copytree( os.path.join(self.home, file_collection_obj.path), os.path.join(self.home, task_obj.task_dirpath)) # Set the parameters set in the task environment_run_options = { "command": task_obj.command, "ports": [] if task_obj.ports is None else task_obj.ports, "gpu": task_obj.gpu, "name": "datmo-task-" + task_obj.id, "volumes": { os.path.join(self.home, task_obj.task_dirpath): { 'bind': '/task/', 'mode': 'rw' }, self.home: { 'bind': '/home/', 'mode': 'rw' } }, "detach": task_obj.interactive, "stdin_open": task_obj.interactive, "tty": False, "api": not task_obj.interactive } # Run environment via the helper function return_code, run_id, logs = \ self._run_helper(before_snapshot_obj.environment_id, environment_run_options, os.path.join(self.home, task_obj.log_filepath)) # Create the after snapshot after execution is completed with new filepaths after_snapshot_dict = snapshot_dict.copy() after_snapshot_dict[ 'message'] = "autogenerated snapshot created after task %s is run" % task_obj.id # Add in absolute filepaths from running task directory absolute_task_dir_path = os.path.join(self.home, task_obj.task_dirpath) absolute_filepaths = [] for item in os.listdir(absolute_task_dir_path): path = os.path.join(absolute_task_dir_path, item) if os.path.isfile(path) or os.path.isdir(path): absolute_filepaths.append(path) after_snapshot_dict.update({ "filepaths": absolute_filepaths, "environment_id": before_snapshot_obj.environment_id, }) after_snapshot_obj = self.snapshot.create(after_snapshot_dict) # (optional) Remove temporary task directory path # Update the task with post-execution parameters end_time = datetime.utcnow() duration = (end_time - task_obj.start_time).total_seconds() return self.dal.task.update({ "id": task_obj.id, "after_snapshot_id": after_snapshot_obj.id, "run_id": run_id, "logs": logs, "results": self._parse_logs_for_results(logs), # "results": task_obj.results, # TODO: update during run "status": "SUCCESS" if return_code == 0 else "FAILED", "end_time": end_time, "duration": duration }) def list(self, session_id=None): query = {} if session_id: query['session_id'] = session_id return self.dal.task.query(query) def get_files(self, task_id, mode="r"): """Get list of file objects for task id. It will look in the following areas in the following order 1) look in the after snapshot for file collection 2) look in the running task file collection 3) look in the before snapshot for file collection Parameters ---------- task_id : str id for the task you would like to get file objects for mode : str file open mode (default is "r" to open file for read) Returns ------- list list of python file objects Raises ------ PathDoesNotExist no file objects exist for the task """ task_obj = self.dal.task.get_by_id(task_id) if task_obj.after_snapshot_id: # perform number 1) and return file list after_snapshot_obj = \ self.dal.snapshot.get_by_id(task_obj.after_snapshot_id) file_collection_obj = \ self.dal.file_collection.get_by_id(after_snapshot_obj.file_collection_id) return self.file_driver.\ get_collection_files(file_collection_obj.filehash, mode=mode) elif task_obj.task_dirpath: # perform number 2) and return file list return self.file_driver.get(task_obj.task_dirpath, mode=mode, directory=True) elif task_obj.before_snapshot_id: # perform number 3) and return file list before_snapshot_obj = \ self.dal.snapshot.get_by_id(task_obj.before_snapshot_id) file_collection_obj = \ self.dal.file_collection.get_by_id(before_snapshot_obj.file_collection_id) return self.file_driver. \ get_collection_files(file_collection_obj.filehash, mode=mode) else: # Error because the task does not have any files associated with it raise PathDoesNotExist() def delete(self, task_id): if not task_id: raise RequiredArgumentMissing( __("error", "controller.task.delete.arg", "id")) return self.dal.task.delete(task_id) def stop(self, task_id): """Stop and remove run for the task Parameters ---------- task_id : str id for the task you would like to stop Returns ------- return_code : bool system return code of the stop """ if not task_id: raise RequiredArgumentMissing( __("error", "controller.task.stop.arg", "id")) task_obj = self.dal.task.get_by_id(task_id) run_id = task_obj.run_id return_code = self.environment.stop(run_id) return return_code
class TestEnvironmentController(): def setup_method(self): self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir) Config().set_home(self.temp_dir) self.project_controller = ProjectController() def teardown_method(self): pass def __setup(self): self.project_controller.init("test_setup", "test description") self.environment_controller = EnvironmentController() with open(os.path.join(self.temp_dir, "test.txt"), "wb") as f: f.write(to_bytes("hello")) self.random_filepath = os.path.join( self.environment_controller.file_driver.environment_directory, "test") with open(self.random_filepath, "wb") as f: f.write(to_bytes("cool")) self.definition_filepath = os.path.join( self.environment_controller.file_driver.environment_directory, "Dockerfile") with open(self.definition_filepath, "wb") as f: f.write(to_bytes("FROM python:3.5-alpine")) def test_init_fail_project_not_init(self): Config().set_home(self.temp_dir) failed = False try: EnvironmentController() except ProjectNotInitialized: failed = True assert failed def test_get_supported_environments(self): self.__setup() result = self.environment_controller.get_supported_environments() assert result def test_setup(self): self.project_controller.init("test_setup", "test description") self.environment_controller = EnvironmentController() # Test success setup once (no files present) options = {"name": "xgboost:cpu"} result = self.environment_controller.setup(options=options) output_definition_filepath = os.path.join( self.environment_controller.file_driver.environment_directory, "Dockerfile") assert isinstance(result, Environment) assert result.name == options['name'] assert result.description == "supported base environment created by datmo" assert os.path.isfile(output_definition_filepath) assert "FROM datmo/xgboost:cpu" in open(output_definition_filepath, "r").read() # Test success setup again (files present, but staged) options = {"name": "xgboost:cpu"} result = self.environment_controller.setup(options=options) output_definition_filepath = os.path.join( self.environment_controller.file_driver.environment_directory, "Dockerfile") assert isinstance(result, Environment) assert result.name == options['name'] assert result.description == "supported base environment created by datmo" assert os.path.isfile(output_definition_filepath) assert "FROM datmo/xgboost:cpu" in open(output_definition_filepath, "r").read() # Test failure in downstream function (e.g. bad inputs, no name given) failed = False try: self.environment_controller.setup(options={}) except EnvironmentDoesNotExist: failed = True assert failed # Change environment file with open(output_definition_filepath, "wb") as f: f.write(to_bytes("new content")) # Test failure setup (unstaged changes) failed = False try: self.environment_controller.setup(options=options) except UnstagedChanges: failed = True assert failed def test_create(self): # 0) Test SUCCESS create when definition path exists in project environment directory (no input, no root) -- with hardware file # 1) Test SUCCESS create when definition path exists in project environment directory (no input, no root) # 5) Test SUCCESS when definition path exists in project environment directory and passed from input dict (takes input) # 2) Test SUCCESS create when definition path exists in root project folder (no input, no project environment dir) # 3) Test SUCCESS create when definition path is passed into input dict (takes input, no project environment dir) # 4) Test SUCCESS create when definition path is passed into input dict along with expected filename to be saved # 6) Test FAIL when passing same filepath with same filename into input dict self.__setup() input_dict_0 = {"name": "test", "description": "test description"} # 0) Test option 0 (cannot test hash because hardware is machine-dependent) environment_obj_0 = self.environment_controller.create(input_dict_0) assert environment_obj_0 assert isinstance(environment_obj_0, Environment) assert environment_obj_0.id assert environment_obj_0.driver_type == "docker" assert environment_obj_0.file_collection_id assert environment_obj_0.definition_filename assert environment_obj_0.hardware_info assert environment_obj_0.unique_hash assert environment_obj_0.name == "test" assert environment_obj_0.description == "test description" # Get file collection path file_collection_obj = self.environment_controller.dal.file_collection. \ get_by_id(environment_obj_0.file_collection_id) file_collection_dir = self.environment_controller.file_driver. \ get_collection_path(file_collection_obj.filehash) assert os.path.isfile(os.path.join(file_collection_dir, "test")) assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile")) output = open(os.path.join(file_collection_dir, "Dockerfile"), "r").read() print(repr(output)) assert os.path.isfile( os.path.join(file_collection_dir, "datmoDockerfile")) output = open(os.path.join(file_collection_dir, "datmoDockerfile"), "r").read() print(repr(output)) assert os.path.isfile( os.path.join(file_collection_dir, "hardware_info")) output = open(os.path.join(file_collection_dir, "hardware_info"), "r").read() print(repr(output)) # 1) Test option 1 environment_obj_0 = self.environment_controller.create( input_dict_0, save_hardware_file=False) assert environment_obj_0 assert isinstance(environment_obj_0, Environment) assert environment_obj_0.id assert environment_obj_0.driver_type == "docker" assert environment_obj_0.file_collection_id assert environment_obj_0.definition_filename assert environment_obj_0.hardware_info # Get file collection path file_collection_obj = self.environment_controller.dal.file_collection. \ get_by_id(environment_obj_0.file_collection_id) file_collection_dir = self.environment_controller.file_driver. \ get_collection_path(file_collection_obj.filehash) assert os.path.isfile(os.path.join(file_collection_dir, "test")) assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile")) output = open(os.path.join(file_collection_dir, "Dockerfile"), "r").read() print(repr(output)) assert os.path.isfile( os.path.join(file_collection_dir, "datmoDockerfile")) output = open(os.path.join(file_collection_dir, "datmoDockerfile"), "r").read() print(repr(output)) assert environment_obj_0.unique_hash == "c309ae4f58163693a91816988d9dc88b" assert environment_obj_0.name == "test" assert environment_obj_0.description == "test description" # Files ["test", "Dockerfile", "datmoDockerfile"] # 5) Test option 5 input_dict_1 = { "name": "test", "description": "test description", "paths": [self.definition_filepath], } environment_obj = self.environment_controller.create( input_dict_1, save_hardware_file=False) assert environment_obj assert isinstance(environment_obj, Environment) assert environment_obj.id assert environment_obj.driver_type == "docker" assert environment_obj.file_collection_id assert environment_obj.definition_filename assert environment_obj.hardware_info # Get file collection path file_collection_obj = self.environment_controller.dal.file_collection. \ get_by_id(environment_obj.file_collection_id) file_collection_dir = self.environment_controller.file_driver. \ get_collection_path(file_collection_obj.filehash) assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile")) output = open(os.path.join(file_collection_dir, "Dockerfile"), "r").read() print(repr(output)) assert os.path.isfile( os.path.join(file_collection_dir, "datmoDockerfile")) output = open(os.path.join(file_collection_dir, "datmoDockerfile"), "r").read() print(repr(output)) assert environment_obj.unique_hash == "6e06d7c4d77cb6ae69e7e0efa883ef4b" assert environment_obj.name == "test" assert environment_obj.description == "test description" # Files ["Dockerfile", "datmoDockerfile"] # remove the project environment directory shutil.rmtree( self.environment_controller.file_driver.environment_directory) # Create environment definition in root directory home_definition_filepath = os.path.join( self.environment_controller.home, "Dockerfile") with open(home_definition_filepath, "wb") as f: f.write(to_bytes("FROM python:3.5-alpine")) # 2) Test option 2 environment_obj_1 = self.environment_controller.create( input_dict_0, save_hardware_file=False) assert environment_obj_1 assert isinstance(environment_obj_1, Environment) assert environment_obj_1.id assert environment_obj_1.driver_type == "docker" assert environment_obj_1.file_collection_id assert environment_obj_1.definition_filename assert environment_obj_1.hardware_info assert environment_obj_1.unique_hash == file_collection_obj.filehash # Get file collection path file_collection_obj = self.environment_controller.dal.file_collection. \ get_by_id(environment_obj_1.file_collection_id) file_collection_dir = self.environment_controller.file_driver. \ get_collection_path(file_collection_obj.filehash) assert environment_obj_1.name == "test" assert environment_obj_1.description == "test description" assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "datmoDockerfile")) assert environment_obj_1.unique_hash == "6e06d7c4d77cb6ae69e7e0efa883ef4b" # 3) Test option 3 input_dict_2 = { "name": "test", "description": "test description", "paths": [home_definition_filepath], } # Create environment in the project environment_obj_2 = self.environment_controller.create( input_dict_2, save_hardware_file=False) assert environment_obj_2 assert isinstance(environment_obj_2, Environment) assert environment_obj_2.id assert environment_obj_2.driver_type == "docker" assert environment_obj_2.file_collection_id assert environment_obj_2.definition_filename assert environment_obj_2.hardware_info assert environment_obj_2.unique_hash == file_collection_obj.filehash # Get file collection path file_collection_obj = self.environment_controller.dal.file_collection. \ get_by_id(environment_obj_2.file_collection_id) file_collection_dir = self.environment_controller.file_driver. \ get_collection_path(file_collection_obj.filehash) assert environment_obj_2.name == "test" assert environment_obj_2.description == "test description" assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "datmoDockerfile")) assert environment_obj_2.unique_hash == "6e06d7c4d77cb6ae69e7e0efa883ef4b" # 4) Test option 4 input_dict_3 = { "paths": [home_definition_filepath + ">Dockerfile"], } # Create environment in the project environment_obj_3 = self.environment_controller.create( input_dict_3, save_hardware_file=False) assert environment_obj_3 assert isinstance(environment_obj_3, Environment) assert environment_obj_3.id assert environment_obj_3.driver_type == "docker" assert environment_obj_3.file_collection_id assert environment_obj_3.definition_filename assert environment_obj_3.hardware_info assert environment_obj_3.unique_hash == file_collection_obj.filehash # Get file collection path file_collection_obj = self.environment_controller.dal.file_collection. \ get_by_id(environment_obj_3.file_collection_id) file_collection_dir = self.environment_controller.file_driver. \ get_collection_path(file_collection_obj.filehash) assert environment_obj_3.name == "test" assert environment_obj_3.description == "test description" assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "datmoDockerfile")) assert environment_obj_3.unique_hash == "6e06d7c4d77cb6ae69e7e0efa883ef4b" # 6) Test option 6 definition_filepath = os.path.join(self.environment_controller.home, "Dockerfile") input_dict = { "paths": [ definition_filepath + ">Dockerfile", definition_filepath + ">Dockerfile" ], } # Create environment in the project failed = False try: _ = self.environment_controller.create(input_dict, save_hardware_file=False) except FileAlreadyExistsError: failed = True assert failed @pytest_docker_environment_failed_instantiation(test_datmo_dir) def test_build(self): # 1) Test build when no environment given # 2) Test build when definition path exists and given # 3) Test build when NO file exists and definition path exists # 4) Test build when file exists and definition path exists # 5) Test build when file exists but NO definition path exists self.project_controller.init("test5", "test description") self.environment_controller = EnvironmentController() # 1) Test option 1 failed = False try: _ = self.environment_controller.build("does_not_exist") except EnvironmentDoesNotExist: failed = True assert failed # Create environment definition definition_filepath = os.path.join(self.environment_controller.home, "Dockerfile") random_text = str(uuid.uuid1()) with open(definition_filepath, "wb") as f: f.write(to_bytes("FROM python:3.5-alpine" + "\n")) f.write(to_bytes(str("RUN echo " + random_text))) input_dict = { "paths": [definition_filepath], } # 2) Test option 2 # Create environment in the project environment_obj_1 = self.environment_controller.create(input_dict) result = self.environment_controller.build(environment_obj_1.id) assert result # 3) Test option 3 # Create environment in the project environment_obj_2 = self.environment_controller.create({}) result = self.environment_controller.build(environment_obj_2.id) assert result # Create script to test test_filepath = os.path.join(self.environment_controller.home, "script.py") with open(test_filepath, "wb") as f: f.write(to_bytes("import numpy\n")) f.write(to_bytes("import sklearn\n")) f.write(to_bytes("print('hello')\n")) # 4) Test option 4 environment_obj_3 = self.environment_controller.create({}) result = self.environment_controller.build(environment_obj_3.id) assert result # test 2), 3), and 4) will result in the same environment assert environment_obj_1.id == environment_obj_2.id assert environment_obj_2.id == environment_obj_3.id # Test for building dockerfile when there exists not os.remove(definition_filepath) # 5) Test option 5 # Create environment definition in project environment directory definition_filepath = os.path.join( self.environment_controller.file_driver.environment_directory, "Dockerfile") random_text = str(uuid.uuid1()) with open(definition_filepath, "wb") as f: f.write(to_bytes("FROM python:3.5-alpine" + "\n")) f.write(to_bytes(str("RUN echo " + random_text))) environment_obj_4 = self.environment_controller.create({}) result = self.environment_controller.build(environment_obj_4.id) assert result # teardown self.environment_controller.delete(environment_obj_1.id) @pytest_docker_environment_failed_instantiation(test_datmo_dir) def test_run(self): # Test run simple command with simple Dockerfile self.project_controller.init("test5", "test description") self.environment_controller = EnvironmentController() # 0) Test option 0 # Create environment definition in project environment directory definition_filepath = os.path.join( self.environment_controller.file_driver.environment_directory, "Dockerfile") random_text = str(uuid.uuid1()) with open(definition_filepath, "wb") as f: f.write(to_bytes("FROM python:3.5-alpine" + "\n")) f.write(to_bytes(str("RUN echo " + random_text))) random_name = ''.join([ random.choice(string.ascii_letters + string.digits) for _ in range(32) ]) run_options = { "command": ["sh", "-c", "echo yo"], "ports": ["8888:8888"], "name": random_name, "volumes": None, "detach": True, "stdin_open": False, "mem_limit": "4g", "tty": False, "api": False } # Create environment in the project environment_obj = self.environment_controller.create({}) log_filepath = os.path.join(self.project_controller.home, "task.log") # Build environment in the project _ = self.environment_controller.build(environment_obj.id) # Run environment in the project return_code, run_id, logs = \ self.environment_controller.run(environment_obj.id, run_options, log_filepath) assert return_code == 0 assert run_id assert logs # teardown self.environment_controller.delete(environment_obj.id) shutil.rmtree( self.environment_controller.file_driver.environment_directory) # 1) Test option 1 # Create environment definition definition_filepath = os.path.join(self.environment_controller.home, "Dockerfile") random_text = str(uuid.uuid1()) with open(definition_filepath, "wb") as f: f.write(to_bytes("FROM python:3.5-alpine" + "\n")) f.write(to_bytes(str("RUN echo " + random_text))) random_name = ''.join([ random.choice(string.ascii_letters + string.digits) for _ in range(32) ]) run_options = { "command": ["sh", "-c", "echo yo"], "ports": ["8888:8888"], "name": random_name, "volumes": None, "mem_limit": "4g", "detach": True, "stdin_open": False, "tty": False, "api": False } input_dict = { "paths": [definition_filepath], } # Create environment in the project environment_obj = self.environment_controller.create(input_dict) log_filepath = os.path.join(self.project_controller.home, "task.log") # Build environment in the project _ = self.environment_controller.build(environment_obj.id) # Run environment in the project return_code, run_id, logs = \ self.environment_controller.run(environment_obj.id, run_options, log_filepath) assert return_code == 0 assert run_id assert logs # teardown self.environment_controller.delete(environment_obj.id) # 2) Test option 2 os.remove(definition_filepath) # Create script to test test_filepath = os.path.join(self.environment_controller.home, "script.py") with open(test_filepath, "wb") as f: f.write(to_bytes("import os\n")) f.write(to_bytes("import sys\n")) f.write(to_bytes("print('hello')\n")) # Create environment in the project environment_obj = self.environment_controller.create({}) self.environment_controller.build(environment_obj.id) random_name = ''.join([ random.choice(string.ascii_letters + string.digits) for _ in range(32) ]) run_options = { "command": ["sh", "-c", "echo yo"], "ports": ["8888:8888"], "name": random_name, "volumes": { self.environment_controller.home: { 'bind': '/home/', 'mode': 'rw' } }, "mem_limit": "4g", "detach": False, "stdin_open": False, "tty": False, "api": False } # Run environment in the project return_code, run_id, logs = \ self.environment_controller.run(environment_obj.id, run_options, log_filepath) assert return_code == 0 assert run_id assert logs # teardown self.environment_controller.delete(environment_obj.id) @pytest_docker_environment_failed_instantiation(test_datmo_dir) def test_interactive_run(self): # 1) Test run interactive terminal in environment # 2) Test run jupyter notebook in environment # Create environment definition self.project_controller.init("test6", "test description") self.environment_controller = EnvironmentController() definition_filepath = os.path.join(self.environment_controller.home, "Dockerfile") random_text = str(uuid.uuid1()) with open(definition_filepath, "wb") as f: f.write(to_bytes("FROM datmo/xgboost:cpu" + "\n")) f.write(to_bytes(str("RUN echo " + random_text))) input_dict = { "paths": [definition_filepath], } # Create environment in the project environment_obj = self.environment_controller.create(input_dict) # 1) Test option 1 @timeout_decorator.timeout(10, use_signals=False) def timed_run(container_name, timed_run): run_options = { "command": [], "ports": ["8888:8888"], "name": container_name, "volumes": None, "mem_limit": "4g", "detach": True, "stdin_open": True, "tty": True, "api": False } log_filepath = os.path.join(self.project_controller.home, "task.log") # Build environment in the project _ = self.environment_controller.build(environment_obj.id) # Run environment in the project self.environment_controller.run(environment_obj.id, run_options, log_filepath) return timed_run container_name = str(uuid.uuid1()) timed_run_result = False try: timed_run_result = timed_run(container_name, timed_run_result) except timeout_decorator.timeout_decorator.TimeoutError: timed_run_result = True assert timed_run_result # teardown self.environment_controller.delete(environment_obj.id) # 2) Test option 2 environment_obj = self.environment_controller.create(input_dict) @timeout_decorator.timeout(10, use_signals=False) def timed_run(container_name, timed_run): run_options = { "command": ["jupyter", "notebook"], "ports": ["8888:8888"], "name": container_name, "volumes": None, "mem_limit": "4g", "detach": True, "stdin_open": False, "tty": False, "api": False } log_filepath = os.path.join(self.project_controller.home, "task.log") # Build environment in the project _ = self.environment_controller.build(environment_obj.id) # Run environment in the project self.environment_controller.run(environment_obj.id, run_options, log_filepath) return timed_run container_name = str(uuid.uuid1()) timed_run_result = False try: timed_run_result = timed_run(container_name, timed_run_result) except timeout_decorator.timeout_decorator.TimeoutError: timed_run_result = True assert timed_run_result # Stop the running environment # self.environment_controller.stop(container_name) # teardown self.environment_controller.delete(environment_obj.id) def test_list(self): self.project_controller.init("test4", "test description") self.environment_controller = EnvironmentController() # Create environment definition for object 1 definition_path_1 = os.path.join(self.environment_controller.home, "Dockerfile") with open(definition_path_1, "wb") as f: f.write(to_bytes("FROM python:3.5-alpine")) input_dict_1 = { "paths": [definition_path_1], } # Create environment in the project environment_obj_1 = self.environment_controller.create(input_dict_1) # Create environment definition for object 2 definition_path_2 = os.path.join(self.environment_controller.home, "Dockerfile2") with open(definition_path_2, "wb") as f: f.write(to_bytes("FROM python:3.4-alpine")) input_dict_2 = { "paths": [definition_path_2 + ">Dockerfile"], } # Create second environment in the project environment_obj_2 = self.environment_controller.create(input_dict_2) # List all environments and ensure they exist result = self.environment_controller.list() assert len(result) == 2 and \ environment_obj_1 in result and \ environment_obj_2 in result def test_update(self): self.project_controller.init("test5", "test description") self.environment_controller = EnvironmentController() # Create environment definition definition_filepath = os.path.join(self.environment_controller.home, "Dockerfile") with open(definition_filepath, "wb") as f: f.write(to_bytes("FROM python:3.5-alpine")) input_dict = { "paths": [definition_filepath], } # Create environment in the project environment_obj = self.environment_controller.create(input_dict) # Test success update new_name = "test name" new_description = "test description" result = self.environment_controller.update( environment_obj.id, name=new_name, description=new_description) assert result assert isinstance(result, Environment) assert result.name == new_name assert result.description == new_description # Test failed update failed = False try: self.environment_controller.update("random_id", name=new_name, description=new_description) except EnvironmentDoesNotExist: failed = True assert failed @pytest_docker_environment_failed_instantiation(test_datmo_dir) def test_delete(self): self.project_controller.init("test5", "test description") self.environment_controller = EnvironmentController() # Create environment definition definition_filepath = os.path.join(self.environment_controller.home, "Dockerfile") with open(definition_filepath, "wb") as f: f.write(to_bytes("FROM python:3.5-alpine")) input_dict = { "paths": [definition_filepath], } # Create environment in the project environment_obj = self.environment_controller.create(input_dict) # Delete environment in the project result = self.environment_controller.delete(environment_obj.id) # Check if environment retrieval throws error thrown = False try: self.environment_controller.dal.environment.get_by_id( environment_obj.id) except EntityNotFound: thrown = True assert result == True and \ thrown == True @pytest_docker_environment_failed_instantiation(test_datmo_dir) def test_stop_failure(self): # 1) Test failure with RequiredArgumentMissing # 2) Test failure with TooManyArgumentsFound self.project_controller.init("test5", "test description") self.environment_controller = EnvironmentController() # 1) Test option 1 failed = False try: self.environment_controller.stop() except RequiredArgumentMissing: failed = True assert failed # 2) Test option 2 failed = False try: self.environment_controller.stop(run_id="hello", match_string="there") except TooManyArgumentsFound: failed = True assert failed @pytest_docker_environment_failed_instantiation(test_datmo_dir) def test_stop_success(self): # TODO: test more run options # 1) Test run_id input to stop # 2) Test match_string input to stop # 3) Test all input to stop self.project_controller.init("test5", "test description") self.environment_controller = EnvironmentController() # Create environment definition definition_filepath = os.path.join(self.environment_controller.home, "Dockerfile") with open(definition_filepath, "wb") as f: f.write(to_bytes("FROM python:3.5-alpine")) run_options = { "command": ["sh", "-c", "echo yo"], "ports": ["8888:8888"], "name": "datmo-task-" + self.environment_controller.model.id + "-" + "test", "volumes": None, "mem_limit": "4g", "detach": False, "stdin_open": False, "tty": False, "api": False } # Create environment definition env_def_path = os.path.join(self.project_controller.home, "Dockerfile") random_text = str(uuid.uuid1()) with open(env_def_path, "wb") as f: f.write(to_bytes("FROM python:3.5-alpine" + "\n")) f.write(to_bytes(str("RUN echo " + random_text))) input_dict = { "paths": [definition_filepath], } # Create environment in the project environment_obj = self.environment_controller.create(input_dict) log_filepath = os.path.join(self.project_controller.home, "task.log") # Build environment in the project _ = self.environment_controller.build(environment_obj.id) # 1) Test option 1 _, run_id, _ = \ self.environment_controller.run(environment_obj.id, run_options, log_filepath) return_code = self.environment_controller.stop(run_id=run_id) assert return_code # 2) Test option 2 _, _, _ = \ self.environment_controller.run(environment_obj.id, run_options, log_filepath) return_code = self.environment_controller.stop( match_string="datmo-task-" + self.environment_controller.model.id) assert return_code # 3) Test option 3 _, _, _ = \ self.environment_controller.run(environment_obj.id, run_options, log_filepath) run_options_2 = { "command": ["sh", "-c", "echo yo"], "ports": ["8888:8888"], "name": "datmo-task-" + self.environment_controller.model.id + "-" + "test2", "volumes": None, "mem_limit": "4g", "detach": False, "stdin_open": False, "tty": False, "api": False } _, _, _ = \ self.environment_controller.run(environment_obj.id, run_options_2, log_filepath) return_code = self.environment_controller.stop(all=True) assert return_code # teardown self.environment_controller.delete(environment_obj.id) def test_exists_env(self): # Test failure, not initialized failed = False try: _ = self.environment_controller.create({}) except: failed = True assert failed # Setup self.__setup() environment_obj = self.environment_controller.create({}) # Check by environment id result = self.environment_controller.exists( environment_id=environment_obj.id) assert result # Check by unique hash result = self.environment_controller.exists( environment_unique_hash=environment_obj.unique_hash) assert result # Test with wrong environment id result = self.environment_controller.exists( environment_id='test_wrong_env_id') assert not result def test_calculate_project_environment_hash(self): # Setup self.__setup() # Test hashing the default (with hardware info) result = self.environment_controller._calculate_project_environment_hash( ) assert result # Test hashing the default Dockerfile result = self.environment_controller._calculate_project_environment_hash( save_hardware_file=False) assert result == "c309ae4f58163693a91816988d9dc88b" # Test if hash is the same as that of create environment_obj = self.environment_controller.create( {}, save_hardware_file=False) result = self.environment_controller._calculate_project_environment_hash( save_hardware_file=False) assert result == "c309ae4f58163693a91816988d9dc88b" assert result == environment_obj.unique_hash # Test if the hash is the same if the same file is passed in as an input input_dict = { "paths": [self.definition_filepath, self.random_filepath] } environment_obj_1 = self.environment_controller.create( input_dict, save_hardware_file=False) result = self.environment_controller._calculate_project_environment_hash( save_hardware_file=False) assert result == "c309ae4f58163693a91816988d9dc88b" assert result == environment_obj_1.unique_hash def test_has_unstaged_changes(self): # Setup self.__setup() _ = self.environment_controller.create({}) # Check for no unstaged changes result = self.environment_controller._has_unstaged_changes() assert not result # Make a change to the file (update python version) with open( os.path.join( self.environment_controller.file_driver. environment_directory, "Dockerfile"), "wb") as f: f.write(to_bytes("FROM python:3.6-alpine")) # Check again, should have unstaged changes result = self.environment_controller._has_unstaged_changes() assert result def test_check_unstaged_changes(self): # Setup self.__setup() obj = self.environment_controller.create({}) # 1) After commiting the changes # Check for no unstaged changes because already committed result = self.environment_controller.check_unstaged_changes() assert not result # Add a new file with open( os.path.join( self.environment_controller.file_driver. environment_directory, "test2"), "wb") as f: f.write(to_bytes("cool")) # 2) Not commiting the changes, should error and raise UnstagedChanges failed = False try: self.environment_controller.check_unstaged_changes() except UnstagedChanges: failed = True assert failed # Remove new file os.remove( os.path.join( self.environment_controller.file_driver.environment_directory, "test2")) # 3) Files are the same as before but no new commit, should have no unstaged changes result = self.environment_controller.check_unstaged_changes() assert not result # 4) Remove another file, now it is different and should have unstaged changes os.remove( os.path.join( self.environment_controller.file_driver.environment_directory, "test")) failed = False try: self.environment_controller.check_unstaged_changes() except UnstagedChanges: failed = True assert failed # 5) Remove the rest of the files, now it is empty and should return as already staged os.remove( os.path.join( self.environment_controller.file_driver.environment_directory, "Dockerfile")) result = self.environment_controller.check_unstaged_changes() assert not result def test_checkout(self): # Setup and create all environment files self.__setup() # Create environment to checkout to with defaults environment_obj = self.environment_controller.create({}) # Checkout success with there are no unstaged changes result = self.environment_controller.checkout(environment_obj.id) assert result current_hash = self.environment_controller._calculate_project_environment_hash( ) assert environment_obj.unique_hash == current_hash # Check the filenames as well because the hash does not take this into account assert os.path.isfile( os.path.join( self.environment_controller.file_driver.environment_directory, "test")) assert os.path.isfile( os.path.join( self.environment_controller.file_driver.environment_directory, "Dockerfile")) assert not os.path.isfile( os.path.join( self.environment_controller.file_driver.environment_directory, "datmoDockerfile")) assert not os.path.isfile( os.path.join( self.environment_controller.file_driver.environment_directory, "hardware_info")) # Change file contents to make it unstaged with open(self.definition_filepath, "wb") as f: f.write(to_bytes("new content")) # Checkout failure with unstaged changes failed = False try: _ = self.environment_controller.checkout(environment_obj.id) except UnstagedChanges: failed = True assert failed # Create new environment to checkout to with defaults (no hardware) environment_obj_1 = self.environment_controller.create( {}, save_hardware_file=False) # Checkout success with there are no unstaged changes result = self.environment_controller.checkout(environment_obj.id) assert result current_hash = self.environment_controller._calculate_project_environment_hash( ) assert environment_obj.unique_hash == current_hash assert environment_obj_1.unique_hash != current_hash # Check the filenames as well because the hash does not take this into account assert os.path.isfile( os.path.join( self.environment_controller.file_driver.environment_directory, "test")) assert os.path.isfile( os.path.join( self.environment_controller.file_driver.environment_directory, "Dockerfile")) assert not os.path.isfile( os.path.join( self.environment_controller.file_driver.environment_directory, "datmoDockerfile")) assert not os.path.isfile( os.path.join( self.environment_controller.file_driver.environment_directory, "hardware_info"))
class TestEnvironmentController(): def setup_method(self): # provide mountable tmp directory for docker tempfile.tempdir = "/tmp" if not platform.system( ) == "Windows" else None test_datmo_dir = os.environ.get('TEST_DATMO_DIR', tempfile.gettempdir()) self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir) self.project = ProjectController(self.temp_dir) self.environment = EnvironmentController(self.temp_dir) def teardown_method(self): pass def test_create(self): # 0) Test create when unsupported language given # 1) Test create when NO file exists and NO definition path exists # 2) Test create when NO file exists and definition path exists # 3) Test create when definition path exists and given # 4) Test create when file exists and definition path exists # 5) Test create when file exists but NO definition path exists # 6) Test create when definition path exists and given for NEW definition filepath self.project.init("test3", "test description") # 0) Test option 0 try: self.environment.create({"language": "java"}) except EnvironmentDoesNotExist: failed = True assert failed # 1) Test option 1 # Creates environment with python3 based docker image environment_obj_0 = self.environment.create({}) assert environment_obj_0 assert environment_obj_0.id assert environment_obj_0.driver_type == "docker" assert environment_obj_0.file_collection_id assert environment_obj_0.definition_filename assert environment_obj_0.hardware_info # Create environment definition definition_filepath = os.path.join(self.environment.home, "Dockerfile") with open(definition_filepath, "w") as f: f.write(to_unicode(str("FROM datmo/xgboost:cpu"))) # 2) Test option 2 environment_obj_1 = self.environment.create({}) # Get file collection path file_collection_obj = self.environment.dal.file_collection. \ get_by_id(environment_obj_1.file_collection_id) file_collection_dir = self.environment.file_driver. \ get_collection_path(file_collection_obj.filehash) assert environment_obj_1 assert environment_obj_1.id assert environment_obj_1.driver_type == "docker" assert environment_obj_1.file_collection_id assert environment_obj_1.definition_filename assert environment_obj_1.hardware_info assert environment_obj_1.unique_hash == file_collection_obj.filehash assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "datmoDockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "hardware_info")) # 3) Test option 3 input_dict = { "definition_filepath": definition_filepath, } # Create environment in the project environment_obj_2 = self.environment.create(input_dict) # Get file collection path file_collection_obj = self.environment.dal.file_collection. \ get_by_id(environment_obj_2.file_collection_id) file_collection_dir = self.environment.file_driver. \ get_collection_path(file_collection_obj.filehash) assert environment_obj_2 assert environment_obj_2.id assert environment_obj_2.driver_type == "docker" assert environment_obj_2.file_collection_id assert environment_obj_2.definition_filename assert environment_obj_2.hardware_info assert environment_obj_2.unique_hash == file_collection_obj.filehash assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "datmoDockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "hardware_info")) # Create script to test test_filepath = os.path.join(self.environment.home, "script.py") with open(test_filepath, "w") as f: f.write(to_unicode("import numpy\n")) f.write(to_unicode("import sklearn\n")) f.write(to_unicode("print('hello')\n")) # 4) Test option 4 environment_obj_3 = self.environment.create({}) # Get file collection path file_collection_obj = self.environment.dal.file_collection. \ get_by_id(environment_obj_3.file_collection_id) file_collection_dir = self.environment.file_driver. \ get_collection_path(file_collection_obj.filehash) assert environment_obj_3 assert environment_obj_3.id assert environment_obj_3.driver_type == "docker" assert environment_obj_3.file_collection_id assert environment_obj_3.definition_filename assert environment_obj_3.hardware_info assert environment_obj_3.unique_hash == file_collection_obj.filehash assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "datmoDockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "hardware_info")) # Remove definition filepath os.remove(definition_filepath) assert environment_obj_1.id == environment_obj_2.id assert environment_obj_2.id == environment_obj_3.id # 5) Test option 5 environment_obj_4 = self.environment.create({}) # Get file collection path file_collection_obj = self.environment.dal.file_collection. \ get_by_id(environment_obj_4.file_collection_id) file_collection_dir = self.environment.file_driver. \ get_collection_path(file_collection_obj.filehash) assert environment_obj_4 assert environment_obj_4.id assert environment_obj_4.driver_type == "docker" assert environment_obj_4.file_collection_id assert environment_obj_4.definition_filename assert environment_obj_4.hardware_info assert environment_obj_4.unique_hash == file_collection_obj.filehash assert os.path.isfile( os.path.join(file_collection_dir, "datmorequirements.txt")) assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "datmoDockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "hardware_info")) assert environment_obj_1.id != environment_obj_4.id # 6) Test option 6 # Create environment definition definition_filepath = os.path.join(self.environment.home, "Dockerfile") with open(definition_filepath, "w") as f: f.write(to_unicode(str("FROM cloudgear/ubuntu:14.04"))) input_dict = { "definition_filepath": definition_filepath, } # Create a new environment obj environment_obj_5 = self.environment.create(input_dict) # Get file collection path file_collection_obj = self.environment.dal.file_collection. \ get_by_id(environment_obj_5.file_collection_id) file_collection_dir = self.environment.file_driver. \ get_collection_path(file_collection_obj.filehash) assert environment_obj_5 assert environment_obj_5.id assert environment_obj_5.driver_type == "docker" assert environment_obj_5.file_collection_id assert environment_obj_5.definition_filename assert environment_obj_5.hardware_info assert environment_obj_5.unique_hash == file_collection_obj.filehash assert os.path.isfile(os.path.join(file_collection_dir, "Dockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "datmoDockerfile")) assert os.path.isfile( os.path.join(file_collection_dir, "hardware_info")) assert environment_obj_5.id != environment_obj_1.id assert environment_obj_5.id != environment_obj_4.id def test_build(self): # 1) Test build when no environment given # 2) Test build when definition path exists and given # 3) Test build when NO file exists and definition path exists # 4) Test build when file exists and definition path exists # 5) Test build when file exists but NO definition path exists self.project.init("test5", "test description") # 1) Test option 1 failed = False try: _ = self.environment.build("does_not_exist") except EntityNotFound: failed = True assert failed # Create environment definition definition_filepath = os.path.join(self.environment.home, "Dockerfile") with open(definition_filepath, "w") as f: f.write(to_unicode(str("FROM datmo/xgboost:cpu"))) input_dict = { "definition_filepath": definition_filepath, } # 2) Test option 2 # Create environment in the project environment_obj_1 = self.environment.create(input_dict) result = self.environment.build(environment_obj_1.id) assert result # 3) Test option 3 # Create environment in the project environment_obj_2 = self.environment.create({}) result = self.environment.build(environment_obj_2.id) assert result # Create script to test test_filepath = os.path.join(self.environment.home, "script.py") with open(test_filepath, "w") as f: f.write(to_unicode("import numpy\n")) f.write(to_unicode("import sklearn\n")) f.write(to_unicode("print('hello')\n")) # 4) Test option 4 environment_obj_3 = self.environment.create({}) result = self.environment.build(environment_obj_3.id) assert result # test 2), 3), and 4) will result in the same environment assert environment_obj_1.id == environment_obj_2.id assert environment_obj_2.id == environment_obj_3.id # Test for building dockerfile when there exists not os.remove(definition_filepath) # 5) Test option 5 environment_obj_4 = self.environment.create({}) result = self.environment.build(environment_obj_4.id) # 2) Test run script, with autogenerated definition assert result assert environment_obj_4.id != environment_obj_1.id # teardown self.environment.delete(environment_obj_1.id) self.environment.delete(environment_obj_4.id) def test_run(self): # 1) Test run simple command with simple Dockerfile self.project.init("test5", "test description") # 1) Test option 1 # Create environment definition definition_filepath = os.path.join(self.environment.home, "Dockerfile") with open(definition_filepath, "w") as f: f.write(to_unicode(str("FROM datmo/xgboost:cpu"))) random_name = ''.join([ random.choice(string.ascii_letters + string.digits) for _ in range(32) ]) run_options = { "command": ["sh", "-c", "echo yo"], "ports": ["8888:8888"], "name": random_name, "volumes": None, "detach": True, "stdin_open": False, "tty": False, "api": False } input_dict = { "definition_filepath": definition_filepath, } # Create environment in the project environment_obj = self.environment.create(input_dict) log_filepath = os.path.join(self.project.home, "task.log") # Build environment in the project _ = self.environment.build(environment_obj.id) # Run environment in the project return_code, run_id, logs = \ self.environment.run(environment_obj.id, run_options, log_filepath) assert return_code == 0 assert run_id assert logs # teardown self.environment.delete(environment_obj.id) # 2) Test option 2 os.remove(definition_filepath) # Create script to test test_filepath = os.path.join(self.environment.home, "script.py") with open(test_filepath, "w") as f: f.write(to_unicode("import numpy\n")) f.write(to_unicode("import sklearn\n")) f.write(to_unicode("print('hello')\n")) # Create environment in the project environment_obj = self.environment.create({}) self.environment.build(environment_obj.id) random_name = ''.join([ random.choice(string.ascii_letters + string.digits) for _ in range(32) ]) run_options = { "command": ["python", "script.py"], "ports": ["8888:8888"], "name": random_name, "volumes": { self.environment.home: { 'bind': '/home/', 'mode': 'rw' } }, "detach": False, "stdin_open": False, "tty": False, "api": False } # Run environment in the project return_code, run_id, logs = \ self.environment.run(environment_obj.id, run_options, log_filepath) assert return_code == 0 assert run_id assert logs # teardown self.environment.delete(environment_obj.id) def test_interactive_run(self): # 1) Test run interactive terminal in environment # 2) Test run jupyter notebook in environment # Create environment definition self.project.init("test6", "test description") definition_filepath = os.path.join(self.environment.home, "Dockerfile") with open(definition_filepath, "w") as f: f.write(to_unicode(str("FROM datmo/xgboost:cpu"))) input_dict = { "definition_filepath": definition_filepath, } # Create environment in the project environment_obj = self.environment.create(input_dict) # 1) Test option 1 @timeout_decorator.timeout(10, use_signals=False) def timed_run(container_name, timed_run): run_options = { "command": [], "ports": ["8888:8888"], "name": container_name, "volumes": None, "detach": True, "stdin_open": True, "tty": True, "api": False } log_filepath = os.path.join(self.project.home, "task.log") # Build environment in the project _ = self.environment.build(environment_obj.id) # Run environment in the project self.environment.run(environment_obj.id, run_options, log_filepath) return timed_run container_name = str(uuid.uuid1()) timed_run_result = False try: timed_run_result = timed_run(container_name, timed_run_result) except timeout_decorator.timeout_decorator.TimeoutError: timed_run_result = True assert timed_run_result # teardown self.environment.delete(environment_obj.id) # 2) Test option 2 environment_obj = self.environment.create(input_dict) @timeout_decorator.timeout(10, use_signals=False) def timed_run(container_name, timed_run): run_options = { "command": ["jupyter", "notebook"], "ports": ["8888:8888"], "name": container_name, "volumes": None, "detach": True, "stdin_open": False, "tty": False, "api": False } log_filepath = os.path.join(self.project.home, "task.log") # Build environment in the project _ = self.environment.build(environment_obj.id) # Run environment in the project self.environment.run(environment_obj.id, run_options, log_filepath) return timed_run container_name = str(uuid.uuid1()) timed_run_result = False try: timed_run_result = timed_run(container_name, timed_run_result) except timeout_decorator.timeout_decorator.TimeoutError: timed_run_result = True assert timed_run_result # Stop the running environment # self.environment.stop(container_name) # teardown self.environment.delete(environment_obj.id) def test_list(self): self.project.init("test4", "test description") # Create environment definition for object 1 definition_path_1 = os.path.join(self.environment.home, "Dockerfile") with open(definition_path_1, "w") as f: f.write(to_unicode(str("FROM datmo/xgboost:cpu"))) input_dict_1 = { "definition_filepath": definition_path_1, } # Create environment in the project environment_obj_1 = self.environment.create(input_dict_1) # Create environment definition for object 2 definition_path_2 = os.path.join(self.environment.home, "Dockerfile2") with open(definition_path_2, "w") as f: f.write(to_unicode(str("FROM datmo/scikit-opencv"))) input_dict_2 = { "definition_filepath": definition_path_2, } # Create second environment in the project environment_obj_2 = self.environment.create(input_dict_2) # List all environments and ensure they exist result = self.environment.list() assert len(result) == 2 and \ environment_obj_1 in result and \ environment_obj_2 in result def test_delete(self): self.project.init("test5", "test description") # Create environment definition definition_filepath = os.path.join(self.environment.home, "Dockerfile") with open(definition_filepath, "w") as f: f.write(to_unicode(str("FROM datmo/xgboost:cpu"))) input_dict = { "definition_filepath": definition_filepath, } # Create environment in the project environment_obj = self.environment.create(input_dict) # Delete environment in the project result = self.environment.delete(environment_obj.id) # Check if environment retrieval throws error thrown = False try: self.environment.dal.environment.get_by_id(environment_obj.id) except EntityNotFound: thrown = True assert result == True and \ thrown == True def test_stop_failure(self): # 1) Test failure with RequiredArgumentMissing # 2) Test failure with TooManyArgumentsFound # 1) Test option 1 failed = False try: self.environment.stop() except RequiredArgumentMissing: failed = True assert failed # 2) Test option 2 failed = False try: self.environment.stop(run_id="hello", match_string="there") except TooManyArgumentsFound: failed = True assert failed def test_stop_success(self): # TODO: test more run options # 1) Test run_id input to stop # 2) Test match_string input to stop # 3) Test all input to stop self.project.init("test5", "test description") # Create environment definition definition_filepath = os.path.join(self.environment.home, "Dockerfile") with open(definition_filepath, "w") as f: f.write(to_unicode(str("FROM datmo/xgboost:cpu"))) run_options = { "command": ["sh", "-c", "echo yo"], "ports": ["8888:8888"], "name": "datmo-task-" + self.environment.model.id + "-" + "test", "volumes": None, "detach": False, "stdin_open": False, "tty": False, "api": False } # Create environment definition env_def_path = os.path.join(self.project.home, "Dockerfile") with open(env_def_path, "w") as f: f.write(to_unicode(str("FROM datmo/xgboost:cpu"))) input_dict = { "definition_filepath": definition_filepath, } # Create environment in the project environment_obj = self.environment.create(input_dict) log_filepath = os.path.join(self.project.home, "task.log") # Build environment in the project _ = self.environment.build(environment_obj.id) # 1) Test option 1 _, run_id, _ = \ self.environment.run(environment_obj.id, run_options, log_filepath) return_code = self.environment.stop(run_id=run_id) assert return_code # 2) Test option 2 _, _, _ = \ self.environment.run(environment_obj.id, run_options, log_filepath) return_code = self.environment.stop(match_string="datmo-task-" + self.environment.model.id) assert return_code # 3) Test option 3 _, _, _ = \ self.environment.run(environment_obj.id, run_options, log_filepath) run_options_2 = { "command": ["sh", "-c", "echo yo"], "ports": ["8888:8888"], "name": "datmo-task-" + self.environment.model.id + "-" + "test2", "volumes": None, "detach": False, "stdin_open": False, "tty": False, "api": False } _, _, _ = \ self.environment.run(environment_obj.id, run_options_2, log_filepath) return_code = self.environment.stop(all=True) assert return_code # teardown self.environment.delete(environment_obj.id)