示例#1
0
class TestSnapshotCommand():
    def setup_method(self):
        self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
        Config().set_home(self.temp_dir)
        self.cli_helper = Helper()

    def teardown_method(self):
        pass

    def __set_variables(self):
        self.project_command = ProjectCommand(self.cli_helper)
        self.project_command.parse(
            ["init", "--name", "foobar", "--description", "test model"])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        dummy(self)
        self.snapshot_command = SnapshotCommand(self.cli_helper)

        # Create environment_driver definition
        self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
        with open(self.env_def_path, "wb") as f:
            f.write(to_bytes("FROM python:3.5-alpine"))

        # Create config file
        self.config_filepath = os.path.join(self.snapshot_command.home,
                                            "config.json")
        with open(self.config_filepath, "wb") as f:
            f.write(to_bytes(str("{}")))

        # Create stats file
        self.stats_filepath = os.path.join(self.snapshot_command.home,
                                           "stats.json")
        with open(self.stats_filepath, "wb") as f:
            f.write(to_bytes(str("{}")))

        # Create test file
        self.filepath = os.path.join(self.snapshot_command.home, "file.txt")
        with open(self.filepath, "wb") as f:
            f.write(to_bytes(str("test")))

        # Create another test file
        self.filepath_2 = os.path.join(self.snapshot_command.home, "file2.txt")
        with open(self.filepath_2, "wb") as f:
            f.write(to_bytes(str("test")))

        # Create config
        self.config = 'foo:bar'
        self.config1 = "{'foo1':'bar1'}"
        self.config2 = "this is test config blob"

        # Create stats
        self.stats = 'foo:bar'
        self.stats1 = "{'foo1':'bar1'}"
        self.stats2 = "this is test stats blob"

    def test_snapshot_help(self):
        self.__set_variables()
        print(
            "\n====================================== datmo snapshot ==========================\n"
        )

        self.snapshot_command.parse(["snapshot"])
        assert self.snapshot_command.execute()

        print(
            "\n====================================== datmo snapshot --help ==========================\n"
        )

        self.snapshot_command.parse(["snapshot", "--help"])
        assert self.snapshot_command.execute()

        print(
            "\n====================================== datmo snapshot create --help ==========================\n"
        )

        self.snapshot_command.parse(["snapshot", "create", "--help"])
        assert self.snapshot_command.execute()

    def test_snapshot_create(self):
        self.__set_variables()
        test_message = "this is a test message"
        test_label = "test label"
        test_run_id = "test_run_id"
        test_environment_definition_filepath = self.env_def_path
        test_config_filepath = self.config_filepath
        test_stats_filepath = self.config_filepath
        test_paths = [self.filepath, self.filepath_2]

        # try single filepath
        self.snapshot_command.parse([
            "snapshot", "create", "--message", test_message, "--run-id",
            test_run_id
        ])

        # testing for proper parsing
        assert self.snapshot_command.args.message == test_message
        assert self.snapshot_command.args.run_id == test_run_id

        # try single filepath
        self.snapshot_command.parse([
            "snapshot",
            "create",
            "--message",
            test_message,
            "--label",
            test_label,
            "--environment-paths",
            test_environment_definition_filepath,
            "--config-filepath",
            test_config_filepath,
            "--stats-filepath",
            test_stats_filepath,
            "--paths",
            test_paths[0],
        ])

        # test for desired side effects
        assert self.snapshot_command.args.message == test_message
        assert self.snapshot_command.args.label == test_label
        assert self.snapshot_command.args.environment_paths == [
            test_environment_definition_filepath
        ]
        assert self.snapshot_command.args.config_filepath == test_config_filepath
        assert self.snapshot_command.args.stats_filepath == test_stats_filepath
        assert self.snapshot_command.args.paths == [test_paths[0]]

        # test multiple paths
        self.snapshot_command.parse([
            "snapshot", "create", "--message", test_message, "--label",
            test_label, "--environment-paths",
            test_environment_definition_filepath, "--config-filepath",
            test_config_filepath, "--stats-filepath", test_stats_filepath,
            "--paths", test_paths[0], "--paths", test_paths[1]
        ])

        # test for desired side effects
        assert self.snapshot_command.args.message == test_message
        assert self.snapshot_command.args.label == test_label
        assert self.snapshot_command.args.environment_paths == [
            test_environment_definition_filepath
        ]
        assert self.snapshot_command.args.config_filepath == test_config_filepath
        assert self.snapshot_command.args.stats_filepath == test_stats_filepath
        assert self.snapshot_command.args.paths == test_paths

        snapshot_obj_1 = self.snapshot_command.execute()
        assert snapshot_obj_1

    def test_snapshot_create_config_stats(self):
        self.__set_variables()
        test_message = "this is a test message"
        test_label = "test label"
        test_config = self.config
        test_stats = self.stats

        # try config
        self.snapshot_command.parse([
            "snapshot", "create", "--message", test_message, "--label",
            test_label, "--config", test_config, "--stats", test_stats
        ])

        # test for desired side effects
        snapshot_obj = self.snapshot_command.execute()
        assert snapshot_obj

        test_config = self.config1
        test_stats = self.stats1

        # try config
        self.snapshot_command.parse([
            "snapshot", "create", "--message", test_message, "--label",
            test_label, "--config", test_config, "--stats", test_stats
        ])

        # test for desired side effects
        snapshot_obj = self.snapshot_command.execute()
        assert snapshot_obj

        test_config = self.config2
        test_stats = self.stats2

        # try config
        self.snapshot_command.parse([
            "snapshot", "create", "--message", test_message, "--label",
            test_label, "--config", test_config, "--stats", test_stats
        ])

        # test for desired side effects
        snapshot_obj = self.snapshot_command.execute()
        assert snapshot_obj

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_snapshot_create_from_run(self):
        self.__set_variables()
        test_message = "this is a test message"

        # create task
        test_command = "sh -c 'echo accuracy:0.45'"
        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
        self.run = RunCommand(self.cli_helper)
        self.run.parse(
            ["run", "--environment-paths", test_dockerfile, test_command])

        # test proper execution of task run command
        run_obj = self.run.execute()

        run_id = run_obj.id

        # test run id
        self.snapshot_command.parse([
            "snapshot", "create", "--message", test_message, "--run-id", run_id
        ])

        # test for desired side effects
        assert self.snapshot_command.args.message == test_message

        snapshot_obj = self.snapshot_command.execute()
        assert snapshot_obj

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_snapshot_create_from_run_fail_user_inputs(self):
        self.__set_variables()
        test_message = "this is a test message"

        # create run
        test_command = "sh -c 'echo accuracy:0.45'"
        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
        self.run = RunCommand(self.cli_helper)
        self.run.parse(
            ["run", "--environment-paths", test_dockerfile, test_command])

        # test proper execution of task run command
        run_obj = self.run.execute()

        run_id = run_obj.id

        failed = False
        try:
            # test run id with environment-id
            self.snapshot_command.parse([
                "snapshot", "create", "--message", test_message, "--run-id",
                run_id, "--environment-id", "test_environment_id"
            ])
            _ = self.snapshot_command.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test run id with environment-def
            self.snapshot_command.parse([
                "snapshot", "create", "--message", test_message, "--run-id",
                run_id, "--environment-paths", "test_environment_path"
            ])
            _ = self.snapshot_command.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test run id with filepaths
            self.snapshot_command.parse([
                "snapshot", "create", "--message", test_message, "--run-id",
                run_id, "--paths", "mypath"
            ])
            _ = self.snapshot_command.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test run id with config-filepath
            self.snapshot_command.parse([
                "snapshot", "create", "--message", test_message, "--run-id",
                run_id, "--config-filepath", "mypath"
            ])
            _ = self.snapshot_command.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test run id with config-filename
            self.snapshot_command.parse([
                "snapshot", "create", "--message", test_message, "--run-id",
                run_id, "--config-filename", "myname"
            ])
            _ = self.snapshot_command.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test run id with stats-filepath
            self.snapshot_command.parse([
                "snapshot", "create", "--message", test_message, "--run-id",
                run_id, "--stats-filepath", "mypath"
            ])
            _ = self.snapshot_command.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test run id with stats-filename
            self.snapshot_command.parse([
                "snapshot", "create", "--message", test_message, "--run-id",
                run_id, "--stats-filename", "myname"
            ])
            _ = self.snapshot_command.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

    def test_snapshot_create_fail_mutually_exclusive_args(self):
        self.__set_variables()
        test_message = "this is a test message"
        test_label = "test label"
        test_environment_id = "test_environment_id"
        test_environment_definition_filepath = self.env_def_path
        test_config_filename = "config.json"
        test_config_filepath = self.config_filepath
        test_stats_filename = "stats.json"
        test_stats_filepath = self.config_filepath

        # Environment exception
        exception_thrown = False
        try:
            self.snapshot_command.parse([
                "snapshot",
                "create",
                "--message",
                test_message,
                "--label",
                test_label,
                "--environment-id",
                test_environment_id,
                "--environment-paths",
                test_environment_definition_filepath,
            ])
            _ = self.snapshot_command.execute()
        except MutuallyExclusiveArguments:
            exception_thrown = True
        assert exception_thrown

        # Config exception
        exception_thrown = False
        try:
            self.snapshot_command.parse([
                "snapshot",
                "create",
                "--message",
                test_message,
                "--label",
                test_label,
                "--config-filename",
                test_config_filename,
                "--config-filepath",
                test_config_filepath,
            ])
            _ = self.snapshot_command.execute()
        except MutuallyExclusiveArguments:
            exception_thrown = True
        assert exception_thrown

        # Stats exception
        exception_thrown = False
        try:
            self.snapshot_command.parse([
                "snapshot",
                "create",
                "--message",
                test_message,
                "--label",
                test_label,
                "--stats-filename",
                test_stats_filename,
                "--stats-filepath",
                test_stats_filepath,
            ])
            _ = self.snapshot_command.execute()
        except MutuallyExclusiveArguments:
            exception_thrown = True
        assert exception_thrown

    def test_snapshot_create_default(self):
        self.__set_variables()
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])

        snapshot_obj_2 = self.snapshot_command.execute()
        assert snapshot_obj_2

    def test_snapshot_create_invalid_arg(self):
        self.__set_variables()
        exception_thrown = False
        try:
            self.snapshot_command.parse(
                ["snapshot", "create"
                 "--foobar", "foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    def test_snapshot_update(self):
        self.__set_variables()

        test_config = ["depth: 10", "learning_rate:0.91"]
        test_stats = ["acc: 91.34", "f1_score:0.91"]
        test_config1 = "{'foo_config': 'bar_config'}"
        test_stats1 = "{'foo_stats': 'bar_stats'}"
        test_config2 = "this is a config blob"
        test_stats2 = "this is a stats blob"
        test_message = "test_message"
        test_label = "test_label"

        # 1. Updating both message and label
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])
        snapshot_obj = self.snapshot_command.execute()

        # Test when optional parameters are not given
        self.snapshot_command.parse([
            "snapshot", "update", snapshot_obj.id, "--message", test_message,
            "--label", test_label
        ])

        result = self.snapshot_command.execute()
        assert result.id == snapshot_obj.id
        assert result.message == test_message
        assert result.label == test_label

        # 2. Updating only message
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])
        snapshot_obj = self.snapshot_command.execute()

        # Test when optional parameters are not given
        self.snapshot_command.parse(
            ["snapshot", "update", snapshot_obj.id, "--message", test_message])

        result = self.snapshot_command.execute()
        assert result.id == snapshot_obj.id
        assert result.message == test_message

        # Updating label
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])
        snapshot_obj = self.snapshot_command.execute()

        # Test when optional parameters are not given
        self.snapshot_command.parse(
            ["snapshot", "update", snapshot_obj.id, "--label", test_label])

        result = self.snapshot_command.execute()
        assert result.id == snapshot_obj.id
        assert result.label == test_label

        # 3. Updating config, message and label
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])
        snapshot_obj = self.snapshot_command.execute()

        # Test when optional parameters are not given
        self.snapshot_command.parse([
            "snapshot", "update", snapshot_obj.id, "--config", test_config[0],
            "--config", test_config[1], "--message", test_message, "--label",
            test_label
        ])

        result = self.snapshot_command.execute()
        assert result.id == snapshot_obj.id
        assert result.config == {"depth": "10", "learning_rate": "0.91"}
        assert result.message == test_message
        assert result.label == test_label

        # 4. Updating stats, message and label
        # Test when optional parameters are not given
        self.snapshot_command.parse([
            "snapshot", "update", snapshot_obj.id, "--stats", test_stats[0],
            "--stats", test_stats[1], "--message", test_message, "--label",
            test_label
        ])

        result = self.snapshot_command.execute()
        assert result.id == snapshot_obj.id
        assert result.stats == {"acc": "91.34", "f1_score": "0.91"}
        assert result.message == test_message
        assert result.label == test_label

        # Adding sleep due to issue with consistency in blitzdb database
        time.sleep(2)

        # 5. Updating config, stats
        # Test when optional parameters are not given
        self.snapshot_command.parse([
            "snapshot", "update", snapshot_obj.id, "--config", test_config1,
            "--stats", test_stats1
        ])

        result = self.snapshot_command.execute()
        assert result.id == snapshot_obj.id
        assert result.config == {
            "depth": "10",
            "learning_rate": "0.91",
            'foo_config': 'bar_config'
        }
        assert result.stats == {
            "acc": "91.34",
            "f1_score": "0.91",
            'foo_stats': 'bar_stats'
        }
        assert result.message == test_message
        assert result.label == test_label

        # Test when optional parameters are not given
        self.snapshot_command.parse([
            "snapshot", "update", snapshot_obj.id, "--config", test_config2,
            "--stats", test_stats2
        ])

        result = self.snapshot_command.execute()
        assert result.id == snapshot_obj.id
        assert result.config == {
            "depth": "10",
            "learning_rate": "0.91",
            'foo_config': 'bar_config',
            'config': test_config2
        }
        assert result.stats == {
            "acc": "91.34",
            "f1_score": "0.91",
            'foo_stats': 'bar_stats',
            'stats': test_stats2
        }
        assert result.message == test_message
        assert result.label == test_label

    def test_snapshot_delete(self):
        self.__set_variables()

        # Test when optional parameters are not given
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])

        snapshot_obj = self.snapshot_command.execute()

        # Test when optional parameters are not given
        self.snapshot_command.parse(["snapshot", "delete", snapshot_obj.id])

        result = self.snapshot_command.execute()
        assert result

    def test_snapshot_delete_invalid_arg(self):
        self.__set_variables()
        exception_thrown = False
        try:
            self.snapshot_command.parse(
                ["snapshot", "delete"
                 "--foobar", "foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_snapshot_ls_invisible(self):
        self.__set_variables()

        # Test invisible snapshot from task
        # create task
        test_command = "sh -c 'echo accuracy:0.45'"
        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
        self.run = RunCommand(self.cli_helper)
        self.run.parse(
            ["run", "--environment-paths", test_dockerfile, test_command])

        # test proper execution of task run command
        task_obj = self.run.execute()

        # test no visible snapshots
        self.snapshot_command.parse(["snapshot", "ls"])

        result = self.snapshot_command.execute()

        assert not result

        # test invisible snapshot from task_id was created
        self.snapshot_command.parse(["snapshot", "ls", "--all"])

        result = self.snapshot_command.execute()

        assert result
        assert task_obj.after_snapshot_id in [obj.id for obj in result]

    def test_snapshot_ls(self):
        self.__set_variables()

        # create a visible snapshot
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])

        created_snapshot_obj = self.snapshot_command.execute()

        # Test when optional parameters are not given
        self.snapshot_command.parse(["snapshot", "ls"])

        result = self.snapshot_command.execute()

        assert result
        assert created_snapshot_obj in result
        # Check if current snapshot id is the same as the latest created id
        current_snapshot_obj = self.snapshot_command.snapshot_controller.current_snapshot(
        )
        current_snapshot_id = current_snapshot_obj.id if current_snapshot_obj else None
        assert current_snapshot_id == created_snapshot_obj.id

        # Test when optional parameters are not given
        self.snapshot_command.parse(["snapshot", "ls", "--details"])

        result = self.snapshot_command.execute()

        assert result
        assert created_snapshot_obj in result

        # Test failure (format)
        failed = False
        try:
            self.snapshot_command.parse(["snapshot", "ls", "--format"])
        except ArgumentError:
            failed = True
        assert failed

        # Test success format csv
        self.snapshot_command.parse(["snapshot", "ls", "--format", "csv"])
        snapshot_objs = self.snapshot_command.execute()
        assert created_snapshot_obj in snapshot_objs

        # Test success format csv, download default
        self.snapshot_command.parse(
            ["snapshot", "ls", "--format", "csv", "--download"])
        snapshot_objs = self.snapshot_command.execute()
        assert created_snapshot_obj in snapshot_objs
        test_wildcard = os.path.join(
            self.snapshot_command.snapshot_controller.home, "snapshot_ls_*")
        paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
        assert paths
        assert open(paths[0], "r").read()
        os.remove(paths[0])

        # Test success format csv, download exact path
        test_path = os.path.join(self.temp_dir, "my_output")
        self.snapshot_command.parse([
            "snapshot", "ls", "--format", "csv", "--download",
            "--download-path", test_path
        ])
        snapshot_objs = self.snapshot_command.execute()
        assert created_snapshot_obj in snapshot_objs
        assert os.path.isfile(test_path)
        assert open(test_path, "r").read()
        os.remove(test_path)

        # Test success format table
        self.snapshot_command.parse(["snapshot", "ls"])
        environment_objs = self.snapshot_command.execute()
        assert created_snapshot_obj in snapshot_objs

        # Test success format table, download default
        self.snapshot_command.parse(["snapshot", "ls", "--download"])
        snapshot_objs = self.snapshot_command.execute()
        assert created_snapshot_obj in snapshot_objs
        test_wildcard = os.path.join(
            self.snapshot_command.snapshot_controller.home, "snapshot_ls_*")
        paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
        assert paths
        assert open(paths[0], "r").read()
        os.remove(paths[0])

        # Test success format table, download exact path
        test_path = os.path.join(self.temp_dir, "my_output")
        self.snapshot_command.parse(
            ["snapshot", "ls", "--download", "--download-path", test_path])
        snapshot_objs = self.snapshot_command.execute()
        assert created_snapshot_obj in snapshot_objs
        assert os.path.isfile(test_path)
        assert open(test_path, "r").read()
        os.remove(test_path)

    def test_snapshot_checkout_invalid_arg(self):
        self.__set_variables()
        exception_thrown = False
        try:
            self.snapshot_command.parse(
                ["snapshot", "checkout"
                 "--foobar", "foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    def test_snapshot_checkout(self):
        self.__set_variables()
        # Test when optional parameters are not given
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])
        snapshot_obj = self.snapshot_command.execute()

        # Test when optional parameters are not given
        self.snapshot_command.parse(["snapshot", "checkout", snapshot_obj.id])

        result = self.snapshot_command.execute()
        assert result

    def test_snapshot_diff(self):
        self.__set_variables()

        # Create config file
        with open(self.config_filepath, "wb") as f:
            f.write(to_bytes(str('{"depth":6}')))

        # Create stats file
        with open(self.stats_filepath, "wb") as f:
            f.write(to_bytes(str('{"acc":0.97}')))

        # Create snapshots to test
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])
        snapshot_obj_1 = self.snapshot_command.execute()

        # Create another test file
        self.filepath_3 = os.path.join(self.snapshot_command.home, "file3.txt")
        with open(self.filepath_3, "wb") as f:
            f.write(to_bytes(str("test")))

        # Create config file
        with open(self.config_filepath, "wb") as f:
            f.write(to_bytes(str('{"depth":5}')))

        # Create stats file
        with open(self.stats_filepath, "wb") as f:
            f.write(to_bytes(str('{"acc":0.91}')))

        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my second snapshot"])
        snapshot_obj_2 = self.snapshot_command.execute()

        # Test diff with the above two snapshots
        self.snapshot_command.parse(
            ["snapshot", "diff", snapshot_obj_1.id, snapshot_obj_2.id])

        result = self.snapshot_command.execute()
        assert result
        assert "message" in result
        assert "label" in result
        assert "code_id" in result
        assert "environment_id" in result
        assert "file_collection_id" in result
        assert "config" in result
        assert "stats" in result

    def test_snapshot_inspect(self):
        self.__set_variables()
        # Create snapshot to test
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])
        snapshot_obj = self.snapshot_command.execute()
        # Test inspect for this snapshot
        self.snapshot_command.parse(["snapshot", "inspect", snapshot_obj.id])

        result = self.snapshot_command.execute()
        assert result
        assert "Code" in result
        assert "Environment" in result
        assert "Files" in result
        assert "Config" in result
        assert "Stats" in result
示例#2
0
class TestWorkspace():
    def setup_method(self):
        self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
        Config().set_home(self.temp_dir)
        self.cli_helper = Helper()

    def __set_variables(self):
        self.project_command = ProjectCommand(self.cli_helper)
        self.project_command.parse(
            ["init", "--name", "foobar", "--description", "test model"])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        dummy(self)

        self.environment_command = EnvironmentCommand(self.cli_helper)
        self.workspace_command = WorkspaceCommand(self.cli_helper)
        self.run_command = RunCommand(self.cli_helper)

        # Create environment_driver definition
        self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
        with open(self.env_def_path, "wb") as f:
            f.write(
                to_bytes(str("FROM datmo/python-base:cpu-py27%s" %
                             os.linesep)))

    def teardown_method(self):
        pass

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_notebook(self):
        self.__set_variables()
        test_mem_limit = "4g"
        # test single ports option before command
        self.workspace_command.parse([
            "notebook",
            "--gpu",
            "--environment-paths",
            self.env_def_path,
            "--mem-limit",
            test_mem_limit,
        ])

        # test for desired side effects
        assert self.workspace_command.args.gpu == True
        assert self.workspace_command.args.environment_paths == [
            self.env_def_path
        ]
        assert self.workspace_command.args.mem_limit == test_mem_limit

        # test notebook command
        self.workspace_command.parse(["notebook"])

        assert self.workspace_command.args.gpu == False

        @timeout_decorator.timeout(10, use_signals=False)
        def timed_run(timed_run_result):
            if self.workspace_command.execute():
                return timed_run_result

        timed_run_result = False
        try:
            timed_run_result = timed_run(timed_run_result)
        except timeout_decorator.timeout_decorator.TimeoutError:
            timed_run_result = True

        assert timed_run_result

        # Stop all running datmo task to not have any containers
        self.run_command.parse(["stop", "--all"])
        self.run_command.execute()

        # creating and passing environment id
        random_text = str(uuid.uuid1())
        with open(self.env_def_path, "wb") as f:
            f.write(
                to_bytes(str("FROM datmo/python-base:cpu-py27%s" %
                             os.linesep)))
            f.write(to_bytes(str("RUN echo " + random_text)))

        self.environment_command.parse([
            "environment", "create", "--name", "test", "--description",
            "test description"
        ])
        result = self.environment_command.execute()

        # test notebook command
        self.workspace_command.parse(
            ["notebook", "--environment-id", result.id])

        @timeout_decorator.timeout(10, use_signals=False)
        def timed_run(timed_run_result):
            if self.workspace_command.execute():
                return timed_run_result

        timed_run_result = False
        try:
            timed_run_result = timed_run(timed_run_result)
        except timeout_decorator.timeout_decorator.TimeoutError:
            timed_run_result = True

        assert timed_run_result

        # Stop all running datmo task
        self.run_command.parse(["stop", "--all"])
        self.run_command.execute()

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_jupyterlab(self):
        self.__set_variables()
        test_mem_limit = "4g"
        # test single ports option before command
        self.workspace_command.parse([
            "jupyterlab",
            "--gpu",
            "--environment-paths",
            self.env_def_path,
            "--mem-limit",
            test_mem_limit,
        ])

        # test for desired side effects
        assert self.workspace_command.args.gpu == True
        assert self.workspace_command.args.environment_paths == [
            self.env_def_path
        ]
        assert self.workspace_command.args.mem_limit == test_mem_limit

        # test multiple ports option before command
        self.workspace_command.parse(["jupyterlab"])

        assert self.workspace_command.args.gpu == False

        @timeout_decorator.timeout(10, use_signals=False)
        def timed_run(timed_run_result):
            if self.workspace_command.execute():
                return timed_run_result

        timed_run_result = False
        try:
            timed_run_result = timed_run(timed_run_result)
        except timeout_decorator.timeout_decorator.TimeoutError:
            timed_run_result = True

        assert timed_run_result

        # Stop all running datmo task to not have any containers
        self.run_command.parse(["stop", "--all"])
        self.run_command.execute()

        # creating and passing environment id
        random_text = str(uuid.uuid1())
        with open(self.env_def_path, "wb") as f:
            f.write(
                to_bytes(str("FROM datmo/python-base:cpu-py27%s" %
                             os.linesep)))
            f.write(to_bytes(str("RUN echo " + random_text)))

        self.environment_command.parse([
            "environment", "create", "--name", "test", "--description",
            "test description"
        ])
        result = self.environment_command.execute()

        # test notebook command
        self.workspace_command.parse(
            ["jupyterlab", "--environment-id", result.id])

        @timeout_decorator.timeout(10, use_signals=False)
        def timed_run(timed_run_result):
            if self.workspace_command.execute():
                return timed_run_result

        timed_run_result = False
        try:
            timed_run_result = timed_run(timed_run_result)
        except timeout_decorator.timeout_decorator.TimeoutError:
            timed_run_result = True

        assert timed_run_result

        # Stop all running datmo task
        self.run_command.parse(["stop", "--all"])
        self.run_command.execute()

    # Test doesn't take tty as True for docker
    # @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    # def test_terminal(self):
    #     self.__set_variables()
    #     test_mem_limit = "4g"
    #     # test single ports option before command
    #     self.workspace_command.parse([
    #         "terminal",
    #         "--gpu",
    #         "--environment-paths",
    #         self.env_def_path,
    #         "--mem-limit",
    #         test_mem_limit,
    #     ])
    #
    #     # test for desired side effects
    #     assert self.workspace_command.args.gpu == True
    #     assert self.workspace_command.args.environment_paths == [
    #         self.env_def_path
    #     ]
    #     assert self.workspace_command.args.mem_limit == test_mem_limit
    #
    #     # test multiple ports option before command
    #     self.workspace_command.parse(["terminal"])
    #
    #     assert self.workspace_command.args.gpu == False
    #     @timeout_decorator.timeout(10, use_signals=False)
    #     def timed_run(timed_run_result):
    #         if self.workspace_command.execute():
    #             return timed_run_result
    #
    #     timed_run_result = False
    #     timed_run_result = timed_run(timed_run_result)
    #
    #     assert timed_run_result
    #
    #     # Stop all running datmo task
    #     self.run_command.parse(["stop", "--all"])
    #     self.run_command.execute()

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_rstudio(self):
        self.__set_variables()
        # Update environment_driver definition
        self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
        with open(self.env_def_path, "wb") as f:
            f.write(to_bytes(str("FROM datmo/r-base:cpu%s" % os.linesep)))

        test_mem_limit = "4g"
        # test single ports option before command
        self.workspace_command.parse([
            "rstudio",
            "--environment-paths",
            self.env_def_path,
            "--mem-limit",
            test_mem_limit,
        ])

        # test for desired side effects
        assert self.workspace_command.args.environment_paths == [
            self.env_def_path
        ]
        assert self.workspace_command.args.mem_limit == test_mem_limit

        # test multiple ports option before command
        self.workspace_command.parse(["rstudio"])

        @timeout_decorator.timeout(10, use_signals=False)
        def timed_run(timed_run_result):
            if self.workspace_command.execute():
                return timed_run_result

        timed_run_result = False
        try:
            timed_run_result = timed_run(timed_run_result)
        except timeout_decorator.timeout_decorator.TimeoutError:
            timed_run_result = True

        assert timed_run_result

        # Stop all running datmo task to not have any containers
        self.run_command.parse(["stop", "--all"])
        self.run_command.execute()

        # creating and passing environment id
        random_text = str(uuid.uuid1())
        with open(self.env_def_path, "wb") as f:
            f.write(to_bytes(str("FROM datmo/r-base:cpu%s" % os.linesep)))
            f.write(to_bytes(str("RUN echo " + random_text)))

        self.environment_command.parse([
            "environment", "create", "--name", "test", "--description",
            "test description"
        ])
        result = self.environment_command.execute()

        # test notebook command
        self.workspace_command.parse(
            ["rstudio", "--environment-id", result.id])

        @timeout_decorator.timeout(10, use_signals=False)
        def timed_run(timed_run_result):
            if self.workspace_command.execute():
                return timed_run_result

        timed_run_result = False
        try:
            timed_run_result = timed_run(timed_run_result)
        except timeout_decorator.timeout_decorator.TimeoutError:
            timed_run_result = True

        assert timed_run_result

        # Stop all running datmo task
        self.run_command.parse(["stop", "--all"])
        self.run_command.execute()
示例#3
0
class TestFlow():
    def setup_method(self):
        self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
        Config().set_home(self.temp_dir)
        self.cli_helper = Helper()

    def teardown_method(self):
        pass

    def __set_variables(self):
        self.project_command = ProjectCommand(self.cli_helper)
        self.project_command.parse(
            ["init", "--name", "foobar", "--description", "test model"])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        dummy(self)
        self.environment_command = EnvironmentCommand(self.cli_helper)
        self.run_command = RunCommand(self.cli_helper)
        self.snapshot_command = SnapshotCommand(self.cli_helper)

        # Create test file
        self.filepath = os.path.join(self.snapshot_command.home, "file.txt")
        with open(self.filepath, "wb") as f:
            f.write(to_bytes(str("test")))

    def __environment_setup(self):
        self.environment_command.parse(["environment", "setup"])

        @self.environment_command.cli_helper.input("\n\n\n")
        def dummy(self):
            return self.environment_command.execute()

        environment_setup_result = dummy(self)
        return environment_setup_result

    def __run(self):
        test_command = ["sh", "-c", "echo accuracy:0.45"]
        test_ports = ["8888:8888"]
        self.run_command.parse(["run", "-p", test_ports[0], test_command])
        run_result = self.run_command.execute()
        return run_result

    def __run_ls(self):
        self.run_command.parse(["ls"])
        run_ls_result = self.run_command.execute()
        return run_ls_result

    def __snapshot_create(self):
        test_message = "creating a snapshot"
        self.snapshot_command.parse(["snapshot", "create", "-m", test_message])

        snapshot_create_result = self.snapshot_command.execute()
        return snapshot_create_result

    def __snapshot_ls(self):
        self.snapshot_command.parse(["snapshot", "ls"])
        snapshot_ls_result = self.snapshot_command.execute()
        return snapshot_ls_result

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_flow_1(self):
        # Flow
        # Step 1: environment setup
        # Step 2: run
        # Step 3: ls
        # Step 4: snapshot create
        # Step 5: snapshot ls
        self.__set_variables()

        # Step 1: environment setup
        environment_setup_result = self.__environment_setup()
        assert environment_setup_result

        # Step 2: run command
        run_result = self.__run()
        assert run_result

        # Step 3: ls
        run_ls_result = self.__run_ls()
        assert run_ls_result

        # Step 4: snapshot create
        snapshot_create_result = self.__snapshot_create()
        assert snapshot_create_result

        # Step 5: snapshot ls
        snapshot_ls_result = self.__snapshot_ls()
        assert snapshot_ls_result

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_flow_2(self):
        # Flow interruption in environment
        # Step 1: interrupted environment setup
        # Step 2: environment setup
        # Step 3: run
        # Step 4: ls
        # Step 5: snapshot create
        # Step 6: snapshot ls
        self.__set_variables()

        # TODO: TEST more fundamental functions first
        # Step 1: interrupted environment setup
        # @timeout_decorator.timeout(0.0001, use_signals=False)
        # def timed_command_with_interuption():
        #     result = self.__environment_setup()
        #     return result
        #
        # failed = False
        # try:
        #     timed_command_with_interuption()
        # except timeout_decorator.timeout_decorator.TimeoutError:
        #     failed = True
        # assert failed

        # Step 2: environment setup
        environment_setup_result = self.__environment_setup()
        assert environment_setup_result

        # Step 3: run
        run_result = self.__run()
        assert run_result

        # Step 4: ls
        run_ls_result = self.__run_ls()
        assert run_ls_result

        # Step 5: snapshot create
        snapshot_create_result = self.__snapshot_create()
        assert snapshot_create_result

        # Step 6: snapshot ls
        snapshot_ls_result = self.__snapshot_ls()
        assert snapshot_ls_result

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_flow_3(self):
        # Flow interruption in run
        # Step 1: environment setup
        # Step 2: interrupted run
        # Step 3: run
        # Step 4: ls
        # Step 5: snapshot create
        # Step 6: snapshot ls
        self.__set_variables()
        # Step 1: environment setup
        environment_setup_result = self.__environment_setup()
        assert environment_setup_result

        # TODO: Test more fundamental functions first
        # Step 2: interrupted run
        # @timeout_decorator.timeout(0.0001, use_signals=False)
        # def timed_command_with_interuption():
        #     result = self.__run()
        #     return result
        #
        # failed = False
        # try:
        #     timed_command_with_interuption()
        # except timeout_decorator.timeout_decorator.TimeoutError:
        #     failed = True
        # assert failed

        # Step 3: run
        run_result = self.__run()
        assert run_result

        # Step 4: task ls
        run_ls_result = self.__run_ls()
        assert run_ls_result

        # Step 5: snapshot create
        snapshot_create_result = self.__snapshot_create()
        assert snapshot_create_result

        # Step 6: snapshot ls
        snapshot_ls_result = self.__snapshot_ls()
        assert snapshot_ls_result

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_flow_4(self):
        # Flow interruption in snapshot create
        # Step 1: environment setup
        # Step 2: run
        # Step 3: ls
        # Step 4: interrupted snapshot create
        # Step 5: snapshot create
        # Step 6: snapshot ls
        self.__set_variables()

        # Step 1: environment setup
        environment_setup_result = self.__environment_setup()
        assert environment_setup_result

        # Step 2: run
        run_result = self.__run()
        assert run_result

        # Step 3: ls
        run_ls_result = self.__run_ls()
        assert run_ls_result

        # TODO: Test more fundamental functions first
        # Step 4: interrupted snapshot create
        # @timeout_decorator.timeout(0.0001, use_signals=False)
        # def timed_command_with_interuption():
        #     result = self.__snapshot_create()
        #     return result
        #
        # failed = False
        # try:
        #     timed_command_with_interuption()
        # except timeout_decorator.timeout_decorator.TimeoutError:
        #     failed = True
        # assert failed
        # snapshot_ls_result = self.__snapshot_ls()
        # assert not snapshot_ls_result

        # Step 5: snapshot create
        snapshot_create_result = self.__snapshot_create()
        assert snapshot_create_result

        # Step 6: snapshot ls
        snapshot_ls_result = self.__snapshot_ls()
        assert snapshot_ls_result
示例#4
0
class TestRunCommand():
    def setup_method(self):
        self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
        Config().set_home(self.temp_dir)
        self.cli_helper = Helper()
        self.snapshot_dict = {
            "id": "test",
            "model_id": "my_model",
            "session_id": "my_session",
            "message": "my message",
            "code_id": "my_code_id",
            "environment_id": "my_environment_id",
            "file_collection_id": "my file collection",
            "config": {
                "test": 0.56
            },
            "stats": {
                "test": 0.34
            }
        }
        self.task_dict = {
            "id": "test",
            "model_id": "my_model",
            "session_id": "my_session",
            "command": "python test.py"
        }

    def teardown_method(self):
        pass

    def __set_variables(self):
        self.project_command = ProjectCommand(self.cli_helper)
        self.project_command.parse(
            ["init", "--name", "foobar", "--description", "test model"])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        dummy(self)

        self.run_command = RunCommand(self.cli_helper)

        # Create environment_driver definition
        self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
        with open(self.env_def_path, "wb") as f:
            f.write(to_bytes("FROM python:3.5-alpine"))

    def test_run_object_instantiate(self):
        task_obj = CoreTask(self.task_dict)
        result = RunObject(task_obj)
        assert result
        assert isinstance(result, RunObject)

    def test_run_should_fail1(self):
        self.__set_variables()
        # Test failure case
        self.run_command.parse(["run"])
        result = self.run_command.execute()
        assert not result

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_run(self):
        # TODO: Adding test with `--interactive` argument and terminate inside container
        self.__set_variables()
        # Test failure command execute
        test_command = ["yo", "yo"]
        self.run_command.parse(["run", test_command])
        result = self.run_command.execute()
        assert result
        # Test success case
        test_command = ["sh", "-c", "echo accuracy:0.45"]
        test_ports = ["8888:8888", "9999:9999"]
        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
        test_mem_limit = "4g"

        # test for single set of ports
        self.run_command.parse([
            "run", "--ports", test_ports[0], "--environment-paths",
            test_dockerfile, "--mem-limit", test_mem_limit, test_command
        ])
        # test for desired side effects
        assert self.run_command.args.cmd == test_command
        assert self.run_command.args.ports == [test_ports[0]]
        assert self.run_command.args.environment_paths == [test_dockerfile]
        assert self.run_command.args.mem_limit == test_mem_limit

        self.run_command.parse([
            "run", "-p", test_ports[0], "-p", test_ports[1],
            "--environment-paths", test_dockerfile, "--mem-limit",
            test_mem_limit, test_command
        ])
        # test for desired side effects
        assert self.run_command.args.cmd == test_command
        assert self.run_command.args.ports == test_ports
        assert self.run_command.args.environment_paths == [test_dockerfile]
        assert self.run_command.args.mem_limit == test_mem_limit

        # test proper execution of run command
        result = self.run_command.execute()
        time.sleep(1)
        assert result
        assert isinstance(result, RunObject)
        assert result.logs
        assert "accuracy" in result.logs
        assert result.results
        assert result.results == {"accuracy": "0.45"}
        assert result.status == "SUCCESS"
        assert result.start_time
        assert result.end_time
        assert result.duration
        assert result.core_snapshot_id
        assert result.core_snapshot_id == result.after_snapshot_id
        assert result.environment_id

        # teardown
        self.run_command.parse(["stop", "--all"])
        # test when all is passed to stop all
        self.run_command.execute()

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_run_data_dir(self):
        # TODO: Adding test with `--interactive` argument and terminate inside container
        self.__set_variables()
        # Test success case
        test_filename = "script.py"
        test_command = ["python", test_filename]
        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
        test_mem_limit = "4g"
        # Test success for run with directory being passed
        test_data_dir_1 = os.path.join(tempfile.mkdtemp(dir=test_datmo_dir),
                                       "data1")
        os.mkdir(test_data_dir_1)
        test_data_dir_2 = os.path.join(tempfile.mkdtemp(dir=test_datmo_dir),
                                       "data2")
        os.mkdir(test_data_dir_2)
        with open(os.path.join(test_data_dir_1, "file.txt"), "wb") as f:
            f.write(to_bytes('my initial line in 1\n'))
        with open(os.path.join(test_data_dir_2, "file.txt"), "wb") as f:
            f.write(to_bytes('my initial line in 2\n'))
        test_filename = "script.py"
        test_filepath = os.path.join(self.temp_dir, test_filename)
        with open(test_filepath, "wb") as f:
            f.write(to_bytes("import os\n"))
            f.write(to_bytes("print('hello')\n"))
            f.write(to_bytes("import shutil\n"))

            f.write(
                to_bytes(
                    "with open(os.path.join('/data', 'data1', 'file.txt'), 'a') as f:\n"
                ))
            f.write(to_bytes("    f.write('my test file in 1')\n"))

            f.write(
                to_bytes(
                    "with open(os.path.join('/data', 'data2', 'file.txt'), 'a') as f:\n"
                ))
            f.write(to_bytes("    f.write('my test file in 2')\n"))

        self.run_command.parse([
            "run", "--environment-paths", test_dockerfile, "--data",
            test_data_dir_1, "--data", test_data_dir_2, "--mem-limit",
            test_mem_limit, test_command
        ])

        # test proper execution of run command
        result = self.run_command.execute()
        time.sleep(1)
        assert result
        assert isinstance(result, RunObject)
        assert result.logs
        assert result.status == "SUCCESS"
        assert result.start_time
        assert result.end_time
        assert result.duration
        assert result.core_snapshot_id
        assert result.core_snapshot_id == result.after_snapshot_id
        assert result.environment_id
        assert "my initial line in 1" in open(
            os.path.join(test_data_dir_1, "file.txt"), "r").read()
        assert "my test file in 1" in open(
            os.path.join(test_data_dir_1, "file.txt"), "r").read()
        assert "my initial line in 2" in open(
            os.path.join(test_data_dir_2, "file.txt"), "r").read()
        assert "my test file in 2" in open(
            os.path.join(test_data_dir_2, "file.txt"), "r").read()
        # teardown
        self.run_command.parse(["stop", "--all"])
        # test when all is passed to stop all
        self.run_command.execute()

        # test failure
        test_data_dir_dne = os.path.join(tempfile.mkdtemp(dir=test_datmo_dir),
                                         "data_dne")
        self.run_command.parse([
            "run", "--environment-paths", test_dockerfile, "--data",
            test_data_dir_dne
        ])
        result = self.run_command.execute()
        assert not result

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_run_string_command(self):
        # TODO: Adding test with `--interactive` argument and terminate inside container
        self.__set_variables()
        # Test success case
        test_command = "sh -c 'echo accuracy:0.45'"
        test_ports = ["8888:8888", "9999:9999"]
        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
        test_mem_limit = "4g"
        self.run_command.parse([
            "run", "--ports", test_ports[0], "--ports", test_ports[1],
            "--environment-paths", test_dockerfile, "--mem-limit",
            test_mem_limit, test_command
        ])
        # test for desired side effects
        assert self.run_command.args.cmd == test_command
        assert self.run_command.args.ports == test_ports
        assert self.run_command.args.environment_paths == [test_dockerfile]
        assert self.run_command.args.mem_limit == test_mem_limit

        # test proper execution of run command
        result = self.run_command.execute()
        assert result
        assert isinstance(result, RunObject)
        assert result.logs
        assert "accuracy" in result.logs
        assert result.results
        assert result.results == {"accuracy": "0.45"}
        assert result.status == "SUCCESS"
        assert result.start_time
        assert result.end_time
        assert result.duration
        assert result.core_snapshot_id
        assert result.core_snapshot_id == result.after_snapshot_id
        assert result.environment_id
        assert result

        # teardown
        self.run_command.parse(["stop", "--all"])
        # test when all is passed to stop all
        self.run_command.execute()

    # def test_multiple_concurrent_run_command(self):
    #     test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
    #     test_command = ["sh", "-c", "echo accuracy:0.45"]
    #     manager = Manager()
    #     return_dict = manager.dict()
    #
    #     def run_exec_func(procnum, return_dict):
    #         print("Creating Task object")
    #         run = RunCommand(self.temp_dir, self.cli_helper)
    #         print("Parsing command")
    #         run.parse(
    #             ["run", "--environment-paths", test_dockerfile, test_command])
    #         print("Executing command")
    #         result = run.execute()
    #         return_dict[procnum] = result
    #
    #     self.__set_variables()
    #     test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
    #
    #     # Run all three runs in parallel
    #     jobs = []
    #     number_runs = 3
    #     for i in range(number_runs):
    #         p = Process(target=run_exec_func, args=(i, return_dict))
    #         jobs.append(p)
    #         p.start()
    #
    #     # Join
    #     for proc in jobs:
    #         proc.join()
    #
    #     results = return_dict.values()
    #     assert len(results) == number_runs
    #     for result in results:
    #         assert result
    #         assert isinstance(result, CoreTask)
    #         assert result.logs
    #         assert "accuracy" in result.logs
    #         assert result.results
    #         assert result.results == {"accuracy": "0.45"}
    #         assert result.status == "SUCCESS"

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_run_notebook(self):
        self.__set_variables()
        # Update the default Dockerfile to test with
        with open(self.env_def_path, "wb") as f:
            f.write(to_bytes("FROM nbgallery/jupyter-alpine:latest"))

        # Test success case
        test_command = ["jupyter", "notebook", "list"]
        test_ports = ["8888:8888", "9999:9999"]
        test_mem_limit = "4g"

        # test single ports option before command
        self.run_command.parse([
            "run", "--ports", test_ports[0], "--mem-limit", test_mem_limit,
            test_command
        ])

        # test for desired side effects
        assert self.run_command.args.cmd == test_command
        assert self.run_command.args.ports == [test_ports[0]]
        assert self.run_command.args.mem_limit == test_mem_limit

        # test multiple ports option before command
        self.run_command.parse([
            "run", "--ports", test_ports[0], "--ports", test_ports[1],
            "--mem-limit", test_mem_limit, "--environment-paths",
            self.env_def_path, test_command
        ])

        # test for desired side effects
        assert self.run_command.args.cmd == test_command
        assert self.run_command.args.ports == test_ports
        assert self.run_command.args.mem_limit == test_mem_limit

        # test proper execution of run command
        result = self.run_command.execute()
        assert result
        assert isinstance(result, RunObject)
        assert result.logs
        assert "Currently running servers" in result.logs
        assert result.status == "SUCCESS"

        # teardown
        self.run_command.parse(["stop", "--all"])
        # test when all is passed to stop all
        self.run_command.execute()

    def test_run_invalid_arg(self):
        self.__set_variables()
        exception_thrown = False
        try:
            self.run_command.parse(["run" "--foobar", "foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_run_ls(self):
        self.__set_variables()

        # Running a task
        test_command = ["sh", "-c", "echo accuracy:0.45"]
        test_ports = ["8888:8888", "9999:9999"]
        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
        test_mem_limit = "4g"

        # test for single set of ports
        self.run_command.parse([
            "run", "--ports", test_ports[0], "--environment-paths",
            test_dockerfile, "--mem-limit", test_mem_limit, test_command
        ])

        # test proper execution of run command
        self.run_command.execute()
        # Test defaults
        self.run_command.parse(["ls"])
        run_objs = self.run_command.execute()
        assert run_objs
        assert run_objs[0].status == "SUCCESS"

        test_session_id = 'test_session_id'
        self.run_command.parse(["ls", "--session-id", test_session_id])

        # test for desired side effects
        assert self.run_command.args.session_id == test_session_id

        # Test failure no session
        failed = False
        try:
            _ = self.run_command.execute()
        except SessionDoesNotExist:
            failed = True
        assert failed

        # Test failure (format)
        failed = False
        try:
            self.run_command.parse(["ls", "--format"])
        except ArgumentError:
            failed = True
        assert failed

        # Test success format csv
        self.run_command.parse(["ls", "--format", "csv"])
        run_objs = self.run_command.execute()
        assert run_objs
        assert run_objs[0].status == "SUCCESS"

        # Test success format csv, download default
        self.run_command.parse(["ls", "--format", "csv", "--download"])
        run_objs = self.run_command.execute()
        assert run_objs
        assert run_objs[0].status == "SUCCESS"
        test_wildcard = os.path.join(os.getcwd(), "run_ls_*")
        paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
        assert paths
        assert open(paths[0], "r").read()
        os.remove(paths[0])

        # Test success format csv, download exact path
        test_path = os.path.join(self.temp_dir, "my_output")
        self.run_command.parse([
            "ls", "--format", "csv", "--download", "--download-path", test_path
        ])
        run_objs = self.run_command.execute()
        assert run_objs
        assert run_objs[0].status == "SUCCESS"
        assert os.path.isfile(test_path)
        assert open(test_path, "r").read()
        os.remove(test_path)

        # Test success format table
        self.run_command.parse(["ls"])
        run_objs = self.run_command.execute()
        assert run_objs
        assert run_objs[0].status == "SUCCESS"

        # Test success format table, download default
        self.run_command.parse(["ls", "--download"])
        run_objs = self.run_command.execute()
        assert run_objs
        assert run_objs[0].status == "SUCCESS"
        test_wildcard = os.path.join(os.getcwd(), "run_ls_*")
        paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
        assert paths
        assert open(paths[0], "r").read()
        os.remove(paths[0])

        # Test success format table, download exact path
        test_path = os.path.join(self.temp_dir, "my_output")
        self.run_command.parse(
            ["ls", "--download", "--download-path", test_path])
        run_objs = self.run_command.execute()
        assert run_objs
        assert run_objs[0].status == "SUCCESS"
        assert os.path.isfile(test_path)
        assert open(test_path, "r").read()
        os.remove(test_path)

    def test_run_ls_invalid_arg(self):
        self.__set_variables()
        exception_thrown = False
        try:
            self.run_command.parse(["ls" "--foobar", "foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_run_stop_success(self):
        # 1) Test stop with task_id
        # 2) Test stop with all
        self.__set_variables()

        test_command = ["sh", "-c", "echo yo"]
        test_ports = "8888:8888"
        test_mem_limit = "4g"
        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")

        self.run_command.parse([
            "run", "--ports", test_ports, "--environment-paths",
            test_dockerfile, "--mem-limit", test_mem_limit, test_command
        ])

        test_run_obj = self.run_command.execute()

        # 1) Test option 1
        self.run_command.parse(["stop", "--id", test_run_obj.id])

        # test for desired side effects
        assert self.run_command.args.id == test_run_obj.id

        # test when task id is passed to stop it
        run_stop_command = self.run_command.execute()
        assert run_stop_command == True

        # 2) Test option 2
        self.run_command.parse(["stop", "--all"])

        # test when all is passed to stop all
        run_stop_command = self.run_command.execute()
        assert run_stop_command == True

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_run_stop_failure_required_args(self):
        self.__set_variables()
        # Passing wrong task id
        self.run_command.parse(["stop"])
        failed = False
        try:
            _ = self.run_command.execute()
        except RequiredArgumentMissing:
            failed = True
        assert failed

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_run_stop_failure_mutually_exclusive_vars(self):
        self.__set_variables()
        # Passing wrong task id
        self.run_command.parse(["stop", "--id", "invalid-task-id", "--all"])
        failed = False
        try:
            _ = self.run_command.execute()
        except MutuallyExclusiveArguments:
            failed = True
        assert failed

    def test_run_stop_failure_invalid_arg(self):
        self.__set_variables()
        exception_thrown = False
        try:
            self.run_command.parse(["stop" "--foobar", "foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    def test_run_stop_invalid_task_id(self):
        self.__set_variables()
        # Passing wrong task id
        self.run_command.parse(["stop", "--id", "invalid-task-id"])

        # test when wrong task id is passed to stop it
        result = self.run_command.execute()
        assert not result

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_run_delete_success(self):
        # 1) Test delete with run_id
        self.__set_variables()

        test_command = ["sh", "-c", "echo yo"]
        test_ports = "8888:8888"
        test_mem_limit = "4g"
        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")

        self.run_command.parse([
            "run", "--ports", test_ports, "--environment-paths",
            test_dockerfile, "--mem-limit", test_mem_limit, test_command
        ])

        test_run_obj = self.run_command.execute()

        # 1) Test option 1
        self.run_command.parse(["delete", test_run_obj.id])

        # test for desired side effects
        assert self.run_command.args.id == test_run_obj.id

        # test when task id is passed to delete it
        run_delete_command = self.run_command.execute()
        assert run_delete_command == True

    def test_run_delete_invalid_task_id(self):
        self.__set_variables()
        # Passing wrong task id
        self.run_command.parse(["delete", "invalid-task-id"])

        # test when wrong task id is passed to stop it
        result = self.run_command.execute()
        assert not result

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_rerun(self):
        self.__set_variables()
        # Test success case
        test_command = ["sh", "-c", "echo accuracy:0.45"]
        test_ports = ["8888:8888", "9999:9999"]
        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
        test_mem_limit = "4g"

        # test for single set of ports
        self.run_command.parse([
            "run", "-p", test_ports[0], "--environment-paths", test_dockerfile,
            "--mem-limit", test_mem_limit, test_command
        ])

        # test proper execution of run command
        run_obj = self.run_command.execute()
        run_id = run_obj.id
        # 1. Test success rerun
        self.run_command.parse(["rerun", run_id])
        result_run_obj = self.run_command.execute()
        assert result_run_obj
        assert isinstance(result_run_obj, RunObject)
        assert result_run_obj.command == run_obj.command
        assert result_run_obj.status == "SUCCESS"
        assert result_run_obj.logs
        assert result_run_obj.before_snapshot_id == run_obj.before_snapshot_id

        # teardown
        self.run_command.parse(["stop", "--all"])
        # test when all is passed to stop all
        self.run_command.execute()

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_rerun_invalid_arg(self):
        self.__set_variables()
        exception_thrown = False
        try:
            self.run_command.parse(["rerun", "foobar"])
            self.run_command.execute()
        except DoesNotExist:
            exception_thrown = True
        assert exception_thrown
示例#5
0
class TestProjectCommand():
    def setup_method(self):
        self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
        Config().set_home(self.temp_dir)
        self.cli_helper = Helper()
        self.project_command = ProjectCommand(self.cli_helper)

    def teardown_method(self):
        pass

    def test_init_create_success_default_name_no_description_no_environment(
            self):
        self.project_command.parse(["init"])

        # Test when environment is created
        @self.project_command.cli_helper.input("\n\ny\n\n\n\n")
        def dummy(self):
            return self.project_command.execute()

        result = dummy(self)

        # Ensure that the name and description are current
        _, default_name = os.path.split(
            self.project_command.project_controller.home)
        assert result
        assert result.name == default_name
        assert result.description == None
        # Ensure environment is correct
        definition_filepath = os.path.join(
            self.temp_dir, self.project_command.project_controller.file_driver.
            environment_directory, "Dockerfile")
        assert os.path.isfile(definition_filepath)
        assert "FROM datmo/python-base:cpu-py27" in open(
            definition_filepath, "r").read()

    def test_init_create_success_no_environment(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])

        # Test when environment is not created
        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        result = dummy(self)

        definition_filepath = os.path.join(
            self.temp_dir, self.project_command.project_controller.file_driver.
            environment_directory, "Dockerfile")

        assert result
        assert not os.path.isfile(definition_filepath)
        assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
        assert result.name == test_name
        assert result.description == test_description

    def test_init_create_success_environment(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])
        # Test when environment is created
        @self.project_command.cli_helper.input("y\n\n\n\n")
        def dummy(self):
            return self.project_command.execute()

        result = dummy(self)

        definition_filepath = os.path.join(
            self.temp_dir, self.project_command.project_controller.file_driver.
            environment_directory, "Dockerfile")

        assert result
        assert os.path.isfile(definition_filepath)
        assert "FROM datmo/python-base:cpu-py27" in open(
            definition_filepath, "r").read()

        # test for desired side effects
        assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
        assert result.name == test_name
        assert result.description == test_description

    def test_init_create_success_blank(self):
        self.project_command.parse(["init", "--name", "", "--description", ""])
        # test if prompt opens
        @self.project_command.cli_helper.input("\n\n\n")
        def dummy(self):
            return self.project_command.execute()

        result = dummy(self)
        assert result
        assert result.name
        assert not result.description

    def test_init_update_success(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        result_1 = dummy(self)
        updated_name = "foobar2"
        updated_description = "test model 2"
        self.project_command.parse([
            "init", "--name", updated_name, "--description",
            updated_description
        ])

        result_2 = dummy(self)
        # test for desired side effects
        assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
        assert result_2.id == result_1.id
        assert result_2.name == updated_name
        assert result_2.description == updated_description

    def test_init_update_success_only_name(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        result_1 = dummy(self)
        updated_name = "foobar2"
        self.project_command.parse(
            ["init", "--name", updated_name, "--description", ""])

        @self.project_command.cli_helper.input("\n\n")
        def dummy(self):
            return self.project_command.execute()

        result_2 = dummy(self)
        # test for desired side effects
        assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
        assert result_2.id == result_1.id
        assert result_2.name == updated_name
        assert result_2.description == result_1.description

    def test_init_update_success_only_description(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        result_1 = dummy(self)
        updated_description = "test model 2"
        self.project_command.parse(
            ["init", "--name", "", "--description", updated_description])

        @self.project_command.cli_helper.input("\n\n")
        def dummy(self):
            return self.project_command.execute()

        result_2 = dummy(self)
        # test for desired side effects
        assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
        assert result_2.id == result_1.id
        assert result_2.name == result_1.name
        assert result_2.description == updated_description

    def test_init_invalid_arg(self):
        exception_thrown = False
        try:
            self.project_command.parse(["init", "--foobar", "foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    def test_version(self):
        self.project_command.parse(["version"])
        result = self.project_command.execute()
        # test for desired side effects
        assert __version__ in result

    def test_version_invalid_arg(self):
        exception_thrown = False
        try:
            self.project_command.parse(["version", "--foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    def test_status_basic(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        _ = dummy(self)

        self.project_command.parse(["status"])
        result = self.project_command.execute()
        status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = result
        assert isinstance(status_dict, dict)
        assert not current_snapshot
        assert not latest_snapshot_user_generated
        assert not latest_snapshot_auto_generated
        assert unstaged_code
        assert not unstaged_environment
        assert not unstaged_files

    def test_status_user_generated_snapshot(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        _ = dummy(self)

        # Create a snapshot
        self.snapshot_command = SnapshotCommand(self.cli_helper)
        with open(os.path.join(self.project_command.home, "test.py"),
                  "wb") as f:
            f.write(to_bytes(str("import xgboost")))
        self.snapshot_command.parse(
            ["snapshot", "create", "--message", "test"])
        snapshot_obj = self.snapshot_command.execute()

        self.project_command.parse(["status"])
        result = self.project_command.execute()
        status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = result
        assert isinstance(status_dict, dict)
        assert isinstance(current_snapshot, Snapshot)
        assert isinstance(latest_snapshot_user_generated, Snapshot)
        assert snapshot_obj == latest_snapshot_user_generated
        assert current_snapshot == latest_snapshot_user_generated
        assert not latest_snapshot_auto_generated
        assert not unstaged_code
        assert not unstaged_environment
        assert not unstaged_files

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_status_autogenerated_snapshot(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        _ = dummy(self)

        self.run_command = RunCommand(self.cli_helper)
        self.task_controller = TaskController()

        # Create and run a task and test if task is shown
        test_command = ["sh", "-c", "echo accuracy:0.45"]
        self.run_command.parse(["run", test_command])

        updated_first_task = self.run_command.execute()
        before_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
            updated_first_task.before_snapshot_id)
        after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
            updated_first_task.after_snapshot_id)
        before_environment_obj = self.task_controller.dal.environment.get_by_id(
            before_snapshot_obj.environment_id)
        after_environment_obj = self.task_controller.dal.environment.get_by_id(
            after_snapshot_obj.environment_id)
        assert before_environment_obj == after_environment_obj

        self.project_command.parse(["status"])
        result = self.project_command.execute()
        status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
            result

        assert status_dict
        assert isinstance(status_dict, dict)
        assert status_dict['name'] == test_name
        assert status_dict['description'] == test_description
        assert isinstance(status_dict['config'], dict)
        assert isinstance(current_snapshot, Snapshot)
        assert not latest_snapshot_user_generated
        assert isinstance(latest_snapshot_auto_generated, Snapshot)
        # current snapshot is the before snapshot for the run
        assert current_snapshot == before_snapshot_obj
        assert current_snapshot != latest_snapshot_auto_generated
        # latest autogenerated snapshot is the after snapshot id
        assert latest_snapshot_auto_generated == after_snapshot_obj
        # autogenerated snapshot is after the user generated snapshot
        assert not unstaged_code
        assert not unstaged_environment
        assert not unstaged_files

        # Create a snapshot
        self.snapshot_command = SnapshotCommand(self.cli_helper)
        with open(os.path.join(self.project_command.home, "test.py"),
                  "wb") as f:
            f.write(to_bytes(str("import xgboost")))
        self.snapshot_command.parse(
            ["snapshot", "create", "--message", "test"])
        first_snapshot = self.snapshot_command.execute()

        # Create and run a task and test if task is shown
        test_command = ["sh", "-c", "echo accuracy:0.45"]
        self.run_command.parse(["run", test_command])

        updated_second_task = self.run_command.execute()
        before_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
            updated_second_task.before_snapshot_id)
        after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
            updated_second_task.after_snapshot_id)
        before_environment_obj = self.task_controller.dal.environment.get_by_id(
            before_snapshot_obj.environment_id)
        after_environment_obj = self.task_controller.dal.environment.get_by_id(
            after_snapshot_obj.environment_id)
        assert before_environment_obj == after_environment_obj

        self.project_command.parse(["status"])
        result = self.project_command.execute()
        status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
            result

        assert status_dict
        assert isinstance(status_dict, dict)
        assert status_dict['name'] == test_name
        assert status_dict['description'] == test_description
        assert isinstance(status_dict['config'], dict)
        assert isinstance(current_snapshot, Snapshot)
        assert isinstance(latest_snapshot_user_generated, Snapshot)
        assert latest_snapshot_user_generated == first_snapshot
        assert isinstance(latest_snapshot_auto_generated, Snapshot)
        # current snapshot is the before snapshot for the run
        assert current_snapshot == before_snapshot_obj
        assert current_snapshot == latest_snapshot_user_generated
        assert current_snapshot != latest_snapshot_auto_generated
        # latest autogenerated snapshot is the after snapshot id
        assert latest_snapshot_auto_generated == after_snapshot_obj
        assert latest_snapshot_auto_generated != latest_snapshot_user_generated
        # user generated snapshot is not associated with any before or after snapshot
        assert latest_snapshot_user_generated == before_snapshot_obj
        assert latest_snapshot_user_generated != after_snapshot_obj
        # autogenerated snapshot is after the user generated snapshot
        assert latest_snapshot_auto_generated.created_at > latest_snapshot_user_generated.created_at
        assert not unstaged_code
        assert not unstaged_environment
        assert not unstaged_files

        # Check if the same snapshot is given back when created
        self.snapshot_command.parse(
            ["snapshot", "create", "--message", "test"])
        new_snapshot = self.snapshot_command.execute()
        assert current_snapshot == new_snapshot

        # Create a user generated snapshot after some changes
        with open(os.path.join(self.project_command.home, "new.py"),
                  "wb") as f:
            f.write(to_bytes(str("import xgboost")))
        self.snapshot_command.parse(
            ["snapshot", "create", "--message", "test"])
        new_snapshot = self.snapshot_command.execute()
        # ensure we created a new snapshot
        assert current_snapshot != new_snapshot

        self.project_command.parse(["status"])
        result = self.project_command.execute()
        status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
            result

        assert status_dict
        assert isinstance(status_dict, dict)
        assert status_dict['name'] == test_name
        assert status_dict['description'] == test_description
        assert isinstance(status_dict['config'], dict)
        assert isinstance(current_snapshot, Snapshot)
        assert isinstance(latest_snapshot_user_generated, Snapshot)
        # current snapshot is the latest user generated one
        assert current_snapshot == latest_snapshot_user_generated
        # latest autogenerated snapshot is not the user generated one
        assert latest_snapshot_auto_generated != latest_snapshot_user_generated
        # autogenerated snapshot is after the user generated snapshot
        assert latest_snapshot_user_generated.created_at > latest_snapshot_auto_generated.created_at
        assert not unstaged_code
        assert not unstaged_environment
        assert not unstaged_files

        # Create a snapshot from the auto generated one and ensure they are the same
        self.run_command = RunCommand(self.cli_helper)
        self.task_controller = TaskController()

        test_command = ["sh", "-c", "echo accuracy:0.45"]
        self.run_command.parse(["run", test_command])
        updated_third_task = self.run_command.execute()
        before_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
            updated_third_task.before_snapshot_id)
        after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
            updated_third_task.after_snapshot_id)
        self.snapshot_command.parse([
            "snapshot", "create", "--run-id", updated_third_task.id,
            "--message", "test"
        ])
        converted_snapshot = self.snapshot_command.execute()

        self.project_command.parse(["status"])
        result = self.project_command.execute()
        status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
            result

        assert status_dict
        assert isinstance(status_dict, dict)
        assert status_dict['name'] == test_name
        assert status_dict['description'] == test_description
        assert isinstance(status_dict['config'], dict)
        assert isinstance(current_snapshot, Snapshot)
        assert isinstance(latest_snapshot_user_generated, Snapshot)
        assert isinstance(latest_snapshot_auto_generated, Snapshot)
        # current snapshot is the before snapshot for the run
        assert current_snapshot == before_snapshot_obj
        assert current_snapshot != latest_snapshot_user_generated
        # latest user generated snapshot is the same as that created by the run
        assert converted_snapshot == latest_snapshot_user_generated
        assert latest_snapshot_user_generated == after_snapshot_obj
        # latest user generated converted the latest task so latest generate is from the last task
        assert latest_snapshot_user_generated.created_at > latest_snapshot_auto_generated.created_at
        assert not unstaged_code
        assert not unstaged_environment
        assert not unstaged_files

    def test_status_invalid_arg(self):
        exception_thrown = False
        try:
            self.project_command.parse(["status", "--foobar"])
        except UnrecognizedCLIArgument:
            exception_thrown = True
        assert exception_thrown

    def test_cleanup(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        _ = dummy(self)

        self.project_command.parse(["cleanup"])

        @self.project_command.cli_helper.input("y\n\n")
        def dummy(self):
            return self.project_command.execute()

        result = dummy(self)
        assert not os.path.exists(os.path.join(self.temp_dir, '.datmo'))
        assert isinstance(result, bool)
        assert result

    def test_cleanup_invalid_arg(self):
        exception_thrown = False
        try:
            self.project_command.parse(["cleanup", "--foobar"])
        except UnrecognizedCLIArgument:
            exception_thrown = True
        assert exception_thrown