예제 #1
0
파일: test_snapshot.py 프로젝트: yyht/datmo
class TestSnapshotCommand():
    def setup_method(self):
        self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
        Config().set_home(self.temp_dir)
        self.cli_helper = Helper()

    def teardown_method(self):
        pass

    def __set_variables(self):
        self.project_command = ProjectCommand(self.cli_helper)
        self.project_command.parse(
            ["init", "--name", "foobar", "--description", "test model"])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        dummy(self)
        self.snapshot_command = SnapshotCommand(self.cli_helper)

        # Create environment_driver definition
        self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
        with open(self.env_def_path, "wb") as f:
            f.write(to_bytes("FROM python:3.5-alpine"))

        # Create config file
        self.config_filepath = os.path.join(self.snapshot_command.home,
                                            "config.json")
        with open(self.config_filepath, "wb") as f:
            f.write(to_bytes(str("{}")))

        # Create stats file
        self.stats_filepath = os.path.join(self.snapshot_command.home,
                                           "stats.json")
        with open(self.stats_filepath, "wb") as f:
            f.write(to_bytes(str("{}")))

        # Create test file
        self.filepath = os.path.join(self.snapshot_command.home, "file.txt")
        with open(self.filepath, "wb") as f:
            f.write(to_bytes(str("test")))

        # Create another test file
        self.filepath_2 = os.path.join(self.snapshot_command.home, "file2.txt")
        with open(self.filepath_2, "wb") as f:
            f.write(to_bytes(str("test")))

        # Create config
        self.config = 'foo:bar'
        self.config1 = "{'foo1':'bar1'}"
        self.config2 = "this is test config blob"

        # Create stats
        self.stats = 'foo:bar'
        self.stats1 = "{'foo1':'bar1'}"
        self.stats2 = "this is test stats blob"

    def test_snapshot_help(self):
        self.__set_variables()
        print(
            "\n====================================== datmo snapshot ==========================\n"
        )

        self.snapshot_command.parse(["snapshot"])
        assert self.snapshot_command.execute()

        print(
            "\n====================================== datmo snapshot --help ==========================\n"
        )

        self.snapshot_command.parse(["snapshot", "--help"])
        assert self.snapshot_command.execute()

        print(
            "\n====================================== datmo snapshot create --help ==========================\n"
        )

        self.snapshot_command.parse(["snapshot", "create", "--help"])
        assert self.snapshot_command.execute()

    def test_snapshot_create(self):
        self.__set_variables()
        test_message = "this is a test message"
        test_label = "test label"
        test_run_id = "test_run_id"
        test_environment_definition_filepath = self.env_def_path
        test_config_filepath = self.config_filepath
        test_stats_filepath = self.config_filepath
        test_paths = [self.filepath, self.filepath_2]

        # try single filepath
        self.snapshot_command.parse([
            "snapshot", "create", "--message", test_message, "--run-id",
            test_run_id
        ])

        # testing for proper parsing
        assert self.snapshot_command.args.message == test_message
        assert self.snapshot_command.args.run_id == test_run_id

        # try single filepath
        self.snapshot_command.parse([
            "snapshot",
            "create",
            "--message",
            test_message,
            "--label",
            test_label,
            "--environment-paths",
            test_environment_definition_filepath,
            "--config-filepath",
            test_config_filepath,
            "--stats-filepath",
            test_stats_filepath,
            "--paths",
            test_paths[0],
        ])

        # test for desired side effects
        assert self.snapshot_command.args.message == test_message
        assert self.snapshot_command.args.label == test_label
        assert self.snapshot_command.args.environment_paths == [
            test_environment_definition_filepath
        ]
        assert self.snapshot_command.args.config_filepath == test_config_filepath
        assert self.snapshot_command.args.stats_filepath == test_stats_filepath
        assert self.snapshot_command.args.paths == [test_paths[0]]

        # test multiple paths
        self.snapshot_command.parse([
            "snapshot", "create", "--message", test_message, "--label",
            test_label, "--environment-paths",
            test_environment_definition_filepath, "--config-filepath",
            test_config_filepath, "--stats-filepath", test_stats_filepath,
            "--paths", test_paths[0], "--paths", test_paths[1]
        ])

        # test for desired side effects
        assert self.snapshot_command.args.message == test_message
        assert self.snapshot_command.args.label == test_label
        assert self.snapshot_command.args.environment_paths == [
            test_environment_definition_filepath
        ]
        assert self.snapshot_command.args.config_filepath == test_config_filepath
        assert self.snapshot_command.args.stats_filepath == test_stats_filepath
        assert self.snapshot_command.args.paths == test_paths

        snapshot_obj_1 = self.snapshot_command.execute()
        assert snapshot_obj_1

    def test_snapshot_create_config_stats(self):
        self.__set_variables()
        test_message = "this is a test message"
        test_label = "test label"
        test_config = self.config
        test_stats = self.stats

        # try config
        self.snapshot_command.parse([
            "snapshot", "create", "--message", test_message, "--label",
            test_label, "--config", test_config, "--stats", test_stats
        ])

        # test for desired side effects
        snapshot_obj = self.snapshot_command.execute()
        assert snapshot_obj

        test_config = self.config1
        test_stats = self.stats1

        # try config
        self.snapshot_command.parse([
            "snapshot", "create", "--message", test_message, "--label",
            test_label, "--config", test_config, "--stats", test_stats
        ])

        # test for desired side effects
        snapshot_obj = self.snapshot_command.execute()
        assert snapshot_obj

        test_config = self.config2
        test_stats = self.stats2

        # try config
        self.snapshot_command.parse([
            "snapshot", "create", "--message", test_message, "--label",
            test_label, "--config", test_config, "--stats", test_stats
        ])

        # test for desired side effects
        snapshot_obj = self.snapshot_command.execute()
        assert snapshot_obj

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_snapshot_create_from_run(self):
        self.__set_variables()
        test_message = "this is a test message"

        # create task
        test_command = "sh -c 'echo accuracy:0.45'"
        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
        self.run = RunCommand(self.cli_helper)
        self.run.parse(
            ["run", "--environment-paths", test_dockerfile, test_command])

        # test proper execution of task run command
        run_obj = self.run.execute()

        run_id = run_obj.id

        # test run id
        self.snapshot_command.parse([
            "snapshot", "create", "--message", test_message, "--run-id", run_id
        ])

        # test for desired side effects
        assert self.snapshot_command.args.message == test_message

        snapshot_obj = self.snapshot_command.execute()
        assert snapshot_obj

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_snapshot_create_from_run_fail_user_inputs(self):
        self.__set_variables()
        test_message = "this is a test message"

        # create run
        test_command = "sh -c 'echo accuracy:0.45'"
        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
        self.run = RunCommand(self.cli_helper)
        self.run.parse(
            ["run", "--environment-paths", test_dockerfile, test_command])

        # test proper execution of task run command
        run_obj = self.run.execute()

        run_id = run_obj.id

        failed = False
        try:
            # test run id with environment-id
            self.snapshot_command.parse([
                "snapshot", "create", "--message", test_message, "--run-id",
                run_id, "--environment-id", "test_environment_id"
            ])
            _ = self.snapshot_command.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test run id with environment-def
            self.snapshot_command.parse([
                "snapshot", "create", "--message", test_message, "--run-id",
                run_id, "--environment-paths", "test_environment_path"
            ])
            _ = self.snapshot_command.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test run id with filepaths
            self.snapshot_command.parse([
                "snapshot", "create", "--message", test_message, "--run-id",
                run_id, "--paths", "mypath"
            ])
            _ = self.snapshot_command.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test run id with config-filepath
            self.snapshot_command.parse([
                "snapshot", "create", "--message", test_message, "--run-id",
                run_id, "--config-filepath", "mypath"
            ])
            _ = self.snapshot_command.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test run id with config-filename
            self.snapshot_command.parse([
                "snapshot", "create", "--message", test_message, "--run-id",
                run_id, "--config-filename", "myname"
            ])
            _ = self.snapshot_command.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test run id with stats-filepath
            self.snapshot_command.parse([
                "snapshot", "create", "--message", test_message, "--run-id",
                run_id, "--stats-filepath", "mypath"
            ])
            _ = self.snapshot_command.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test run id with stats-filename
            self.snapshot_command.parse([
                "snapshot", "create", "--message", test_message, "--run-id",
                run_id, "--stats-filename", "myname"
            ])
            _ = self.snapshot_command.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

    def test_snapshot_create_fail_mutually_exclusive_args(self):
        self.__set_variables()
        test_message = "this is a test message"
        test_label = "test label"
        test_environment_id = "test_environment_id"
        test_environment_definition_filepath = self.env_def_path
        test_config_filename = "config.json"
        test_config_filepath = self.config_filepath
        test_stats_filename = "stats.json"
        test_stats_filepath = self.config_filepath

        # Environment exception
        exception_thrown = False
        try:
            self.snapshot_command.parse([
                "snapshot",
                "create",
                "--message",
                test_message,
                "--label",
                test_label,
                "--environment-id",
                test_environment_id,
                "--environment-paths",
                test_environment_definition_filepath,
            ])
            _ = self.snapshot_command.execute()
        except MutuallyExclusiveArguments:
            exception_thrown = True
        assert exception_thrown

        # Config exception
        exception_thrown = False
        try:
            self.snapshot_command.parse([
                "snapshot",
                "create",
                "--message",
                test_message,
                "--label",
                test_label,
                "--config-filename",
                test_config_filename,
                "--config-filepath",
                test_config_filepath,
            ])
            _ = self.snapshot_command.execute()
        except MutuallyExclusiveArguments:
            exception_thrown = True
        assert exception_thrown

        # Stats exception
        exception_thrown = False
        try:
            self.snapshot_command.parse([
                "snapshot",
                "create",
                "--message",
                test_message,
                "--label",
                test_label,
                "--stats-filename",
                test_stats_filename,
                "--stats-filepath",
                test_stats_filepath,
            ])
            _ = self.snapshot_command.execute()
        except MutuallyExclusiveArguments:
            exception_thrown = True
        assert exception_thrown

    def test_snapshot_create_default(self):
        self.__set_variables()
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])

        snapshot_obj_2 = self.snapshot_command.execute()
        assert snapshot_obj_2

    def test_snapshot_create_invalid_arg(self):
        self.__set_variables()
        exception_thrown = False
        try:
            self.snapshot_command.parse(
                ["snapshot", "create"
                 "--foobar", "foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    def test_snapshot_update(self):
        self.__set_variables()

        test_config = ["depth: 10", "learning_rate:0.91"]
        test_stats = ["acc: 91.34", "f1_score:0.91"]
        test_config1 = "{'foo_config': 'bar_config'}"
        test_stats1 = "{'foo_stats': 'bar_stats'}"
        test_config2 = "this is a config blob"
        test_stats2 = "this is a stats blob"
        test_message = "test_message"
        test_label = "test_label"

        # 1. Updating both message and label
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])
        snapshot_obj = self.snapshot_command.execute()

        # Test when optional parameters are not given
        self.snapshot_command.parse([
            "snapshot", "update", snapshot_obj.id, "--message", test_message,
            "--label", test_label
        ])

        result = self.snapshot_command.execute()
        assert result.id == snapshot_obj.id
        assert result.message == test_message
        assert result.label == test_label

        # 2. Updating only message
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])
        snapshot_obj = self.snapshot_command.execute()

        # Test when optional parameters are not given
        self.snapshot_command.parse(
            ["snapshot", "update", snapshot_obj.id, "--message", test_message])

        result = self.snapshot_command.execute()
        assert result.id == snapshot_obj.id
        assert result.message == test_message

        # Updating label
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])
        snapshot_obj = self.snapshot_command.execute()

        # Test when optional parameters are not given
        self.snapshot_command.parse(
            ["snapshot", "update", snapshot_obj.id, "--label", test_label])

        result = self.snapshot_command.execute()
        assert result.id == snapshot_obj.id
        assert result.label == test_label

        # 3. Updating config, message and label
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])
        snapshot_obj = self.snapshot_command.execute()

        # Test when optional parameters are not given
        self.snapshot_command.parse([
            "snapshot", "update", snapshot_obj.id, "--config", test_config[0],
            "--config", test_config[1], "--message", test_message, "--label",
            test_label
        ])

        result = self.snapshot_command.execute()
        assert result.id == snapshot_obj.id
        assert result.config == {"depth": "10", "learning_rate": "0.91"}
        assert result.message == test_message
        assert result.label == test_label

        # 4. Updating stats, message and label
        # Test when optional parameters are not given
        self.snapshot_command.parse([
            "snapshot", "update", snapshot_obj.id, "--stats", test_stats[0],
            "--stats", test_stats[1], "--message", test_message, "--label",
            test_label
        ])

        result = self.snapshot_command.execute()
        assert result.id == snapshot_obj.id
        assert result.stats == {"acc": "91.34", "f1_score": "0.91"}
        assert result.message == test_message
        assert result.label == test_label

        # Adding sleep due to issue with consistency in blitzdb database
        time.sleep(2)

        # 5. Updating config, stats
        # Test when optional parameters are not given
        self.snapshot_command.parse([
            "snapshot", "update", snapshot_obj.id, "--config", test_config1,
            "--stats", test_stats1
        ])

        result = self.snapshot_command.execute()
        assert result.id == snapshot_obj.id
        assert result.config == {
            "depth": "10",
            "learning_rate": "0.91",
            'foo_config': 'bar_config'
        }
        assert result.stats == {
            "acc": "91.34",
            "f1_score": "0.91",
            'foo_stats': 'bar_stats'
        }
        assert result.message == test_message
        assert result.label == test_label

        # Test when optional parameters are not given
        self.snapshot_command.parse([
            "snapshot", "update", snapshot_obj.id, "--config", test_config2,
            "--stats", test_stats2
        ])

        result = self.snapshot_command.execute()
        assert result.id == snapshot_obj.id
        assert result.config == {
            "depth": "10",
            "learning_rate": "0.91",
            'foo_config': 'bar_config',
            'config': test_config2
        }
        assert result.stats == {
            "acc": "91.34",
            "f1_score": "0.91",
            'foo_stats': 'bar_stats',
            'stats': test_stats2
        }
        assert result.message == test_message
        assert result.label == test_label

    def test_snapshot_delete(self):
        self.__set_variables()

        # Test when optional parameters are not given
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])

        snapshot_obj = self.snapshot_command.execute()

        # Test when optional parameters are not given
        self.snapshot_command.parse(["snapshot", "delete", snapshot_obj.id])

        result = self.snapshot_command.execute()
        assert result

    def test_snapshot_delete_invalid_arg(self):
        self.__set_variables()
        exception_thrown = False
        try:
            self.snapshot_command.parse(
                ["snapshot", "delete"
                 "--foobar", "foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_snapshot_ls_invisible(self):
        self.__set_variables()

        # Test invisible snapshot from task
        # create task
        test_command = "sh -c 'echo accuracy:0.45'"
        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
        self.run = RunCommand(self.cli_helper)
        self.run.parse(
            ["run", "--environment-paths", test_dockerfile, test_command])

        # test proper execution of task run command
        task_obj = self.run.execute()

        # test no visible snapshots
        self.snapshot_command.parse(["snapshot", "ls"])

        result = self.snapshot_command.execute()

        assert not result

        # test invisible snapshot from task_id was created
        self.snapshot_command.parse(["snapshot", "ls", "--all"])

        result = self.snapshot_command.execute()

        assert result
        assert task_obj.after_snapshot_id in [obj.id for obj in result]

    def test_snapshot_ls(self):
        self.__set_variables()

        # create a visible snapshot
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])

        created_snapshot_obj = self.snapshot_command.execute()

        # Test when optional parameters are not given
        self.snapshot_command.parse(["snapshot", "ls"])

        result = self.snapshot_command.execute()

        assert result
        assert created_snapshot_obj in result
        # Check if current snapshot id is the same as the latest created id
        current_snapshot_obj = self.snapshot_command.snapshot_controller.current_snapshot(
        )
        current_snapshot_id = current_snapshot_obj.id if current_snapshot_obj else None
        assert current_snapshot_id == created_snapshot_obj.id

        # Test when optional parameters are not given
        self.snapshot_command.parse(["snapshot", "ls", "--details"])

        result = self.snapshot_command.execute()

        assert result
        assert created_snapshot_obj in result

        # Test failure (format)
        failed = False
        try:
            self.snapshot_command.parse(["snapshot", "ls", "--format"])
        except ArgumentError:
            failed = True
        assert failed

        # Test success format csv
        self.snapshot_command.parse(["snapshot", "ls", "--format", "csv"])
        snapshot_objs = self.snapshot_command.execute()
        assert created_snapshot_obj in snapshot_objs

        # Test success format csv, download default
        self.snapshot_command.parse(
            ["snapshot", "ls", "--format", "csv", "--download"])
        snapshot_objs = self.snapshot_command.execute()
        assert created_snapshot_obj in snapshot_objs
        test_wildcard = os.path.join(
            self.snapshot_command.snapshot_controller.home, "snapshot_ls_*")
        paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
        assert paths
        assert open(paths[0], "r").read()
        os.remove(paths[0])

        # Test success format csv, download exact path
        test_path = os.path.join(self.temp_dir, "my_output")
        self.snapshot_command.parse([
            "snapshot", "ls", "--format", "csv", "--download",
            "--download-path", test_path
        ])
        snapshot_objs = self.snapshot_command.execute()
        assert created_snapshot_obj in snapshot_objs
        assert os.path.isfile(test_path)
        assert open(test_path, "r").read()
        os.remove(test_path)

        # Test success format table
        self.snapshot_command.parse(["snapshot", "ls"])
        environment_objs = self.snapshot_command.execute()
        assert created_snapshot_obj in snapshot_objs

        # Test success format table, download default
        self.snapshot_command.parse(["snapshot", "ls", "--download"])
        snapshot_objs = self.snapshot_command.execute()
        assert created_snapshot_obj in snapshot_objs
        test_wildcard = os.path.join(
            self.snapshot_command.snapshot_controller.home, "snapshot_ls_*")
        paths = [n for n in glob.glob(test_wildcard) if os.path.isfile(n)]
        assert paths
        assert open(paths[0], "r").read()
        os.remove(paths[0])

        # Test success format table, download exact path
        test_path = os.path.join(self.temp_dir, "my_output")
        self.snapshot_command.parse(
            ["snapshot", "ls", "--download", "--download-path", test_path])
        snapshot_objs = self.snapshot_command.execute()
        assert created_snapshot_obj in snapshot_objs
        assert os.path.isfile(test_path)
        assert open(test_path, "r").read()
        os.remove(test_path)

    def test_snapshot_checkout_invalid_arg(self):
        self.__set_variables()
        exception_thrown = False
        try:
            self.snapshot_command.parse(
                ["snapshot", "checkout"
                 "--foobar", "foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    def test_snapshot_checkout(self):
        self.__set_variables()
        # Test when optional parameters are not given
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])
        snapshot_obj = self.snapshot_command.execute()

        # Test when optional parameters are not given
        self.snapshot_command.parse(["snapshot", "checkout", snapshot_obj.id])

        result = self.snapshot_command.execute()
        assert result

    def test_snapshot_diff(self):
        self.__set_variables()

        # Create config file
        with open(self.config_filepath, "wb") as f:
            f.write(to_bytes(str('{"depth":6}')))

        # Create stats file
        with open(self.stats_filepath, "wb") as f:
            f.write(to_bytes(str('{"acc":0.97}')))

        # Create snapshots to test
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])
        snapshot_obj_1 = self.snapshot_command.execute()

        # Create another test file
        self.filepath_3 = os.path.join(self.snapshot_command.home, "file3.txt")
        with open(self.filepath_3, "wb") as f:
            f.write(to_bytes(str("test")))

        # Create config file
        with open(self.config_filepath, "wb") as f:
            f.write(to_bytes(str('{"depth":5}')))

        # Create stats file
        with open(self.stats_filepath, "wb") as f:
            f.write(to_bytes(str('{"acc":0.91}')))

        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my second snapshot"])
        snapshot_obj_2 = self.snapshot_command.execute()

        # Test diff with the above two snapshots
        self.snapshot_command.parse(
            ["snapshot", "diff", snapshot_obj_1.id, snapshot_obj_2.id])

        result = self.snapshot_command.execute()
        assert result
        assert "message" in result
        assert "label" in result
        assert "code_id" in result
        assert "environment_id" in result
        assert "file_collection_id" in result
        assert "config" in result
        assert "stats" in result

    def test_snapshot_inspect(self):
        self.__set_variables()
        # Create snapshot to test
        self.snapshot_command.parse(
            ["snapshot", "create", "-m", "my test snapshot"])
        snapshot_obj = self.snapshot_command.execute()
        # Test inspect for this snapshot
        self.snapshot_command.parse(["snapshot", "inspect", snapshot_obj.id])

        result = self.snapshot_command.execute()
        assert result
        assert "Code" in result
        assert "Environment" in result
        assert "Files" in result
        assert "Config" in result
        assert "Stats" in result
예제 #2
0
class TestFlow():
    def setup_method(self):
        self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
        Config().set_home(self.temp_dir)
        self.cli_helper = Helper()

    def teardown_method(self):
        pass

    def __set_variables(self):
        self.project_command = ProjectCommand(self.cli_helper)
        self.project_command.parse(
            ["init", "--name", "foobar", "--description", "test model"])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        dummy(self)
        self.environment_command = EnvironmentCommand(self.cli_helper)
        self.run_command = RunCommand(self.cli_helper)
        self.snapshot_command = SnapshotCommand(self.cli_helper)

        # Create test file
        self.filepath = os.path.join(self.snapshot_command.home, "file.txt")
        with open(self.filepath, "wb") as f:
            f.write(to_bytes(str("test")))

    def __environment_setup(self):
        self.environment_command.parse(["environment", "setup"])

        @self.environment_command.cli_helper.input("\n\n\n")
        def dummy(self):
            return self.environment_command.execute()

        environment_setup_result = dummy(self)
        return environment_setup_result

    def __run(self):
        test_command = ["sh", "-c", "echo accuracy:0.45"]
        test_ports = ["8888:8888"]
        self.run_command.parse(["run", "-p", test_ports[0], test_command])
        run_result = self.run_command.execute()
        return run_result

    def __run_ls(self):
        self.run_command.parse(["ls"])
        run_ls_result = self.run_command.execute()
        return run_ls_result

    def __snapshot_create(self):
        test_message = "creating a snapshot"
        self.snapshot_command.parse(["snapshot", "create", "-m", test_message])

        snapshot_create_result = self.snapshot_command.execute()
        return snapshot_create_result

    def __snapshot_ls(self):
        self.snapshot_command.parse(["snapshot", "ls"])
        snapshot_ls_result = self.snapshot_command.execute()
        return snapshot_ls_result

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_flow_1(self):
        # Flow
        # Step 1: environment setup
        # Step 2: run
        # Step 3: ls
        # Step 4: snapshot create
        # Step 5: snapshot ls
        self.__set_variables()

        # Step 1: environment setup
        environment_setup_result = self.__environment_setup()
        assert environment_setup_result

        # Step 2: run command
        run_result = self.__run()
        assert run_result

        # Step 3: ls
        run_ls_result = self.__run_ls()
        assert run_ls_result

        # Step 4: snapshot create
        snapshot_create_result = self.__snapshot_create()
        assert snapshot_create_result

        # Step 5: snapshot ls
        snapshot_ls_result = self.__snapshot_ls()
        assert snapshot_ls_result

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_flow_2(self):
        # Flow interruption in environment
        # Step 1: interrupted environment setup
        # Step 2: environment setup
        # Step 3: run
        # Step 4: ls
        # Step 5: snapshot create
        # Step 6: snapshot ls
        self.__set_variables()

        # TODO: TEST more fundamental functions first
        # Step 1: interrupted environment setup
        # @timeout_decorator.timeout(0.0001, use_signals=False)
        # def timed_command_with_interuption():
        #     result = self.__environment_setup()
        #     return result
        #
        # failed = False
        # try:
        #     timed_command_with_interuption()
        # except timeout_decorator.timeout_decorator.TimeoutError:
        #     failed = True
        # assert failed

        # Step 2: environment setup
        environment_setup_result = self.__environment_setup()
        assert environment_setup_result

        # Step 3: run
        run_result = self.__run()
        assert run_result

        # Step 4: ls
        run_ls_result = self.__run_ls()
        assert run_ls_result

        # Step 5: snapshot create
        snapshot_create_result = self.__snapshot_create()
        assert snapshot_create_result

        # Step 6: snapshot ls
        snapshot_ls_result = self.__snapshot_ls()
        assert snapshot_ls_result

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_flow_3(self):
        # Flow interruption in run
        # Step 1: environment setup
        # Step 2: interrupted run
        # Step 3: run
        # Step 4: ls
        # Step 5: snapshot create
        # Step 6: snapshot ls
        self.__set_variables()
        # Step 1: environment setup
        environment_setup_result = self.__environment_setup()
        assert environment_setup_result

        # TODO: Test more fundamental functions first
        # Step 2: interrupted run
        # @timeout_decorator.timeout(0.0001, use_signals=False)
        # def timed_command_with_interuption():
        #     result = self.__run()
        #     return result
        #
        # failed = False
        # try:
        #     timed_command_with_interuption()
        # except timeout_decorator.timeout_decorator.TimeoutError:
        #     failed = True
        # assert failed

        # Step 3: run
        run_result = self.__run()
        assert run_result

        # Step 4: task ls
        run_ls_result = self.__run_ls()
        assert run_ls_result

        # Step 5: snapshot create
        snapshot_create_result = self.__snapshot_create()
        assert snapshot_create_result

        # Step 6: snapshot ls
        snapshot_ls_result = self.__snapshot_ls()
        assert snapshot_ls_result

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_flow_4(self):
        # Flow interruption in snapshot create
        # Step 1: environment setup
        # Step 2: run
        # Step 3: ls
        # Step 4: interrupted snapshot create
        # Step 5: snapshot create
        # Step 6: snapshot ls
        self.__set_variables()

        # Step 1: environment setup
        environment_setup_result = self.__environment_setup()
        assert environment_setup_result

        # Step 2: run
        run_result = self.__run()
        assert run_result

        # Step 3: ls
        run_ls_result = self.__run_ls()
        assert run_ls_result

        # TODO: Test more fundamental functions first
        # Step 4: interrupted snapshot create
        # @timeout_decorator.timeout(0.0001, use_signals=False)
        # def timed_command_with_interuption():
        #     result = self.__snapshot_create()
        #     return result
        #
        # failed = False
        # try:
        #     timed_command_with_interuption()
        # except timeout_decorator.timeout_decorator.TimeoutError:
        #     failed = True
        # assert failed
        # snapshot_ls_result = self.__snapshot_ls()
        # assert not snapshot_ls_result

        # Step 5: snapshot create
        snapshot_create_result = self.__snapshot_create()
        assert snapshot_create_result

        # Step 6: snapshot ls
        snapshot_ls_result = self.__snapshot_ls()
        assert snapshot_ls_result
예제 #3
0
class TestProjectCommand():
    def setup_method(self):
        self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
        Config().set_home(self.temp_dir)
        self.cli_helper = Helper()
        self.project_command = ProjectCommand(self.cli_helper)

    def teardown_method(self):
        pass

    def test_init_create_success_default_name_no_description_no_environment(
            self):
        self.project_command.parse(["init"])

        # Test when environment is created
        @self.project_command.cli_helper.input("\n\ny\n\n\n\n")
        def dummy(self):
            return self.project_command.execute()

        result = dummy(self)

        # Ensure that the name and description are current
        _, default_name = os.path.split(
            self.project_command.project_controller.home)
        assert result
        assert result.name == default_name
        assert result.description == None
        # Ensure environment is correct
        definition_filepath = os.path.join(
            self.temp_dir, self.project_command.project_controller.file_driver.
            environment_directory, "Dockerfile")
        assert os.path.isfile(definition_filepath)
        assert "FROM datmo/python-base:cpu-py27" in open(
            definition_filepath, "r").read()

    def test_init_create_success_no_environment(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])

        # Test when environment is not created
        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        result = dummy(self)

        definition_filepath = os.path.join(
            self.temp_dir, self.project_command.project_controller.file_driver.
            environment_directory, "Dockerfile")

        assert result
        assert not os.path.isfile(definition_filepath)
        assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
        assert result.name == test_name
        assert result.description == test_description

    def test_init_create_success_environment(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])
        # Test when environment is created
        @self.project_command.cli_helper.input("y\n\n\n\n")
        def dummy(self):
            return self.project_command.execute()

        result = dummy(self)

        definition_filepath = os.path.join(
            self.temp_dir, self.project_command.project_controller.file_driver.
            environment_directory, "Dockerfile")

        assert result
        assert os.path.isfile(definition_filepath)
        assert "FROM datmo/python-base:cpu-py27" in open(
            definition_filepath, "r").read()

        # test for desired side effects
        assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
        assert result.name == test_name
        assert result.description == test_description

    def test_init_create_success_blank(self):
        self.project_command.parse(["init", "--name", "", "--description", ""])
        # test if prompt opens
        @self.project_command.cli_helper.input("\n\n\n")
        def dummy(self):
            return self.project_command.execute()

        result = dummy(self)
        assert result
        assert result.name
        assert not result.description

    def test_init_update_success(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        result_1 = dummy(self)
        updated_name = "foobar2"
        updated_description = "test model 2"
        self.project_command.parse([
            "init", "--name", updated_name, "--description",
            updated_description
        ])

        result_2 = dummy(self)
        # test for desired side effects
        assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
        assert result_2.id == result_1.id
        assert result_2.name == updated_name
        assert result_2.description == updated_description

    def test_init_update_success_only_name(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        result_1 = dummy(self)
        updated_name = "foobar2"
        self.project_command.parse(
            ["init", "--name", updated_name, "--description", ""])

        @self.project_command.cli_helper.input("\n\n")
        def dummy(self):
            return self.project_command.execute()

        result_2 = dummy(self)
        # test for desired side effects
        assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
        assert result_2.id == result_1.id
        assert result_2.name == updated_name
        assert result_2.description == result_1.description

    def test_init_update_success_only_description(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        result_1 = dummy(self)
        updated_description = "test model 2"
        self.project_command.parse(
            ["init", "--name", "", "--description", updated_description])

        @self.project_command.cli_helper.input("\n\n")
        def dummy(self):
            return self.project_command.execute()

        result_2 = dummy(self)
        # test for desired side effects
        assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
        assert result_2.id == result_1.id
        assert result_2.name == result_1.name
        assert result_2.description == updated_description

    def test_init_invalid_arg(self):
        exception_thrown = False
        try:
            self.project_command.parse(["init", "--foobar", "foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    def test_version(self):
        self.project_command.parse(["version"])
        result = self.project_command.execute()
        # test for desired side effects
        assert __version__ in result

    def test_version_invalid_arg(self):
        exception_thrown = False
        try:
            self.project_command.parse(["version", "--foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    def test_status_basic(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        _ = dummy(self)

        self.project_command.parse(["status"])
        result = self.project_command.execute()
        status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = result
        assert isinstance(status_dict, dict)
        assert not current_snapshot
        assert not latest_snapshot_user_generated
        assert not latest_snapshot_auto_generated
        assert unstaged_code
        assert not unstaged_environment
        assert not unstaged_files

    def test_status_user_generated_snapshot(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        _ = dummy(self)

        # Create a snapshot
        self.snapshot_command = SnapshotCommand(self.cli_helper)
        with open(os.path.join(self.project_command.home, "test.py"),
                  "wb") as f:
            f.write(to_bytes(str("import xgboost")))
        self.snapshot_command.parse(
            ["snapshot", "create", "--message", "test"])
        snapshot_obj = self.snapshot_command.execute()

        self.project_command.parse(["status"])
        result = self.project_command.execute()
        status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = result
        assert isinstance(status_dict, dict)
        assert isinstance(current_snapshot, Snapshot)
        assert isinstance(latest_snapshot_user_generated, Snapshot)
        assert snapshot_obj == latest_snapshot_user_generated
        assert current_snapshot == latest_snapshot_user_generated
        assert not latest_snapshot_auto_generated
        assert not unstaged_code
        assert not unstaged_environment
        assert not unstaged_files

    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
    def test_status_autogenerated_snapshot(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        _ = dummy(self)

        self.run_command = RunCommand(self.cli_helper)
        self.task_controller = TaskController()

        # Create and run a task and test if task is shown
        test_command = ["sh", "-c", "echo accuracy:0.45"]
        self.run_command.parse(["run", test_command])

        updated_first_task = self.run_command.execute()
        before_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
            updated_first_task.before_snapshot_id)
        after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
            updated_first_task.after_snapshot_id)
        before_environment_obj = self.task_controller.dal.environment.get_by_id(
            before_snapshot_obj.environment_id)
        after_environment_obj = self.task_controller.dal.environment.get_by_id(
            after_snapshot_obj.environment_id)
        assert before_environment_obj == after_environment_obj

        self.project_command.parse(["status"])
        result = self.project_command.execute()
        status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
            result

        assert status_dict
        assert isinstance(status_dict, dict)
        assert status_dict['name'] == test_name
        assert status_dict['description'] == test_description
        assert isinstance(status_dict['config'], dict)
        assert isinstance(current_snapshot, Snapshot)
        assert not latest_snapshot_user_generated
        assert isinstance(latest_snapshot_auto_generated, Snapshot)
        # current snapshot is the before snapshot for the run
        assert current_snapshot == before_snapshot_obj
        assert current_snapshot != latest_snapshot_auto_generated
        # latest autogenerated snapshot is the after snapshot id
        assert latest_snapshot_auto_generated == after_snapshot_obj
        # autogenerated snapshot is after the user generated snapshot
        assert not unstaged_code
        assert not unstaged_environment
        assert not unstaged_files

        # Create a snapshot
        self.snapshot_command = SnapshotCommand(self.cli_helper)
        with open(os.path.join(self.project_command.home, "test.py"),
                  "wb") as f:
            f.write(to_bytes(str("import xgboost")))
        self.snapshot_command.parse(
            ["snapshot", "create", "--message", "test"])
        first_snapshot = self.snapshot_command.execute()

        # Create and run a task and test if task is shown
        test_command = ["sh", "-c", "echo accuracy:0.45"]
        self.run_command.parse(["run", test_command])

        updated_second_task = self.run_command.execute()
        before_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
            updated_second_task.before_snapshot_id)
        after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
            updated_second_task.after_snapshot_id)
        before_environment_obj = self.task_controller.dal.environment.get_by_id(
            before_snapshot_obj.environment_id)
        after_environment_obj = self.task_controller.dal.environment.get_by_id(
            after_snapshot_obj.environment_id)
        assert before_environment_obj == after_environment_obj

        self.project_command.parse(["status"])
        result = self.project_command.execute()
        status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
            result

        assert status_dict
        assert isinstance(status_dict, dict)
        assert status_dict['name'] == test_name
        assert status_dict['description'] == test_description
        assert isinstance(status_dict['config'], dict)
        assert isinstance(current_snapshot, Snapshot)
        assert isinstance(latest_snapshot_user_generated, Snapshot)
        assert latest_snapshot_user_generated == first_snapshot
        assert isinstance(latest_snapshot_auto_generated, Snapshot)
        # current snapshot is the before snapshot for the run
        assert current_snapshot == before_snapshot_obj
        assert current_snapshot == latest_snapshot_user_generated
        assert current_snapshot != latest_snapshot_auto_generated
        # latest autogenerated snapshot is the after snapshot id
        assert latest_snapshot_auto_generated == after_snapshot_obj
        assert latest_snapshot_auto_generated != latest_snapshot_user_generated
        # user generated snapshot is not associated with any before or after snapshot
        assert latest_snapshot_user_generated == before_snapshot_obj
        assert latest_snapshot_user_generated != after_snapshot_obj
        # autogenerated snapshot is after the user generated snapshot
        assert latest_snapshot_auto_generated.created_at > latest_snapshot_user_generated.created_at
        assert not unstaged_code
        assert not unstaged_environment
        assert not unstaged_files

        # Check if the same snapshot is given back when created
        self.snapshot_command.parse(
            ["snapshot", "create", "--message", "test"])
        new_snapshot = self.snapshot_command.execute()
        assert current_snapshot == new_snapshot

        # Create a user generated snapshot after some changes
        with open(os.path.join(self.project_command.home, "new.py"),
                  "wb") as f:
            f.write(to_bytes(str("import xgboost")))
        self.snapshot_command.parse(
            ["snapshot", "create", "--message", "test"])
        new_snapshot = self.snapshot_command.execute()
        # ensure we created a new snapshot
        assert current_snapshot != new_snapshot

        self.project_command.parse(["status"])
        result = self.project_command.execute()
        status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
            result

        assert status_dict
        assert isinstance(status_dict, dict)
        assert status_dict['name'] == test_name
        assert status_dict['description'] == test_description
        assert isinstance(status_dict['config'], dict)
        assert isinstance(current_snapshot, Snapshot)
        assert isinstance(latest_snapshot_user_generated, Snapshot)
        # current snapshot is the latest user generated one
        assert current_snapshot == latest_snapshot_user_generated
        # latest autogenerated snapshot is not the user generated one
        assert latest_snapshot_auto_generated != latest_snapshot_user_generated
        # autogenerated snapshot is after the user generated snapshot
        assert latest_snapshot_user_generated.created_at > latest_snapshot_auto_generated.created_at
        assert not unstaged_code
        assert not unstaged_environment
        assert not unstaged_files

        # Create a snapshot from the auto generated one and ensure they are the same
        self.run_command = RunCommand(self.cli_helper)
        self.task_controller = TaskController()

        test_command = ["sh", "-c", "echo accuracy:0.45"]
        self.run_command.parse(["run", test_command])
        updated_third_task = self.run_command.execute()
        before_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
            updated_third_task.before_snapshot_id)
        after_snapshot_obj = self.task_controller.dal.snapshot.get_by_id(
            updated_third_task.after_snapshot_id)
        self.snapshot_command.parse([
            "snapshot", "create", "--run-id", updated_third_task.id,
            "--message", "test"
        ])
        converted_snapshot = self.snapshot_command.execute()

        self.project_command.parse(["status"])
        result = self.project_command.execute()
        status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
            result

        assert status_dict
        assert isinstance(status_dict, dict)
        assert status_dict['name'] == test_name
        assert status_dict['description'] == test_description
        assert isinstance(status_dict['config'], dict)
        assert isinstance(current_snapshot, Snapshot)
        assert isinstance(latest_snapshot_user_generated, Snapshot)
        assert isinstance(latest_snapshot_auto_generated, Snapshot)
        # current snapshot is the before snapshot for the run
        assert current_snapshot == before_snapshot_obj
        assert current_snapshot != latest_snapshot_user_generated
        # latest user generated snapshot is the same as that created by the run
        assert converted_snapshot == latest_snapshot_user_generated
        assert latest_snapshot_user_generated == after_snapshot_obj
        # latest user generated converted the latest task so latest generate is from the last task
        assert latest_snapshot_user_generated.created_at > latest_snapshot_auto_generated.created_at
        assert not unstaged_code
        assert not unstaged_environment
        assert not unstaged_files

    def test_status_invalid_arg(self):
        exception_thrown = False
        try:
            self.project_command.parse(["status", "--foobar"])
        except UnrecognizedCLIArgument:
            exception_thrown = True
        assert exception_thrown

    def test_cleanup(self):
        test_name = "foobar"
        test_description = "test model"
        self.project_command.parse(
            ["init", "--name", test_name, "--description", test_description])

        @self.project_command.cli_helper.input("\n")
        def dummy(self):
            return self.project_command.execute()

        _ = dummy(self)

        self.project_command.parse(["cleanup"])

        @self.project_command.cli_helper.input("y\n\n")
        def dummy(self):
            return self.project_command.execute()

        result = dummy(self)
        assert not os.path.exists(os.path.join(self.temp_dir, '.datmo'))
        assert isinstance(result, bool)
        assert result

    def test_cleanup_invalid_arg(self):
        exception_thrown = False
        try:
            self.project_command.parse(["cleanup", "--foobar"])
        except UnrecognizedCLIArgument:
            exception_thrown = True
        assert exception_thrown
예제 #4
0
class TestSnapshot():
    def setup_class(self):
        # provide mountable tmp directory for docker
        tempfile.tempdir = "/tmp" if not platform.system(
        ) == "Windows" else None
        test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
                                        tempfile.gettempdir())
        self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
        self.cli_helper = Helper()

    def teardown_class(self):
        pass

    def __set_variables(self):
        self.project = ProjectCommand(self.temp_dir, self.cli_helper)
        self.project.parse(
            ["init", "--name", "foobar", "--description", "test model"])
        self.project.execute()
        self.snapshot = SnapshotCommand(self.temp_dir, self.cli_helper)

        # Create environment_driver definition
        self.env_def_path = os.path.join(self.temp_dir, "Dockerfile")
        with open(self.env_def_path, "w") as f:
            f.write(to_unicode(str("FROM datmo/xgboost:cpu")))

        # Create config
        self.config_filepath = os.path.join(self.snapshot.home, "config.json")
        with open(self.config_filepath, "w") as f:
            f.write(to_unicode(str("{}")))

        # Create stats
        self.stats_filepath = os.path.join(self.snapshot.home, "stats.json")
        with open(self.stats_filepath, "w") as f:
            f.write(to_unicode(str("{}")))

        # Create test file
        self.filepath = os.path.join(self.snapshot.home, "file.txt")
        with open(self.filepath, "w") as f:
            f.write(to_unicode(str("test")))

        # Create another test file
        self.filepath_2 = os.path.join(self.snapshot.home, "file2.txt")
        with open(self.filepath_2, "w") as f:
            f.write(to_unicode(str("test")))

    def test_snapshot_project_not_init(self):
        failed = False
        try:
            self.snapshot = SnapshotCommand(self.temp_dir, self.cli_helper)
        except ProjectNotInitializedException:
            failed = True
        assert failed

    def test_snapshot_help(self):
        self.__set_variables()
        print(
            "\n====================================== datmo snapshot ==========================\n"
        )

        self.snapshot.parse(["snapshot"])
        assert self.snapshot.execute()

        print(
            "\n====================================== datmo snapshot --help ==========================\n"
        )

        self.snapshot.parse(["snapshot", "--help"])
        assert self.snapshot.execute()

        print(
            "\n====================================== datmo snapshot create --help ==========================\n"
        )

        self.snapshot.parse(["snapshot", "create", "--help"])
        assert self.snapshot.execute()

    def test_snapshot_command(self):
        self.__set_variables()
        test_message = "this is a test message"
        test_label = "test label"
        test_session_id = "test_session_id"
        test_task_id = "test_task_id"
        test_code_id = "test_code_id"
        test_environment_definition_filepath = self.env_def_path
        test_config_filepath = self.config_filepath
        test_stats_filepath = self.config_filepath
        test_filepaths = [self.filepath, self.filepath_2]

        # try single filepath
        self.snapshot.parse([
            "snapshot", "create", "--message", test_message, "--task-id",
            test_task_id
        ])

        # testing for proper parsing
        assert self.snapshot.args.message == test_message
        assert self.snapshot.args.task_id == test_task_id

        # try single filepath
        self.snapshot.parse([
            "snapshot",
            "create",
            "--message",
            test_message,
            "--label",
            test_label,
            "--session-id",
            test_session_id,
            "--code-id",
            test_code_id,
            "--environment-def",
            test_environment_definition_filepath,
            "--config-filepath",
            test_config_filepath,
            "--stats-filepath",
            test_stats_filepath,
            "--filepaths",
            test_filepaths[0],
        ])

        # test for desired side effects
        assert self.snapshot.args.message == test_message
        assert self.snapshot.args.label == test_label
        assert self.snapshot.args.session_id == test_session_id
        assert self.snapshot.args.code_id == test_code_id
        assert self.snapshot.args.environment_definition_filepath == test_environment_definition_filepath
        assert self.snapshot.args.config_filepath == test_config_filepath
        assert self.snapshot.args.stats_filepath == test_stats_filepath
        assert self.snapshot.args.filepaths == [test_filepaths[0]]

        # test multiple filepaths
        self.snapshot.parse([
            "snapshot", "create", "--message", test_message, "--label",
            test_label, "--session-id", test_session_id, "--code-id",
            test_code_id, "--environment-def",
            test_environment_definition_filepath, "--config-filepath",
            test_config_filepath, "--stats-filepath", test_stats_filepath,
            "--filepaths", test_filepaths[0], "--filepaths", test_filepaths[1]
        ])

        # test for desired side effects
        assert self.snapshot.args.message == test_message
        assert self.snapshot.args.label == test_label
        assert self.snapshot.args.session_id == test_session_id
        assert self.snapshot.args.code_id == test_code_id
        assert self.snapshot.args.environment_definition_filepath == test_environment_definition_filepath
        assert self.snapshot.args.config_filepath == test_config_filepath
        assert self.snapshot.args.stats_filepath == test_stats_filepath
        assert self.snapshot.args.filepaths == test_filepaths

        snapshot_id_1 = self.snapshot.execute()
        assert snapshot_id_1

    def test_datmo_snapshot_create_from_task(self):
        self.__set_variables()
        test_message = "this is a test message"
        test_code_id = "test_code_id"

        # create task
        test_command = "sh -c 'echo accuracy:0.45'"
        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
        self.task = TaskCommand(self.temp_dir, self.cli_helper)
        self.task.parse([
            "task", "run", "--environment-def", test_dockerfile, test_command
        ])

        # test proper execution of task run command
        task_obj = self.task.execute()

        task_id = task_obj.id

        # test task id
        self.snapshot.parse([
            "snapshot", "create", "--message", test_message, "--task-id",
            task_id
        ])

        # test for desired side effects
        assert self.snapshot.args.message == test_message

        snapshot_id = self.snapshot.execute()
        assert snapshot_id

    def test_snapshot_create_from_task_fail_user_inputs(self):
        self.__set_variables()
        test_message = "this is a test message"

        # create task
        test_command = "sh -c 'echo accuracy:0.45'"
        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
        self.task = TaskCommand(self.temp_dir, self.cli_helper)
        self.task.parse([
            "task", "run", "--environment-def", test_dockerfile, test_command
        ])

        # test proper execution of task run command
        task_obj = self.task.execute()

        task_id = task_obj.id

        failed = False
        try:
            # test task id with code id
            self.snapshot.parse([
                "snapshot", "create", "--message", test_message, "--task-id",
                task_id, "--code-id", "test_code_id"
            ])
            _ = self.snapshot.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test task id with code id
            self.snapshot.parse([
                "snapshot", "create", "--message", test_message, "--task-id",
                task_id, "--commit-id", "test_commit_id"
            ])
            _ = self.snapshot.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test task id with code id
            self.snapshot.parse([
                "snapshot", "create", "--message", test_message, "--task-id",
                task_id, "--environment-id", "test_environment_id"
            ])
            _ = self.snapshot.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test task id with code id
            self.snapshot.parse([
                "snapshot", "create", "--message", test_message, "--task-id",
                task_id, "--environment-def", "test_environment_def"
            ])
            _ = self.snapshot.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test task id with code id
            self.snapshot.parse([
                "snapshot", "create", "--message", test_message, "--task-id",
                task_id, "--file-collection-id", "test_file_collection_id"
            ])
            _ = self.snapshot.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test task id with code id
            self.snapshot.parse([
                "snapshot", "create", "--message", test_message, "--task-id",
                task_id, "--filepaths", "mypath"
            ])
            _ = self.snapshot.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test task id with code id
            self.snapshot.parse([
                "snapshot", "create", "--message", test_message, "--task-id",
                task_id, "--config-filepath", "mypath"
            ])
            _ = self.snapshot.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test task id with code id
            self.snapshot.parse([
                "snapshot", "create", "--message", test_message, "--task-id",
                task_id, "--config-filename", "myname"
            ])
            _ = self.snapshot.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test task id with code id
            self.snapshot.parse([
                "snapshot", "create", "--message", test_message, "--task-id",
                task_id, "--stats-filepath", "mypath"
            ])
            _ = self.snapshot.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

        failed = False
        try:
            # test task id with code id
            self.snapshot.parse([
                "snapshot", "create", "--message", test_message, "--task-id",
                task_id, "--stats-filename", "myname"
            ])
            _ = self.snapshot.execute()
        except SnapshotCreateFromTaskArgs:
            failed = True
        assert failed

    def test_datmo_snapshot_create_fail_mutually_exclusive_args(self):
        self.__set_variables()
        test_message = "this is a test message"
        test_label = "test label"
        test_session_id = "test_session_id"
        test_code_id = "test_code_id"
        test_commit_id = "test_commit_id"
        test_environment_id = "test_environment_id"
        test_environment_definition_filepath = self.env_def_path
        test_file_collection_id = "test_file_collection_id"
        test_filepaths = [self.filepath]
        test_config_filename = "config.json"
        test_config_filepath = self.config_filepath
        test_stats_filename = "stats.json"
        test_stats_filepath = self.config_filepath

        # Code exception
        exception_thrown = False
        try:
            self.snapshot.parse([
                "snapshot", "create", "--message", test_message, "--label",
                test_label, "--session-id", test_session_id, "--code-id",
                test_code_id, "--commit-id", test_commit_id
            ])
            _ = self.snapshot.execute()
        except MutuallyExclusiveArguments:
            exception_thrown = True
        assert exception_thrown

        # Environment exception
        exception_thrown = False
        try:
            self.snapshot.parse([
                "snapshot",
                "create",
                "--message",
                test_message,
                "--label",
                test_label,
                "--session-id",
                test_session_id,
                "--environment-id",
                test_environment_id,
                "--environment-def",
                test_environment_definition_filepath,
            ])
            _ = self.snapshot.execute()
        except MutuallyExclusiveArguments:
            exception_thrown = True
        assert exception_thrown

        # File exception
        exception_thrown = False
        try:
            self.snapshot.parse([
                "snapshot",
                "create",
                "--message",
                test_message,
                "--label",
                test_label,
                "--session-id",
                test_session_id,
                "--file-collection-id",
                test_file_collection_id,
                "--filepaths",
                test_filepaths[0],
            ])
            _ = self.snapshot.execute()
        except MutuallyExclusiveArguments:
            exception_thrown = True
        assert exception_thrown

        # Config exception
        exception_thrown = False
        try:
            self.snapshot.parse([
                "snapshot",
                "create",
                "--message",
                test_message,
                "--label",
                test_label,
                "--session-id",
                test_session_id,
                "--config-filename",
                test_config_filename,
                "--config-filepath",
                test_config_filepath,
            ])
            _ = self.snapshot.execute()
        except MutuallyExclusiveArguments:
            exception_thrown = True
        assert exception_thrown

        # Stats exception
        exception_thrown = False
        try:
            self.snapshot.parse([
                "snapshot",
                "create",
                "--message",
                test_message,
                "--label",
                test_label,
                "--session-id",
                test_session_id,
                "--stats-filename",
                test_stats_filename,
                "--stats-filepath",
                test_stats_filepath,
            ])
            _ = self.snapshot.execute()
        except MutuallyExclusiveArguments:
            exception_thrown = True
        assert exception_thrown

    def test_datmo_snapshot_create_default(self):
        self.__set_variables()
        self.snapshot.parse(["snapshot", "create", "-m", "my test snapshot"])

        snapshot_id_2 = self.snapshot.execute()
        assert snapshot_id_2

    def test_datmo_snapshot_create_invalid_arg(self):
        self.__set_variables()
        exception_thrown = False
        try:
            self.snapshot.parse(["snapshot", "create" "--foobar", "foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    def test_datmo_snapshot_delete(self):
        self.__set_variables()

        # Test when optional parameters are not given
        self.snapshot.parse(["snapshot", "create", "-m", "my test snapshot"])

        snapshot_id = self.snapshot.execute()

        # Test when optional parameters are not given
        self.snapshot.parse(["snapshot", "delete", "--id", snapshot_id])

        result = self.snapshot.execute()
        assert result

    def test_datmo_snapshot_delete_invalid_arg(self):
        self.__set_variables()
        exception_thrown = False
        try:
            self.snapshot.parse(["snapshot", "delete" "--foobar", "foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    def test_datmo_snapshot_ls(self):
        self.__set_variables()
        # Test when optional parameters are not given
        self.snapshot.parse(["snapshot", "create", "-m", "my test snapshot"])

        snapshot_id = self.snapshot.execute()

        # Test when optional parameters are not given
        self.snapshot.parse(["snapshot", "ls"])

        result = self.snapshot.execute()

        assert result
        assert snapshot_id in result

        # Test when optional parameters are not given
        self.snapshot.parse(["snapshot", "ls", "-a"])

        result = self.snapshot.execute()

        assert result
        assert snapshot_id in result

    def test_datmo_snapshot_checkout_invalid_arg(self):
        self.__set_variables()
        exception_thrown = False
        try:
            self.snapshot.parse(["snapshot", "checkout" "--foobar", "foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    def test_datmo_snapshot_checkout(self):
        self.__set_variables()
        # Test when optional parameters are not given
        self.snapshot.parse(["snapshot", "create", "-m", "my test snapshot"])
        snapshot_id = self.snapshot.execute()

        # remove datmo_task folder to have no changes before checkout
        datmo_tasks_dirpath = os.path.join(self.snapshot.home, "datmo_tasks")
        shutil.rmtree(datmo_tasks_dirpath)

        # Test when optional parameters are not given
        self.snapshot.parse(["snapshot", "checkout", "--id", snapshot_id])

        result = self.snapshot.execute()
        assert result
예제 #5
0
class TestSnapshot():
    def setup_class(self):
        # provide mountable tmp directory for docker
        tempfile.tempdir = "/tmp" if not platform.system() == "Windows" else None
        test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
                                        tempfile.gettempdir())
        self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
        self.cli_helper = Helper()

    def teardown_class(self):
        pass

    def __set_variables(self):
        self.init = ProjectCommand(self.temp_dir, self.cli_helper)
        self.init.parse([
            "init",
            "--name", "foobar",
            "--description", "test model"])
        self.init.execute()
        self.snapshot = SnapshotCommand(self.temp_dir, self.cli_helper)

        # Create environment_driver definition
        self.env_def_path = os.path.join(self.temp_dir,
                                         "Dockerfile")
        with open(self.env_def_path, "w") as f:
            f.write(to_unicode(str("FROM datmo/xgboost:cpu")))

        # Create config
        self.config_filepath = os.path.join(self.snapshot.home,
                                       "config.json")
        with open(self.config_filepath, "w") as f:
            f.write(to_unicode(str("{}")))

        # Create stats
        self.stats_filepath = os.path.join(self.snapshot.home,
                                      "stats.json")
        with open(self.stats_filepath, "w") as f:
            f.write(to_unicode(str("{}")))

        # Create test file
        self.filepath = os.path.join(self.snapshot.home,
                                           "file.txt")
        with open(self.filepath, "w") as f:
            f.write(to_unicode(str("test")))

    def test_snapshot_project_not_init(self):
        failed = False
        try:
            self.snapshot = SnapshotCommand(self.temp_dir, self.cli_helper)
        except ProjectNotInitializedException:
            failed = True
        assert failed

    def test_datmo_snapshot_create(self):
        self.__set_variables()
        test_message = "this is a test message"
        test_label = "test label"
        test_session_id = "test_session_id"
        test_task_id = "test_task_id"
        test_code_id = "test_code_id"
        test_environment_def_path = self.env_def_path
        test_config_filepath = self.config_filepath
        test_stats_filepath = self.config_filepath
        test_filepaths = [self.filepath]

        self.snapshot.parse([
            "snapshot",
            "create",
            "--message", test_message,
            "--label", test_label,
            "--session-id", test_session_id,
            "--task-id", test_task_id,
            "--code-id", test_code_id,
            "--environment-def-path", test_environment_def_path,
            "--config-filepath", test_config_filepath,
            "--stats-filepath", test_stats_filepath,
            "--filepaths", test_filepaths[0],
        ])

        # test for desired side effects
        assert self.snapshot.args.message == test_message
        assert self.snapshot.args.label == test_label
        assert self.snapshot.args.session_id == test_session_id
        assert self.snapshot.args.task_id == test_task_id
        assert self.snapshot.args.code_id == test_code_id
        assert self.snapshot.args.environment_def_path == test_environment_def_path
        assert self.snapshot.args.config_filepath == test_config_filepath
        assert self.snapshot.args.stats_filepath == test_stats_filepath
        assert self.snapshot.args.filepaths == test_filepaths

        snapshot_id_1 = self.snapshot.execute()
        assert snapshot_id_1

    def test_datmo_snapshot_create_fail_mutually_exclusive_args(self):
        self.__set_variables()
        test_message = "this is a test message"
        test_label = "test label"
        test_session_id = "test_session_id"
        test_task_id = "test_task_id"
        test_code_id = "test_code_id"
        test_commit_id = "test_commit_id"
        test_environment_id = "test_environment_id"
        test_environment_def_path = self.env_def_path
        test_file_collection_id = "test_file_collection_id"
        test_filepaths = [self.filepath]
        test_config_filename = "config.json"
        test_config_filepath = self.config_filepath
        test_stats_filename = "stats.json"
        test_stats_filepath = self.config_filepath

        # Code exception
        exception_thrown = False
        try:
            self.snapshot.parse([
                "snapshot",
                "create",
                "--message", test_message,
                "--label", test_label,
                "--session-id", test_session_id,
                "--task-id", test_task_id,
                "--code-id", test_code_id,
                "--commit-id", test_commit_id
            ])
            _ = self.snapshot.execute()
        except MutuallyExclusiveArguments:
            exception_thrown = True
        assert exception_thrown

        # Environment exception
        exception_thrown = False
        try:
            self.snapshot.parse([
                "snapshot",
                "create",
                "--message", test_message,
                "--label", test_label,
                "--session-id", test_session_id,
                "--task-id", test_task_id,
                "--environment-id", test_environment_id,
                "--environment-def-path", test_environment_def_path,
            ])
            _ = self.snapshot.execute()
        except MutuallyExclusiveArguments:
            exception_thrown = True
        assert exception_thrown

        # File exception
        exception_thrown = False
        try:
            self.snapshot.parse([
                "snapshot",
                "create",
                "--message", test_message,
                "--label", test_label,
                "--session-id", test_session_id,
                "--task-id", test_task_id,
                "--file-collection-id", test_file_collection_id,
                "--filepaths", test_filepaths[0],
            ])
            _ = self.snapshot.execute()
        except MutuallyExclusiveArguments:
            exception_thrown = True
        assert exception_thrown

        # Config exception
        exception_thrown = False
        try:
            self.snapshot.parse([
                "snapshot",
                "create",
                "--message", test_message,
                "--label", test_label,
                "--session-id", test_session_id,
                "--task-id", test_task_id,
                "--config-filename", test_config_filename,
                "--config-filepath", test_config_filepath,
            ])
            _ = self.snapshot.execute()
        except MutuallyExclusiveArguments:
            exception_thrown = True
        assert exception_thrown

        # Stats exception
        exception_thrown = False
        try:
            self.snapshot.parse([
                "snapshot",
                "create",
                "--message", test_message,
                "--label", test_label,
                "--session-id", test_session_id,
                "--task-id", test_task_id,
                "--stats-filename", test_stats_filename,
                "--stats-filepath", test_stats_filepath,
            ])
            _ = self.snapshot.execute()
        except MutuallyExclusiveArguments:
            exception_thrown = True
        assert exception_thrown


    def test_datmo_snapshot_create_default(self):
        self.__set_variables()
        self.snapshot.parse([
            "snapshot",
            "create",
            "-m", "my test snapshot"
        ])

        snapshot_id_2 = self.snapshot.execute()
        assert snapshot_id_2

    def test_datmo_snapshot_create_invalid_arg(self):
        self.__set_variables()
        exception_thrown = False
        try:
          self.snapshot.parse([
            "snapshot",
            "create"
            "--foobar","foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    def test_datmo_snapshot_delete(self):
        self.__set_variables()

        # Test when optional parameters are not given
        self.snapshot.parse([
            "snapshot",
            "create",
            "-m", "my test snapshot"
        ])

        snapshot_id = self.snapshot.execute()

        # Test when optional parameters are not given
        self.snapshot.parse([
            "snapshot",
            "delete",
            "--id", snapshot_id
        ])

        result = self.snapshot.execute()
        assert result

    def test_datmo_snapshot_delete_invalid_arg(self):
        self.__set_variables()
        exception_thrown = False
        try:
          self.snapshot.parse([
            "snapshot",
            "delete"
            "--foobar","foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    def test_datmo_snapshot_ls(self):
        self.__set_variables()

        # Test when optional parameters are not given
        self.snapshot.parse([
            "snapshot",
            "create",
            "-m", "my test snapshot"
        ])

        self.snapshot.execute()

        # Test when optional parameters are not given
        self.snapshot.parse([
            "snapshot",
            "ls"
        ])

        result = self.snapshot.execute()

        assert result

        # Test when optional parameters are not given
        self.snapshot.parse([
            "snapshot",
            "ls", "-a"
        ])

        result = self.snapshot.execute()

        assert result

    def test_datmo_snapshot_checkout_invalid_arg(self):
        self.__set_variables()
        exception_thrown = False
        try:
          self.snapshot.parse([
            "snapshot",
            "checkout"
            "--foobar","foobar"])
        except Exception:
            exception_thrown = True
        assert exception_thrown

    def test_datmo_snapshot_checkout(self):
        self.__set_variables()

        # Test when optional parameters are not given
        self.snapshot.parse([
            "snapshot",
            "create",
            "-m", "my test snapshot"
        ])

        snapshot_id = self.snapshot.execute()

        # Test when optional parameters are not given
        self.snapshot.parse([
            "snapshot",
            "checkout",
            "--id", snapshot_id
        ])

        result = self.snapshot.execute()
        assert result