Esempio n. 1
0
    def test_get_by_id_model_same_dir(self):
        test_dir = "test-dir"
        datadriver = BlitzDBDALDriver("file", test_dir)
        dal = LocalDAL(datadriver)
        model1 = dal.model.create(Model(self.model_input_dict))
        del datadriver
        del dal
        datadriver = BlitzDBDALDriver("file", test_dir)
        dal = LocalDAL(datadriver)
        model2 = dal.model.create(Model(self.model_input_dict))
        del datadriver
        del dal
        datadriver = BlitzDBDALDriver("file", test_dir)
        dal = LocalDAL(datadriver)
        model3 = dal.model.create(Model(self.model_input_dict))

        model1 = dal.model.get_by_id(model1.id)
        model2 = dal.model.get_by_id(model2.id)
        model3 = dal.model.get_by_id(model3.id)

        assert model1
        assert model2
        assert model3

        shutil.rmtree(test_dir)
Esempio n. 2
0
 def test_query_models_range_query(self):
     _ = self.dal.model.create(Model(self.model_input_dict))
     _ = self.dal.model.create(Model(self.model_input_dict))
     _ = self.dal.model.create(Model(self.model_input_dict))
     models = self.dal.model.query({},
                                   sort_key="created_at",
                                   sort_order="descending")
     result = self.dal.model.query({
         "created_at": {
             "$lt": models[1].created_at.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
         }
     })
     assert len(models) == 3
     assert len(result) == 1
Esempio n. 3
0
    def test_query_models_multiple(self):
        # Query for multiple models
        model_1 = self.dal.model.create(Model(self.model_input_dict))
        model_2 = self.dal.model.create(Model(self.model_input_dict))
        model_3 = self.dal.model.create(Model(self.model_input_dict))

        results = self.dal.model.query({},
                                       sort_key="created_at",
                                       sort_order="ascending")
        assert len(results) == 3
        assert results[0].created_at == model_1.created_at
        assert results[1].created_at == model_2.created_at

        results = self.dal.model.query({},
                                       sort_key="created_at",
                                       sort_order="descending")
        assert len(results) == 3
        assert results[0].created_at == model_3.created_at
        assert results[1].created_at == model_2.created_at

        # Wrong order being passed in
        failed = False
        try:
            _ = self.dal.model.query({},
                                     sort_key='created_at',
                                     sort_order='wrong_order')
        except InvalidArgumentType:
            failed = True
        assert failed

        # Wrong key and order being passed in
        failed = False
        try:
            _ = self.dal.model.query({},
                                     sort_key='wrong_key',
                                     sort_order='wrong_order')
        except InvalidArgumentType:
            failed = True
        assert failed

        # wrong key and right order being passed in
        expected_items = self.dal.model.query({},
                                              sort_key='created_at',
                                              sort_order='ascending')
        items = self.dal.model.query({},
                                     sort_key='wrong_key',
                                     sort_order='ascending')
        expected_ids = [item.id for item in expected_items]
        ids = [item.id for item in items]
        assert set(expected_ids) == set(ids)
Esempio n. 4
0
    def test_create_model_by_dictionary(self):
        model = self.dal.model.create(Model(self.model_input_dict))
        assert model.id
        assert model.name == self.model_input_dict['name']
        assert model.created_at
        assert model.updated_at

        model_2 = self.dal.model.create(Model(self.model_input_dict))
        assert model.id != model_2.id

        test_model_input_dict = self.model_input_dict.copy()
        test_model_input_dict['id'] = "cool"
        model_3 = self.dal.model.create(Model(test_model_input_dict))
        assert model_3.id == test_model_input_dict['id']
Esempio n. 5
0
    def setup_method(self):
        # provide mountable tmp directory for docker
        tempfile.tempdir = "/tmp" if not platform.system(
        ) == "Windows" else None
        test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
                                        tempfile.gettempdir())
        self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
        self.datadriver = BlitzDBDALDriver("file", self.temp_dir)

        self.dal = LocalDAL(self.datadriver)
        model_name = "model_1"
        model = self.dal.model.create(Model({"name": model_name}))
        session_name = "session_1"
        session = self.dal.session.create(
            Session({
                "name": session_name,
                "model_id": model.id
            }))

        self.snapshot_input_dict = {
            "model_id": model.id,
            "session_id": session.id,
            "message": "my message",
            "code_id": "code_id",
            "environment_id": "environment_id",
            "file_collection_id": "file_collection_id",
            "config": {
                "test": 0.45
            },
            "stats": {
                "test": 0.98
            }
        }
Esempio n. 6
0
    def setup_method(self):
        # provide mountable tmp directory for docker
        tempfile.tempdir = "/tmp" if not platform.system(
        ) == "Windows" else None
        test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
                                        tempfile.gettempdir())
        self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
        self.datadriver = BlitzDBDALDriver("file", self.temp_dir)

        self.dal = LocalDAL(self.datadriver)
        model_name = "model_1"
        model = self.dal.model.create(Model({"name": model_name}))
        session_name = "session_1"
        session = self.dal.session.create(
            Session({
                "name": session_name,
                "model_id": model.id
            }))

        self.task_input_dict = {
            "model_id": model.id,
            "session_id": session.id,
            "command": "task_1",
            "start_time": datetime.utcnow(),
            "end_time": datetime.utcnow(),
            "duration": 0.004,
        }
Esempio n. 7
0
    def test_current_session(self):
        # Test failure case
        failed = False
        try:
            _ = self.base_controller.current_session
        except DatmoModelNotInitialized:
            failed = True
        assert failed

        # Test success case
        self.base_controller.dal.model.create(
            Model({
                "name": "test",
                "description": "test"
            }))
        _ = self.base_controller.model
        self.base_controller.dal.session.create(
            Session({
                "name": "test",
                "model_id": "test",
                "current": True
            }))
        session = self.base_controller.current_session

        assert session.id
        assert session.name == "test"
        assert session.model_id == "test"
        assert session.current == True
Esempio n. 8
0
    def setup_method(self):
        # provide mountable tmp directory for docker
        tempfile.tempdir = "/tmp" if not platform.system(
        ) == "Windows" else None
        test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
                                        tempfile.gettempdir())
        self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
        self.driver_type = "blitzdb"
        self.driver_options = {
            "driver_type": "file",
            "connection_string": self.temp_dir
        }
        self.dal = LocalDAL(self.driver_type, self.driver_options)
        model_name = "model_1"
        model = self.dal.model.create(Model({"name": model_name}))

        self.environment_input_dict = {
            "model_id": model.id,
            "driver_type": "docker",
            "file_collection_id": "test_file_id",
            "definition_filename": "Dockerfile",
            "hardware_info": {
                "system": "macosx"
            },
            "unique_hash": "slkdjfa23dk",
            "language": "python3"
        }
Esempio n. 9
0
    def test_init_with_id(self):
        self.input_dict['id'] = "test"
        model_entity = Model(self.input_dict)

        for k, v in self.input_dict.items():
            assert getattr(model_entity, k) == v
        assert model_entity.description == None
        assert model_entity.created_at
        assert model_entity.updated_at
Esempio n. 10
0
 def test_get_by_id_model_new_driver_instance(self):
     model = self.dal.model.create(Model(self.model_input_dict))
     # create new dal with new driver instance (success)
     new_driver_instance = BlitzDBDALDriver("file", self.temp_dir)
     new_dal_instance = LocalDAL(new_driver_instance)
     new_model_1 = new_dal_instance.model.get_by_id(model.id)
     assert new_model_1.id == model.id
     # create new dal instance with same driver (success)
     new_dal_instance = LocalDAL(self.datadriver)
     new_model_2 = new_dal_instance.model.get_by_id(model.id)
     assert new_model_2.id == model.id
Esempio n. 11
0
    def test_delete_model(self):
        model = self.dal.model.create(Model(self.model_input_dict))

        self.dal.model.delete(model.id)

        deleted = False
        try:
            self.dal.model.get_by_id(model.id)
        except EntityNotFound:
            deleted = True
        assert deleted
Esempio n. 12
0
    def test_update_model(self):
        model = self.dal.model.create(Model(self.model_input_dict))

        # Update required and optional parameters
        updated_model_input_dict = self.model_input_dict.copy()
        updated_model_input_dict['id'] = model.id
        updated_model_input_dict['name'] = "model_4a"
        updated_model_input_dict['description'] = "this is a cool model"
        updated_model = self.dal.model.update(updated_model_input_dict)
        assert model.id == updated_model.id
        assert model.updated_at < updated_model.updated_at
        assert updated_model.name == updated_model_input_dict['name']
        assert updated_model.description == updated_model_input_dict[
            'description']
Esempio n. 13
0
    def setup_class(self):
        # provide mountable tmp directory for docker
        tempfile.tempdir = "/tmp" if not platform.system(
        ) == "Windows" else None
        test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
                                        tempfile.gettempdir())
        self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
        self.datadriver = BlitzDBDALDriver("file", self.temp_dir)

        self.dal = LocalDAL(self.datadriver)
        model_name = "model_1"
        model = self.dal.model.create(Model({"name": model_name}))

        self.session_input_dict = {"name": "session_1", "model_id": model.id}
Esempio n. 14
0
    def test_datmo_model(self):
        # Test failure case
        assert self.base_controller.model == None

        # Test success case
        self.base_controller.dal.model.create(
            Model({
                "name": "test",
                "description": "test"
            }))
        model = self.base_controller.model

        assert model.id
        assert model.name == "test"
        assert model.description == "test"
Esempio n. 15
0
    def setup_method(self):
        # provide mountable tmp directory for docker
        tempfile.tempdir = "/tmp" if not platform.system(
        ) == "Windows" else None
        test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
                                        tempfile.gettempdir())
        self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
        self.datadriver = BlitzDBDALDriver("file", self.temp_dir)

        self.dal = LocalDAL(self.datadriver)
        model_name = "model_1"
        model = self.dal.model.create(Model({"name": model_name}))

        self.file_collection_input_dict = {
            "model_id": model.id,
            "driver_type": "local",
            "filehash": "myhash",
            "path": "test_path",
        }
Esempio n. 16
0
    def setup_method(self):
        # provide mountable tmp directory for docker
        tempfile.tempdir = "/tmp" if not platform.system(
        ) == "Windows" else None
        test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
                                        tempfile.gettempdir())
        self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
        self.driver_type = "blitzdb"
        self.driver_options = {
            "driver_type": "file",
            "connection_string": self.temp_dir
        }

        self.dal = LocalDAL(self.driver_type, self.driver_options)
        model_name = "model_1"
        model = self.dal.model.create(Model({"name": model_name}))

        self.code_input_dict = {
            "model_id": model.id,
            "driver_type": "git",
            "commit_id": "commit_id"
        }
Esempio n. 17
0
    def init(self, name, description):
        """ Initialize the project

        This function will initialize the project or reinitialize it the project is
        already initialized.

        Parameters
        ----------
        name : str
        description : str

        Returns
        -------
        bool
        """
        is_new_model = False
        old_model = self.model
        if not self.model:
            is_new_model = True

        try:
            # Always validate inputs to the init function
            validate("create_project", {
                "name": name,
                "description": description
            })

            # Initialize File Driver if needed
            if not self.file_driver.is_initialized:
                self.file_driver.init()

            # Initialize the dal
            if not self.dal.is_initialized:
                self.dal.init()

            # Initialize Code Driver if needed
            if not self.code_driver.is_initialized:
                self.code_driver.init()

            # Initialize Environment Driver if needed
            if not self.environment_driver.is_initialized:
                self.environment_driver.init()

            # Initialize the config JSON store
            self.config_store = JSONStore(
                os.path.join(self.home,
                             Config().datmo_directory_name, ".config"))

            # Create model if new else update
            if is_new_model:
                _ = self.dal.model.create(
                    Model({
                        "name": name,
                        "description": description
                    }))
            else:
                self._model = self.dal.model.update({
                    "id": self.model.id,
                    "name": name,
                    "description": description
                })

            # Connect Environment Driver if needed
            # (not required but will warn if not present)
            try:
                if not self.environment_driver.is_connected:
                    self.environment_driver.connect()
            except EnvironmentConnectFailed:
                self.logger.warning(
                    __("warn", "controller.general.environment.failed"))

            # Build the initial default Environment (NOT NECESSARY)
            # self.environment_driver.build_image(tag="datmo-" + \
            #                                  self.model.name)
            return True
        except Exception:
            # if any error occurred with new model, ensure no initialize occurs and raise previous error
            # if any error occurred with existing model, ensure no updates were made, raise previous error
            if is_new_model:
                self.cleanup()
            else:
                self._model = self.dal.model.update({
                    "id": old_model.id,
                    "name": old_model.name,
                    "description": old_model.description
                })
            raise
Esempio n. 18
0
    def init(self, name, description):
        # Error if name is not given
        if not name:
            raise RequiredArgumentMissing(
                __("error", "controller.project.init.arg", "name"))

        # Create the Model, is it new or update?
        is_new_model = False
        if not self.model:
            _ = self.dal.model.create(
                Model({
                    "name": name,
                    "description": description
                }))
            is_new_model = True
        else:
            self._model = self.dal.model.update({
                "id": self.model.id,
                "name": name,
                "description": description
            })

        # Initialize Code Manager if needed
        if not self.code_driver.is_initialized:
            self.code_driver.init()

        # Initialize File Manager if needed
        if not self.file_driver.is_initialized:
            self.file_driver.init()

        # Initialize Environment Manager if needed
        if not self.environment_driver.is_initialized:
            self.environment_driver.init()

        # Build the initial default Environment (NOT NECESSARY)
        # self.environment_driver.build_image(tag="datmo-" + \
        #                                  self.model.name)

        # Add in Project template files if specified
        # TODO: Add in project template files

        # Create and set current session
        if is_new_model:
            # Create new default session
            _ = self.dal.session.create(
                Session({
                    "name": "default",
                    "model_id": self.model.id,
                    "current": True
                }))
        else:
            if not self.current_session:
                default_session_obj = self.dal.session.query({
                    "name":
                    "default",
                    "model_id":
                    self.model.id
                })
                if not default_session_obj:
                    raise SessionDoesNotExistException(
                        __("error", "controller.project.init"))
                # Update default session to be current
                self.dal.session.update({
                    "id": default_session_obj.id,
                    "current": True
                })
        return True
Esempio n. 19
0
    def test_eq(self):
        model_entity_1 = Model(self.input_dict)
        model_entity_2 = Model(self.input_dict)

        assert model_entity_1 == model_entity_2
Esempio n. 20
0
    def init(self, name, description):
        """ Initialize the project

        This function will initialize the project or reinitialize it the project is
        already initialized.

        Parameters
        ----------
        name : str
        description : str

        Returns
        -------
        bool

        Raises
        ------
        SessionDoesNotExist
        """
        # Create the Model, is it new or update?
        is_new_model = False
        old_model = self.model
        if not self.model:
            is_new_model = True

        try:
            # Always validate inputs to the init function
            validate("create_project", {
                "name": name,
                "description": description
            })

            # Create model if new else update
            if is_new_model:
                _ = self.dal.model.create(
                    Model({
                        "name": name,
                        "description": description
                    }))
            else:
                self._model = self.dal.model.update({
                    "id": self.model.id,
                    "name": name,
                    "description": description
                })

            # Initialize Code Driver if needed
            if not self.code_driver.is_initialized:
                self.code_driver.init()

            # Initialize File Driver if needed
            if not self.file_driver.is_initialized:
                self.file_driver.init()

            # Initialize Environment Driver if needed
            # (not required but will warn if not present)
            try:
                if not self.environment_driver.is_initialized:
                    self.environment_driver.init()
            except EnvironmentInitFailed:
                self.logger.warning(
                    __("warn", "controller.general.environment.failed"))

            # Build the initial default Environment (NOT NECESSARY)
            # self.environment_driver.build_image(tag="datmo-" + \
            #                                  self.model.name)

            # Create and set current session
            if is_new_model:
                # Create new default session
                _ = self.dal.session.create(
                    Session({
                        "name": "default",
                        "model_id": self.model.id,
                        "current": True
                    }))
            else:
                if not self.current_session:
                    default_session_objs = self.dal.session.query({
                        "name": "default",
                        "model_id": self.model.id
                    })
                    if not default_session_objs:
                        # Creating a default session since none exists
                        _ = self.dal.session.create(
                            Session({
                                "name": "default",
                                "model_id": self.model.id,
                                "current": True
                            }))
                    else:
                        # Update default session to be current
                        default_session_obj = default_session_objs[0]
                        self.dal.session.update({
                            "id": default_session_obj.id,
                            "current": True
                        })
            return True
        except Exception:
            # if any error occurred with new model, ensure no initialize occurs and raise previous error
            # if any error occurred with existing model, ensure no updates were made, raise previous error
            if is_new_model:
                self.cleanup()
            else:
                self._model = self.dal.model.update({
                    "id": old_model.id,
                    "name": old_model.name,
                    "description": old_model.description
                })
            raise
Esempio n. 21
0
    def test_to_dictionary(self):
        model_entity = Model(self.input_dict)
        output_dict = model_entity.to_dictionary()

        for k, v in output_dict.items():
            assert v == getattr(model_entity, k)
Esempio n. 22
0
 def test_sanity_check_for_dal(self):
     model = self.base_controller.dal.model.create(Model({"name": "test"}))
     model2 = self.base_controller.dal.model.get_by_id(model.id)
     assert model and model2
     assert model.id == model2.id
Esempio n. 23
0
 def test_query_models(self):
     model = self.dal.model.create(Model(self.model_input_dict))
     assert len(self.dal.model.query({"id": model.id})) == 1
     assert len(
         self.dal.model.query({"name": self.model_input_dict['name']})) == 6
Esempio n. 24
0
 def test_get_by_id_model(self):
     model = self.dal.model.create(Model(self.model_input_dict))
     result = self.dal.model.get_by_id(model.id)
     assert model.id == result.id