Пример #1
0
 def connect(self):
     self._connected = True
     try:
         if client.base.Client.REST_ENDPOINT not in os.environ:
             if os.path.exists("/dbfs/"):
                 # databricks
                 client.init(
                     "external",
                     self._host,
                     self._port,
                     self._project,
                     self._region_name,
                     self._secrets_store,
                     self._hostname_verification,
                     os.path.join("/dbfs", self._trust_store_path)
                     if self._trust_store_path is not None else None,
                     os.path.join("/dbfs", self._cert_folder),
                     os.path.join("/dbfs", self._api_key_file)
                     if self._api_key_file is not None else None,
                     self._api_key_value,
                 )
                 engine.init("spark")
             else:
                 # aws
                 client.init(
                     "external",
                     self._host,
                     self._port,
                     self._project,
                     self._region_name,
                     self._secrets_store,
                     self._hostname_verification,
                     self._trust_store_path,
                     self._cert_folder,
                     self._api_key_file,
                     self._api_key_value,
                 )
                 engine.init(
                     "hive",
                     self._host,
                     self._cert_folder,
                     self._project,
                     client.get_instance()._cert_key,
                 )
         else:
             client.init("hopsworks")
             engine.init("spark")
         self._feature_store_api = feature_store_api.FeatureStoreApi()
         self._project_api = project_api.ProjectApi()
         self._hosts_api = hosts_api.HostsApi()
         self._services_api = services_api.ServicesApi()
     except (TypeError, ConnectionError):
         self._connected = False
         raise
     print("Connected. Call `.close()` to terminate connection gracefully.")
Пример #2
0
    def connect(self):
        """Instantiate the connection.

        Creating a `Connection` object implicitly calls this method for you to
        instantiate the connection. However, it is possible to close the connection
        gracefully with the `close()` method, in order to clean up materialized
        certificates. This might be desired when working on external environments such
        as AWS SageMaker. Subsequently you can call `connect()` again to reopen the
        connection.

        !!! example
            ```python
            import hsfs
            conn = hsfs.connection()
            conn.close()
            conn.connect()
            ```
        """
        self._connected = True
        try:
            # determine engine, needed to init client
            if (self._engine is not None and self._engine.lower() == "spark") or (
                self._engine is None and importlib.util.find_spec("pyspark")
            ):
                self._engine = "spark"
            elif (self._engine is not None and self._engine.lower() == "hive") or (
                self._engine is None and not importlib.util.find_spec("pyspark")
            ):
                self._engine = "hive"
            else:
                raise ConnectionError(
                    "Engine you are trying to initialize is unknown. "
                    "Supported engines are `'spark'` and `'hive'`."
                )

            # init client
            if client.base.Client.REST_ENDPOINT not in os.environ:
                client.init(
                    "external",
                    self._host,
                    self._port,
                    self._project,
                    self._engine,
                    self._region_name,
                    self._secrets_store,
                    self._hostname_verification,
                    self._trust_store_path,
                    self._cert_folder,
                    self._api_key_file,
                    self._api_key_value,
                )
            else:
                client.init("hopsworks")

            # init engine
            engine.init(self._engine)

            self._feature_store_api = feature_store_api.FeatureStoreApi()
            self._project_api = project_api.ProjectApi()
            self._hosts_api = hosts_api.HostsApi()
            self._services_api = services_api.ServicesApi()
        except (TypeError, ConnectionError):
            self._connected = False
            raise
        print("Connected. Call `.close()` to terminate connection gracefully.")
Пример #3
0
    def connect(self):
        """Instantiate the connection.

        Creating a `Connection` object implicitly calls this method for you to
        instantiate the connection. However, it is possible to close the connection
        gracefully with the `close()` method, in order to clean up materialized
        certificates. This might be desired when working on external environments such
        as AWS SageMaker. Subsequently you can call `connect()` again to reopen the
        connection.

        !!! example
            ```python
            import hsfs
            conn = hsfs.connection()
            conn.close()
            conn.connect()
            ```
        """
        self._connected = True
        try:
            if client.base.Client.REST_ENDPOINT not in os.environ:
                if os.path.exists("/dbfs/"):
                    # databricks
                    client.init(
                        "external",
                        self._host,
                        self._port,
                        self._project,
                        self._region_name,
                        self._secrets_store,
                        self._hostname_verification,
                        os.path.join("/dbfs", self._trust_store_path)
                        if self._trust_store_path is not None else None,
                        os.path.join("/dbfs", self._cert_folder),
                        os.path.join("/dbfs", self._api_key_file)
                        if self._api_key_file is not None else None,
                        self._api_key_value,
                    )
                    engine.init("spark")
                else:
                    # aws
                    client.init(
                        "external",
                        self._host,
                        self._port,
                        self._project,
                        self._region_name,
                        self._secrets_store,
                        self._hostname_verification,
                        self._trust_store_path,
                        self._cert_folder,
                        self._api_key_file,
                        self._api_key_value,
                    )
                    engine.init(
                        "hive",
                        self._host,
                        self._cert_folder,
                        self._project,
                        client.get_instance()._cert_key,
                    )
            else:
                client.init("hopsworks")
                engine.init("spark")
            self._feature_store_api = feature_store_api.FeatureStoreApi()
            self._project_api = project_api.ProjectApi()
            self._hosts_api = hosts_api.HostsApi()
            self._services_api = services_api.ServicesApi()
        except (TypeError, ConnectionError):
            self._connected = False
            raise
        print("Connected. Call `.close()` to terminate connection gracefully.")