def __init__(self):
"""Initializes a client being run from a job/notebook directly on Hopsworks."""
self._base_url = self._get_hopsworks_rest_endpoint()
self._host, self._port = self._get_host_port_pair()
self._secrets_dir = (
os.environ[self.SECRETS_DIR] if self.SECRETS_DIR in os.environ else ""
)
self._cert_key = self._get_cert_pw()
trust_store_path = self._get_trust_store_path()
hostname_verification = (
os.environ[self.REQUESTS_VERIFY]
if self.REQUESTS_VERIFY in os.environ
else "true"
)
self._project_id = os.environ[self.PROJECT_ID]
self._project_name = self._project_name()
try:
self._auth = auth.BearerAuth(self._read_jwt())
except FileNotFoundError:
self._auth = auth.ApiKeyAuth(self._read_apikey())
self._verify = self._get_verify(hostname_verification, trust_store_path)
self._session = requests.session()
self._connected = True
credentials = self._get_credentials(self._project_id)
self._write_pem_file(credentials["caChain"], self._get_ca_chain_path())
self._write_pem_file(credentials["clientCert"], self._get_client_cert_path())
self._write_pem_file(credentials["clientKey"], self._get_client_key_path())
def __init__(
self,
host,
port,
project,
region_name,
secrets_store,
hostname_verification,
trust_store_path,
cert_folder,
api_key_file,
api_key_value,
):
"""Initializes a client in an external environment such as AWS Sagemaker."""
if not host:
raise exceptions.ExternalClientError("host")
if not project:
raise exceptions.ExternalClientError("project")
self._host = host
self._port = port
self._base_url = "https://" + self._host + ":" + str(self._port)
self._project_name = project
self._region_name = region_name or self.DEFAULT_REGION
if api_key_value is not None:
api_key = api_key_value
else:
api_key = self._get_secret(secrets_store, "api-key", api_key_file)
self._auth = auth.ApiKeyAuth(api_key)
self._session = requests.session()
self._connected = True
self._verify = self._get_verify(self._host, trust_store_path)
project_info = self._get_project_info(self._project_name)
self._project_id = str(project_info["projectId"])
if cert_folder:
# On external Spark clients (Databricks, Spark Cluster),
# certificates need to be provided before the Spark application starts.
self._cert_folder_base = cert_folder
self._cert_folder = os.path.join(cert_folder, host, project)
os.makedirs(self._cert_folder, exist_ok=True)
credentials = self._get_credentials(self._project_id)
self._write_b64_cert_to_bytes(
str(credentials["kStore"]),
path=os.path.join(self._cert_folder, "keyStore.jks"),
)
self._write_b64_cert_to_bytes(
str(credentials["tStore"]),
path=os.path.join(self._cert_folder, "trustStore.jks"),
)
self._cert_key = str(credentials["password"])
with open(os.path.join(self._cert_folder, "material_passwd"),
"w") as f:
f.write(str(credentials["password"]))
def __init__(
self,
host,
port,
project,
engine,
region_name,
secrets_store,
hostname_verification,
trust_store_path,
cert_folder,
api_key_file,
api_key_value,
):
"""Initializes a client in an external environment such as AWS Sagemaker."""
if not host:
raise exceptions.ExternalClientError("host")
if not project:
raise exceptions.ExternalClientError("project")
self._host = host
self._port = port
self._base_url = "https://" + self._host + ":" + str(self._port)
self._project_name = project
self._region_name = region_name or self.DEFAULT_REGION
if api_key_value is not None:
api_key = api_key_value
else:
api_key = self._get_secret(secrets_store, "api-key", api_key_file)
self._auth = auth.ApiKeyAuth(api_key)
self._session = requests.session()
self._connected = True
self._verify = self._get_verify(self._host, trust_store_path)
project_info = self._get_project_info(self._project_name)
self._project_id = str(project_info["projectId"])
self._cert_key = None
self._cert_folder_base = None
if engine == "python":
# On external Spark clients (Databricks, Spark Cluster),
# certificates need to be provided before the Spark application starts.
self._cert_folder_base = cert_folder
self._cert_folder = os.path.join(cert_folder, host, project)
self._trust_store_path = os.path.join(self._cert_folder, "trustStore.jks")
self._key_store_path = os.path.join(self._cert_folder, "keyStore.jks")
os.makedirs(self._cert_folder, exist_ok=True)
credentials = self._get_credentials(self._project_id)
self._write_b64_cert_to_bytes(
str(credentials["kStore"]),
path=self._get_jks_key_store_path(),
)
self._write_b64_cert_to_bytes(
str(credentials["tStore"]),
path=self._get_jks_trust_store_path(),
)
self._write_pem_file(credentials["caChain"], self._get_ca_chain_path())
self._write_pem_file(
credentials["clientCert"], self._get_client_cert_path()
)
self._write_pem_file(credentials["clientKey"], self._get_client_key_path())
self._cert_key = str(credentials["password"])
with open(os.path.join(self._cert_folder, "material_passwd"), "w") as f:
f.write(str(credentials["password"]))
elif engine == "spark":
_spark_session = SparkSession.builder.getOrCreate()
self.validate_spark_configuration(_spark_session)
with open(
_spark_session.conf.get("spark.hadoop.hops.ssl.keystores.passwd.name"),
"r",
) as f:
self._cert_key = f.read()
self._trust_store_path = _spark_session.conf.get(
"spark.hadoop.hops.ssl.trustore.name"
)
self._key_store_path = _spark_session.conf.get(
"spark.hadoop.hops.ssl.keystore.name"
)