def init_automl():
from google.cloud import automl, automl_v1beta1
if not is_user_secrets_token_set():
return
from kaggle_gcp import get_integrations
if not get_integrations().has_cloudai():
return
from kaggle_secrets import GcpTarget
from kaggle_gcp import KaggleKernelCredentials
kaggle_kernel_credentials = KaggleKernelCredentials(
target=GcpTarget.CLOUDAI)
# Patch the 2 GA clients: AutoMlClient and PreditionServiceClient
monkeypatch_client(automl.AutoMlClient, kaggle_kernel_credentials)
monkeypatch_client(automl.PredictionServiceClient,
kaggle_kernel_credentials)
# The AutoML client library exposes 3 different client classes (AutoMlClient,
# TablesClient, PredictionServiceClient), so patch each of them.
# The same KaggleKernelCredentials are passed to all of them.
# The GcsClient class is only used internally by TablesClient.
# The beta version of the clients that are now GA are included here for now.
# They are deprecated and will be removed by 1 May 2020.
monkeypatch_client(automl_v1beta1.AutoMlClient, kaggle_kernel_credentials)
monkeypatch_client(automl_v1beta1.PredictionServiceClient,
kaggle_kernel_credentials)
# The TablesClient is still in beta, so this will not be deprecated until
# the TablesClient is GA.
monkeypatch_client(automl_v1beta1.TablesClient, kaggle_kernel_credentials)
def init_automl():
is_user_secrets_token_set = "KAGGLE_USER_SECRETS_TOKEN" in os.environ
from google.cloud import automl_v1beta1 as automl
if not is_user_secrets_token_set:
return automl
from kaggle_gcp import get_integrations
if not get_integrations().has_automl():
return automl
from kaggle_secrets import GcpTarget
from kaggle_gcp import KaggleKernelCredentials
kaggle_kernel_credentials = KaggleKernelCredentials(
target=GcpTarget.AUTOML)
# The AutoML client library exposes 4 different client classes (AutoMlClient,
# TablesClient, PredictionServiceClient and GcsClient), so patch each of them.
# The same KaggleKernelCredentials are passed to all of them.
monkeypatch_client(automl.AutoMlClient, kaggle_kernel_credentials)
monkeypatch_client(automl.TablesClient, kaggle_kernel_credentials)
monkeypatch_client(automl.PredictionServiceClient,
kaggle_kernel_credentials)
# TODO(markcollins): The GcsClient in the AutoML client library version
# 0.5.0 doesn't handle credentials properly. I wrote PR:
# https://github.com/googleapis/google-cloud-python/pull/9299
# to address this issue. Add patching for GcsClient when we get a version of
# the library that includes the fixes.
return automl
def refresh(self, request):
try:
client = UserSecretsClient()
if self.target == GcpTarget.BIGQUERY:
self.token, self.expiry = client.get_bigquery_access_token()
elif self.target == GcpTarget.GCS:
self.token, self.expiry = client._get_gcs_access_token()
elif self.target == GcpTarget.CLOUDAI:
self.token, self.expiry = client._get_cloudai_access_token()
except ConnectionError as e:
Log.error(f"Connection error trying to refresh access token: {e}")
print(
"There was a connection error trying to fetch the access token. "
f"Please ensure internet is on in order to use the {self.target.service} Integration."
)
raise RefreshError(
'Unable to refresh access token due to connection error.'
) from e
except Exception as e:
Log.error(f"Error trying to refresh access token: {e}")
if (not get_integrations().has_integration(self.target)):
Log.error(f"No {self.target.service} integration found.")
print(
f"Please ensure you have selected a {self.target.service} account in the Notebook Add-ons menu."
)
raise RefreshError('Unable to refresh access token.') from e
def init_gcs():
is_user_secrets_token_set = "KAGGLE_USER_SECRETS_TOKEN" in os.environ
from google.cloud import storage
if not is_user_secrets_token_set:
return storage
from kaggle_gcp import get_integrations
if not get_integrations().has_gcs():
return storage
from kaggle_secrets import GcpTarget
from kaggle_gcp import KaggleKernelCredentials
gcs_client_init = storage.Client.__init__
def monkeypatch_gcs(self, *args, **kwargs):
specified_credentials = kwargs.get('credentials')
if specified_credentials is None:
Log.info(
"No credentials specified, using KaggleKernelCredentials.")
kwargs['credentials'] = KaggleKernelCredentials(
target=GcpTarget.GCS)
return gcs_client_init(self, *args, **kwargs)
storage.Client.__init__ = monkeypatch_gcs
return storage
def monkeypatch_bq(bq_client, *args, **kwargs):
from kaggle_gcp import get_integrations, PublicBigqueryClient, KaggleKernelCredentials
specified_credentials = kwargs.get('credentials')
has_bigquery = get_integrations().has_bigquery()
# Prioritize passed in project id, but if it is missing look for env var.
arg_project = kwargs.get('project')
explicit_project_id = arg_project or os.environ.get(environment_vars.PROJECT)
# This is a hack to get around the bug in google-cloud library.
# Remove these two lines once this is resolved:
# https://github.com/googleapis/google-cloud-python/issues/8108
if explicit_project_id:
Log.info(f"Explicit project set to {explicit_project_id}")
kwargs['project'] = explicit_project_id
if explicit_project_id is None and specified_credentials is None and not has_bigquery:
msg = "Using Kaggle's public dataset BigQuery integration."
Log.info(msg)
print(msg)
return PublicBigqueryClient(*args, **kwargs)
else:
if specified_credentials is None:
Log.info("No credentials specified, using KaggleKernelCredentials.")
kwargs['credentials'] = KaggleKernelCredentials()
if (not has_bigquery):
Log.info("No bigquery integration found, creating client anyways.")
print('Please ensure you have selected a BigQuery '
'account in the Kernels Settings sidebar.')
if explicit_project_id is None:
Log.info("No project specified while using the unmodified client.")
print('Please ensure you specify a project id when creating the client'
' in order to use your BigQuery account.')
return bq_client(*args, **kwargs)
def init_bigquery():
from google.cloud import bigquery
if not (is_proxy_token_set() or is_user_secrets_token_set()):
return bigquery
# If this Notebook has bigquery integration on startup, preload the Kaggle Credentials
# object for magics to work.
if get_integrations().has_bigquery():
from google.cloud.bigquery import magics
magics.context.credentials = KaggleKernelCredentials()
def monkeypatch_bq(bq_client, *args, **kwargs):
from kaggle_gcp import get_integrations, PublicBigqueryClient, KaggleKernelCredentials
specified_credentials = kwargs.get('credentials')
has_bigquery = get_integrations().has_bigquery()
# Prioritize passed in project id, but if it is missing look for env var.
arg_project = kwargs.get('project')
explicit_project_id = arg_project or os.environ.get(
environment_vars.PROJECT)
# This is a hack to get around the bug in google-cloud library.
# Remove these two lines once this is resolved:
# https://github.com/googleapis/google-cloud-python/issues/8108
if explicit_project_id:
Log.info(f"Explicit project set to {explicit_project_id}")
kwargs['project'] = explicit_project_id
if explicit_project_id is None and specified_credentials is None and not has_bigquery:
msg = "Using Kaggle's public dataset BigQuery integration."
Log.info(msg)
print(msg)
return PublicBigqueryClient(*args, **kwargs)
else:
if specified_credentials is None:
Log.info(
"No credentials specified, using KaggleKernelCredentials.")
kwargs['credentials'] = KaggleKernelCredentials()
if (not has_bigquery):
Log.info(
"No bigquery integration found, creating client anyways."
)
print('Please ensure you have selected a BigQuery '
'account in the Notebook Add-ons menu.')
if explicit_project_id is None:
Log.info(
"No project specified while using the unmodified client.")
print(
'Please ensure you specify a project id when creating the client'
' in order to use your BigQuery account.')
kwargs['client_info'] = set_kaggle_user_agent(
kwargs.get('client_info'))
return bq_client(*args, **kwargs)
# Monkey patches BigQuery client creation to use proxy or user-connected GCP account.
# Deprecated in favor of Kaggle.DataProxyClient().
# TODO: Remove this once uses have migrated to that new interface.
bq_client = bigquery.Client
if (not has_been_monkeypatched(bigquery.Client)):
bigquery.Client = lambda *args, **kwargs: monkeypatch_bq(
bq_client, *args, **kwargs)
return bigquery
def init():
is_jwe_set = "KAGGLE_USER_SECRETS_TOKEN" in os.environ
if kaggle_proxy_token or is_jwe_set:
init_bigquery()
if is_jwe_set:
from kaggle_gcp import get_integrations
if get_integrations().has_gcs():
init_gcs()
def init_translation_v2():
from google.cloud import translate_v2
if not is_user_secrets_token_set():
return translate_v2
from kaggle_gcp import get_integrations
if not get_integrations().has_cloudai():
return translate_v2
from kaggle_secrets import GcpTarget
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
monkeypatch_client(translate_v2.Client, kernel_credentials)
return translate_v2
def init_bigquery():
from google.auth import credentials, environment_vars
from google.cloud import bigquery
from google.cloud.bigquery._http import Connection
# TODO: Update this to the correct kaggle.gcp path once we no longer inject modules
# from the worker.
from kaggle_gcp import get_integrations, PublicBigqueryClient, KaggleKernelCredentials
# If this Kernel has bigquery integration on startup, preload the Kaggle Credentials
# object for magics to work.
if get_integrations().has_bigquery():
from google.cloud.bigquery import magics
magics.context.credentials = KaggleKernelCredentials()
def monkeypatch_bq(bq_client, *args, **kwargs):
specified_credentials = kwargs.get('credentials')
has_bigquery = get_integrations().has_bigquery()
# Prioritize passed in project id, but if it is missing look for env var.
arg_project = kwargs.get('project')
explicit_project_id = arg_project or os.environ.get(
environment_vars.PROJECT)
# This is a hack to get around the bug in google-cloud library.
# Remove these two lines once this is resolved:
# https://github.com/googleapis/google-cloud-python/issues/8108
if explicit_project_id:
Log.info(f"Explicit project set to {explicit_project_id}")
kwargs['project'] = explicit_project_id
if explicit_project_id is None and specified_credentials is None and not has_bigquery:
msg = "Using Kaggle's public dataset BigQuery integration."
Log.info(msg)
print(msg)
return PublicBigqueryClient(*args, **kwargs)
else:
if specified_credentials is None:
Log.info(
"No credentials specified, using KaggleKernelCredentials.")
kwargs['credentials'] = KaggleKernelCredentials()
if (not has_bigquery):
Log.info(
"No bigquery integration found, creating client anyways."
)
print('Please ensure you have selected a BigQuery '
'account in the Kernels Settings sidebar.')
return bq_client(*args, **kwargs)
# Monkey patches BigQuery client creation to use proxy or user-connected GCP account.
# Deprecated in favor of Kaggle.DataProxyClient().
# TODO: Remove this once uses have migrated to that new interface.
bq_client = bigquery.Client
bigquery.Client = lambda *args, **kwargs: monkeypatch_bq(
bq_client, *args, **kwargs)
def init_translation_v3():
# Translate v3 exposes different client than translate v2.
from google.cloud import translate_v3
if not is_user_secrets_token_set():
return translate_v3
from kaggle_gcp import get_integrations
if not get_integrations().has_cloudai():
return translate_v3
from kaggle_secrets import GcpTarget
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
monkeypatch_client(translate_v3.TranslationServiceClient, kernel_credentials)
return translate_v3
def monkeypatch_bq(bq_client, *args, **kwargs):
data_proxy_project = os.getenv("KAGGLE_DATA_PROXY_PROJECT")
specified_project = kwargs.get('project')
specified_credentials = kwargs.get('credentials')
kernel_integrations = get_integrations()
if specified_project is None and specified_credentials is None and not kernel_integrations.has_bigquery():
print("Using Kaggle's public dataset BigQuery integration.")
return PublicBigqueryClient(*args, **kwargs)
else:
if specified_credentials is None:
kwargs['credentials'] = KaggleKernelCredentials()
return bq_client(*args, **kwargs)
def init_vision():
from google.cloud import vision
if not is_user_secrets_token_set():
return vision
from kaggle_gcp import get_integrations
if not get_integrations().has_cloudai():
return vision
from kaggle_secrets import GcpTarget
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
monkeypatch_client(vision.ImageAnnotatorClient, kernel_credentials)
monkeypatch_client(vision.ImageAnnotatorAsyncClient, kernel_credentials)
return vision
def init_natural_language():
from google.cloud import language
if not is_user_secrets_token_set():
return language
from kaggle_gcp import get_integrations
if not get_integrations().has_cloudai():
return language
from kaggle_secrets import GcpTarget
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
monkeypatch_client(language.LanguageServiceClient, kernel_credentials)
monkeypatch_client(language.LanguageServiceAsyncClient, kernel_credentials)
return language
def init_gcs():
from google.cloud import storage
if not is_user_secrets_token_set():
return storage
from kaggle_gcp import get_integrations
if not get_integrations().has_gcs():
return storage
from kaggle_secrets import GcpTarget
from kaggle_gcp import KaggleKernelCredentials
monkeypatch_client(storage.Client,
KaggleKernelCredentials(target=GcpTarget.GCS))
return storage
def init_ucaip():
from google.cloud import aiplatform
if not is_user_secrets_token_set():
return
from kaggle_gcp import get_integrations
if not get_integrations().has_cloudai():
return
from kaggle_secrets import GcpTarget
from kaggle_gcp import KaggleKernelCredentials
kaggle_kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
# Patch the ucaip init method, this flows down to all ucaip services
monkeypatch_aiplatform_init(aiplatform, kaggle_kernel_credentials)
def init_gcs():
is_user_secrets_token_set = "KAGGLE_USER_SECRETS_TOKEN" in os.environ
from google.cloud import storage
if not is_user_secrets_token_set:
return storage
from kaggle_gcp import get_integrations
if not get_integrations().has_gcs():
return storage
from kaggle_secrets import GcpTarget
from kaggle_gcp import KaggleKernelCredentials
monkeypatch_client(storage.Client,
KaggleKernelCredentials(target=GcpTarget.GCS))
return storage
def monkeypatch_bq(bq_client, *args, **kwargs):
specified_project = kwargs.get('project')
specified_credentials = kwargs.get('credentials')
has_bigquery = get_integrations().has_bigquery()
if specified_project is None and specified_credentials is None and not has_bigquery:
print("Using Kaggle's public dataset BigQuery integration.")
return PublicBigqueryClient(*args, **kwargs)
else:
if specified_credentials is None:
kwargs['credentials'] = KaggleKernelCredentials()
if (not has_bigquery):
print('Please ensure you have selected a BigQuery '
'account in the Kernels Settings sidebar.')
return bq_client(*args, **kwargs)
def init_video_intelligence():
from google.cloud import videointelligence
if not is_user_secrets_token_set():
return videointelligence
from kaggle_gcp import get_integrations
if not get_integrations().has_cloudai():
return videointelligence
from kaggle_secrets import GcpTarget
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
monkeypatch_client(videointelligence.VideoIntelligenceServiceClient,
kernel_credentials)
monkeypatch_client(videointelligence.VideoIntelligenceServiceAsyncClient,
kernel_credentials)
return videointelligence
def monkeypatch_bq(bq_client, *args, **kwargs):
specified_credentials = kwargs.get('credentials')
has_bigquery = get_integrations().has_bigquery()
# Prioritize passed in project id, but if it is missing look for env var.
arg_project = kwargs.get('project')
explicit_project_id = arg_project or os.environ.get(environment_vars.PROJECT)
# This is a hack to get around the bug in google-cloud library.
# Remove these two lines once this is resolved:
# https://github.com/googleapis/google-cloud-python/issues/8108
if explicit_project_id:
kwargs['project'] = explicit_project_id
if explicit_project_id is None and specified_credentials is None and not has_bigquery:
print("Using Kaggle's public dataset BigQuery integration.")
return PublicBigqueryClient(*args, **kwargs)
else:
if specified_credentials is None:
kwargs['credentials'] = KaggleKernelCredentials()
if (not has_bigquery):
print('Please ensure you have selected a BigQuery '
'account in the Kernels Settings sidebar.')
return bq_client(*args, **kwargs)
def refresh(self, request):
try:
client = UserSecretsClient()
if self.target == GcpTarget.BIGQUERY:
self.token, self.expiry = client.get_bigquery_access_token()
elif self.target == GcpTarget.GCS:
self.token, self.expiry = client._get_gcs_access_token()
elif self.target == GcpTarget.AUTOML:
self.token, self.expiry = client._get_automl_access_token()
elif self.target == GcpTarget.TRANSLATION:
self.token, self.expiry = client._get_translation_access_token(
)
elif self.target == GcpTarget.NATURAL_LANGUAGE:
self.token, self.expiry = client._get_natural_language_access_token(
)
elif self.target == GcpTarget.VIDEO_INTELLIGENCE:
self.token, self.expiry = client._get_video_intelligence_access_token(
)
elif self.target == GcpTarget.VISION:
self.token, self.expiry = client._get_vision_access_token()
except ConnectionError as e:
Log.error(f"Connection error trying to refresh access token: {e}")
print(
"There was a connection error trying to fetch the access token. "
f"Please ensure internet is on in order to use the {self.target.service} Integration."
)
raise RefreshError(
'Unable to refresh access token due to connection error.'
) from e
except Exception as e:
Log.error(f"Error trying to refresh access token: {e}")
if (not get_integrations().has_integration(self.target)):
Log.error(f"No {self.target.service} integration found.")
print(
f"Please ensure you have selected a {self.target.service} account in the Notebook Add-ons menu."
)
raise RefreshError('Unable to refresh access token.') from e
