def _open_gcs_url(self, binary) -> object:
mode = "rb" if binary else "r"
service_account_json = self._provider.get("service_account_json")
credentials = None
if service_account_json:
try:
credentials = json.loads(
self._provider["service_account_json"])
except json.decoder.JSONDecodeError as err:
error_msg = f"Failed to parse gcs service account json: {repr(err)}\n{traceback.format_exc()}"
logger.error(error_msg)
raise ConfigurationError(error_msg) from err
if credentials:
credentials = service_account.Credentials.from_service_account_info(
credentials)
client = GCSClient(credentials=credentials,
project=credentials._project_id)
else:
client = GCSClient.create_anonymous_client()
file_to_close = smart_open.open(self.full_url,
transport_params=dict(client=client),
mode=mode)
return file_to_close
def open_gcs_url(config, logger, storage, url):
reader_impl = SourceFile.extract_reader_impl(config)
use_gcs_service_account = "service_account_json" in config["provider"] and storage == "gs://"
file_to_close = None
if reader_impl == "gcsfs":
if use_gcs_service_account:
try:
token_dict = json.loads(config["provider"]["service_account_json"])
except json.decoder.JSONDecodeError as err:
logger.error(f"Failed to parse gcs service account json: {repr(err)}\n{traceback.format_exc()}")
raise err
else:
token_dict = "anon"
fs = gcsfs.GCSFileSystem(token=token_dict)
file_to_close = fs.open(f"gs://{url}")
result = file_to_close
else:
if use_gcs_service_account:
try:
credentials = json.dumps(json.loads(config["provider"]["service_account_json"]))
tmp_service_account = tempfile.NamedTemporaryFile(delete=False)
with open(tmp_service_account, "w") as f:
f.write(credentials)
tmp_service_account.close()
client = Client.from_service_account_json(tmp_service_account.name)
result = open(f"gs://{url}", transport_params=dict(client=client))
os.remove(tmp_service_account.name)
except json.decoder.JSONDecodeError as err:
logger.error(f"Failed to parse gcs service account json: {repr(err)}\n{traceback.format_exc()}")
raise err
else:
client = Client.create_anonymous_client()
result = open(f"{storage}{url}", transport_params=dict(client=client))
return result, file_to_close
def __init__(
self,
application_credentials: Optional[Union[str, os.PathLike]] = None,
credentials: Optional["Credentials"] = None,
project: Optional[str] = None,
storage_client: Optional["StorageClient"] = None,
local_cache_dir: Optional[Union[str, os.PathLike]] = None,
):
"""Class constructor. Sets up a [`Storage
Client`](https://googleapis.dev/python/storage/latest/client.html).
Supports the following authentication methods of `Storage Client`.
- Environment variable `"GOOGLE_APPLICATION_CREDENTIALS"` containing a
path to a JSON credentials file for a Google service account. See
[Authenticating as a Service
Account](https://cloud.google.com/docs/authentication/production).
- File path to a JSON credentials file for a Google service account.
- OAuth2 Credentials object and a project name.
- Instantiated and already authenticated `Storage Client`.
If multiple methods are used, priority order is reverse of list above
(later in list takes priority). If no authentication methods are used,
then the client will be instantiated as anonymous, which will only have
access to public buckets.
Args:
application_credentials (Optional[Union[str, os.PathLike]]): Path to Google service
account credentials file.
credentials (Optional[Credentials]): The OAuth2 Credentials to use for this client.
See documentation for [`StorageClient`](
https://googleapis.dev/python/storage/latest/client.html).
project (Optional[str]): The project which the client acts on behalf of. See
documentation for [`StorageClient`](
https://googleapis.dev/python/storage/latest/client.html).
storage_client (Optional[StorageClient]): Instantiated [`StorageClient`](
https://googleapis.dev/python/storage/latest/client.html).
local_cache_dir (Optional[Union[str, os.PathLike]]): Path to directory to use as cache
for downloaded files. If None, will use a temporary directory.
"""
if application_credentials is None:
application_credentials = os.getenv(
"GOOGLE_APPLICATION_CREDENTIALS")
if storage_client is not None:
self.client = storage_client
elif credentials is not None:
self.client = StorageClient(credentials=credentials,
project=project)
elif application_credentials is not None:
self.client = StorageClient.from_service_account_json(
application_credentials)
else:
self.client = StorageClient.create_anonymous_client()
super().__init__(local_cache_dir=local_cache_dir)
def _open_fs(self, user_context):
props = self._serialization_props(user_context)
bucket_name = props.pop('bucket_name', None)
root_path = props.pop('root_path', None)
project = props.pop('project', None)
args = {}
if props.get('anonymous'):
args['client'] = Client.create_anonymous_client()
elif props.get('token'):
args['client'] = Client(project=project,
credentials=Credentials(**props))
handle = GCSFS(bucket_name, root_path=root_path, retry=0, **args)
return handle
def get_public_client(self):
from google.cloud.storage import Client
return Client.create_anonymous_client()
