def _open_gcs_url(self, binary) -> object: mode = "rb" if binary else "r" service_account_json = self._provider.get("service_account_json") credentials = None if service_account_json: try: credentials = json.loads( self._provider["service_account_json"]) except json.decoder.JSONDecodeError as err: error_msg = f"Failed to parse gcs service account json: {repr(err)}\n{traceback.format_exc()}" logger.error(error_msg) raise ConfigurationError(error_msg) from err if credentials: credentials = service_account.Credentials.from_service_account_info( credentials) client = GCSClient(credentials=credentials, project=credentials._project_id) else: client = GCSClient.create_anonymous_client() file_to_close = smart_open.open(self.full_url, transport_params=dict(client=client), mode=mode) return file_to_close
def open_gcs_url(config, logger, storage, url): reader_impl = SourceFile.extract_reader_impl(config) use_gcs_service_account = "service_account_json" in config["provider"] and storage == "gs://" file_to_close = None if reader_impl == "gcsfs": if use_gcs_service_account: try: token_dict = json.loads(config["provider"]["service_account_json"]) except json.decoder.JSONDecodeError as err: logger.error(f"Failed to parse gcs service account json: {repr(err)}\n{traceback.format_exc()}") raise err else: token_dict = "anon" fs = gcsfs.GCSFileSystem(token=token_dict) file_to_close = fs.open(f"gs://{url}") result = file_to_close else: if use_gcs_service_account: try: credentials = json.dumps(json.loads(config["provider"]["service_account_json"])) tmp_service_account = tempfile.NamedTemporaryFile(delete=False) with open(tmp_service_account, "w") as f: f.write(credentials) tmp_service_account.close() client = Client.from_service_account_json(tmp_service_account.name) result = open(f"gs://{url}", transport_params=dict(client=client)) os.remove(tmp_service_account.name) except json.decoder.JSONDecodeError as err: logger.error(f"Failed to parse gcs service account json: {repr(err)}\n{traceback.format_exc()}") raise err else: client = Client.create_anonymous_client() result = open(f"{storage}{url}", transport_params=dict(client=client)) return result, file_to_close
def __init__( self, application_credentials: Optional[Union[str, os.PathLike]] = None, credentials: Optional["Credentials"] = None, project: Optional[str] = None, storage_client: Optional["StorageClient"] = None, local_cache_dir: Optional[Union[str, os.PathLike]] = None, ): """Class constructor. Sets up a [`Storage Client`](https://googleapis.dev/python/storage/latest/client.html). Supports the following authentication methods of `Storage Client`. - Environment variable `"GOOGLE_APPLICATION_CREDENTIALS"` containing a path to a JSON credentials file for a Google service account. See [Authenticating as a Service Account](https://cloud.google.com/docs/authentication/production). - File path to a JSON credentials file for a Google service account. - OAuth2 Credentials object and a project name. - Instantiated and already authenticated `Storage Client`. If multiple methods are used, priority order is reverse of list above (later in list takes priority). If no authentication methods are used, then the client will be instantiated as anonymous, which will only have access to public buckets. Args: application_credentials (Optional[Union[str, os.PathLike]]): Path to Google service account credentials file. credentials (Optional[Credentials]): The OAuth2 Credentials to use for this client. See documentation for [`StorageClient`]( https://googleapis.dev/python/storage/latest/client.html). project (Optional[str]): The project which the client acts on behalf of. See documentation for [`StorageClient`]( https://googleapis.dev/python/storage/latest/client.html). storage_client (Optional[StorageClient]): Instantiated [`StorageClient`]( https://googleapis.dev/python/storage/latest/client.html). local_cache_dir (Optional[Union[str, os.PathLike]]): Path to directory to use as cache for downloaded files. If None, will use a temporary directory. """ if application_credentials is None: application_credentials = os.getenv( "GOOGLE_APPLICATION_CREDENTIALS") if storage_client is not None: self.client = storage_client elif credentials is not None: self.client = StorageClient(credentials=credentials, project=project) elif application_credentials is not None: self.client = StorageClient.from_service_account_json( application_credentials) else: self.client = StorageClient.create_anonymous_client() super().__init__(local_cache_dir=local_cache_dir)
def _open_fs(self, user_context): props = self._serialization_props(user_context) bucket_name = props.pop('bucket_name', None) root_path = props.pop('root_path', None) project = props.pop('project', None) args = {} if props.get('anonymous'): args['client'] = Client.create_anonymous_client() elif props.get('token'): args['client'] = Client(project=project, credentials=Credentials(**props)) handle = GCSFS(bucket_name, root_path=root_path, retry=0, **args) return handle
def get_public_client(self): from google.cloud.storage import Client return Client.create_anonymous_client()