def DownloadDiscoveryDocument(api_name, version, path=_DEFAULT_DISCOVERY_PATH, label=None): """Downloads a discovery document for the given api_name and version. This utility assumes that the API for which a discovery document is being retrieved is publicly accessible. However, you may access whitelisted resources for a public API if you are added to its whitelist and specify the associated label. Args: api_name: a str indicating the name of the API for which a discovery document is to be downloaded. version: a str indicating the version number of the API. path: a str indicating the path to which you want to save the downloaded discovery document. label: a str indicating a label to be applied to the discovery service request. This is not applicable when downloading the discovery document of a legacy API. For non-legacy APIs, this may be used as a means of programmatically retrieving a copy of a discovery document containing whitelisted content. Raises: ValueError: if labels are specified for a legacy API, which is incompatible with labels. """ credentials = _GetCredentials() auth_session = AuthorizedSession(credentials) discovery_service = build('discovery', 'v1') discovery_rest_url = None discovery_response = discovery_service.apis().list( name=api_name).execute() if 'items' in discovery_response: for api in discovery_response['items']: if api['version'] == version: discovery_rest_url = api['discoveryRestUrl'] break if discovery_rest_url: is_legacy = _IsLegacy(discovery_rest_url) else: raise ValueError('API with name "%s" and version "%s" was not found.' % (api_name, version)) if all((is_legacy, label)): raise ValueError('The discovery URL associated with the api_name "%s" and ' 'version "%s" is for a legacy API. These are not ' 'compatible with labels.' % (api_name, version)) if label: # Apply the label query parameter if it exists. path_params = '&labels=%s' % label discovery_rest_url += path_params discovery_response = auth_session.get(discovery_rest_url) if discovery_response.status_code == 200: with open(path, 'wb') as handler: handler.write(discovery_response.text) else: raise ValueError('Unable to retrieve discovery document for api name "%s" ' 'and version "%s" via discovery URL: %s' % (api_name, version, discovery_rest_url))
def session(self): return AuthorizedSession( self._credentials())
def setup_session(): scopes = ['https://www.googleapis.com/auth/cloud-platform'] creds, _ = google.auth.default(scopes) return AuthorizedSession(creds)
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ A script to illustrate / test using ID token credentials generated from the standard service account credential json file. """ import json import os from google.auth.transport.requests import AuthorizedSession from google.oauth2.service_account import IDTokenCredentials APP_CREDENTIALS = os.getenv('GOOGLE_APPLICATION_CREDENTIALS') APPENGINE_CLIENT_ID = os.getenv('APPENGINE_CLIENT_ID') credentials = IDTokenCredentials.from_service_account_file( APP_CREDENTIALS, target_audience=APPENGINE_CLIENT_ID) authed_session = AuthorizedSession(credentials) with open('example.json', 'r') as f: payload = json.loads(f.read()) resp = authed_session.post('http://localhost:8080/events', json=payload) print(resp, resp.content)
import google.auth from google.auth.transport.requests import AuthorizedSession from requests.exceptions import HTTPError import os credentials, project = google.auth.default( scopes=['openid', 'email', 'profile']) base_url = os.environ["API_URL"] dataset_id = os.environ["DATASET_ID"] source_path = os.environ["SOURCE_PATH"] table_name = os.environ["TABLE_NAME"] authed_session = AuthorizedSession(credentials) def ingest_table(dataset_id: str, **kwargs): response = authed_session.post( f"{base_url}/api/repository/v1/datasets/{dataset_id}/ingest", json=kwargs) if response.ok: return response.json()["id"] else: raise HTTPError(f"Bad response, got code of: {response.status_code}") # print the job id to std out print( ingest_table(dataset_id, format="json", ignore_unknown_values=False,
class MachineTypesRestTransport(MachineTypesTransport): """REST backend transport for MachineTypes. The MachineTypes API. This class defines the same methods as the primary client, so the primary client can load the underlying transport implementation and call it. It sends JSON representations of protocol buffers over HTTP/1.1 """ def __init__( self, *, host: str = "compute.googleapis.com", credentials: ga_credentials.Credentials = None, credentials_file: str = None, scopes: Sequence[str] = None, client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, quota_project_id: Optional[str] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, always_use_jwt_access: Optional[bool] = False, ) -> None: """Instantiate the transport. Args: host (Optional[str]): The hostname to connect to. credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none are specified, the client will attempt to ascertain the credentials from the environment. credentials_file (Optional[str]): A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. This argument is ignored if ``channel`` is provided. scopes (Optional(Sequence[str])): A list of scopes. This argument is ignored if ``channel`` is provided. client_cert_source_for_mtls (Callable[[], Tuple[bytes, bytes]]): Client certificate to configure mutual TLS HTTP channel. It is ignored if ``channel`` is provided. quota_project_id (Optional[str]): An optional project to use for billing and quota. client_info (google.api_core.gapic_v1.client_info.ClientInfo): The client info used to send a user-agent string along with API requests. If ``None``, then default info will be used. Generally, you only need to set this if you're developing your own client library. """ # Run the base constructor # TODO(yon-mg): resolve other ctor params i.e. scopes, quota, etc. # TODO: When custom host (api_endpoint) is set, `scopes` must *also* be set on the # credentials object super().__init__( host=host, credentials=credentials, client_info=client_info, always_use_jwt_access=always_use_jwt_access, ) self._session = AuthorizedSession(self._credentials, default_host=self.DEFAULT_HOST) if client_cert_source_for_mtls: self._session.configure_mtls_channel(client_cert_source_for_mtls) self._prep_wrapped_messages(client_info) def aggregated_list( self, request: compute.AggregatedListMachineTypesRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.MachineTypeAggregatedList: r"""Call the aggregated list method over HTTP. Args: request (~.compute.AggregatedListMachineTypesRequest): The request object. A request message for MachineTypes.AggregatedList. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.MachineTypeAggregatedList: """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/aggregated/machineTypes".format( host=self._host, project=request.project, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.AggregatedListMachineTypesRequest.filter in request: query_params["filter"] = request.filter if compute.AggregatedListMachineTypesRequest.include_all_scopes in request: query_params["includeAllScopes"] = request.include_all_scopes if compute.AggregatedListMachineTypesRequest.max_results in request: query_params["maxResults"] = request.max_results if compute.AggregatedListMachineTypesRequest.order_by in request: query_params["orderBy"] = request.order_by if compute.AggregatedListMachineTypesRequest.page_token in request: query_params["pageToken"] = request.page_token if compute.AggregatedListMachineTypesRequest.return_partial_success in request: query_params[ "returnPartialSuccess"] = request.return_partial_success # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.get( url, headers=headers, params=query_params, ) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.MachineTypeAggregatedList.from_json( response.content, ignore_unknown_fields=True) def get( self, request: compute.GetMachineTypeRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.MachineType: r"""Call the get method over HTTP. Args: request (~.compute.GetMachineTypeRequest): The request object. A request message for MachineTypes.Get. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.MachineType: Represents a Machine Type resource. You can use specific machine types for your VM instances based on performance and pricing requirements. For more information, read Machine Types. (== resource_for {$api_version}.machineTypes ==) """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/zones/{zone}/machineTypes/{machine_type}".format( host=self._host, project=request.project, zone=request.zone, machine_type=request.machine_type, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.get( url, headers=headers, params=query_params, ) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.MachineType.from_json(response.content, ignore_unknown_fields=True) def list( self, request: compute.ListMachineTypesRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.MachineTypeList: r"""Call the list method over HTTP. Args: request (~.compute.ListMachineTypesRequest): The request object. A request message for MachineTypes.List. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.MachineTypeList: Contains a list of machine types. """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/zones/{zone}/machineTypes".format( host=self._host, project=request.project, zone=request.zone, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.ListMachineTypesRequest.filter in request: query_params["filter"] = request.filter if compute.ListMachineTypesRequest.max_results in request: query_params["maxResults"] = request.max_results if compute.ListMachineTypesRequest.order_by in request: query_params["orderBy"] = request.order_by if compute.ListMachineTypesRequest.page_token in request: query_params["pageToken"] = request.page_token if compute.ListMachineTypesRequest.return_partial_success in request: query_params[ "returnPartialSuccess"] = request.return_partial_success # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.get( url, headers=headers, params=query_params, ) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.MachineTypeList.from_json(response.content, ignore_unknown_fields=True)
def _connect_service(self, fn): # raises exception if file does not match expectation credentials = service_account.Credentials.from_service_account_file( fn, scopes=[self.scope]) self.session = AuthorizedSession(credentials)
class RoutesRestTransport(RoutesTransport): """REST backend transport for Routes. The Routes API. This class defines the same methods as the primary client, so the primary client can load the underlying transport implementation and call it. It sends JSON representations of protocol buffers over HTTP/1.1 """ _STUBS: Dict[str, RoutesRestStub] = {} def __init__( self, *, host: str = "compute.googleapis.com", credentials: ga_credentials.Credentials = None, credentials_file: str = None, scopes: Sequence[str] = None, client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, quota_project_id: Optional[str] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, always_use_jwt_access: Optional[bool] = False, url_scheme: str = "https", ) -> None: """Instantiate the transport. Args: host (Optional[str]): The hostname to connect to. credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none are specified, the client will attempt to ascertain the credentials from the environment. credentials_file (Optional[str]): A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. This argument is ignored if ``channel`` is provided. scopes (Optional(Sequence[str])): A list of scopes. This argument is ignored if ``channel`` is provided. client_cert_source_for_mtls (Callable[[], Tuple[bytes, bytes]]): Client certificate to configure mutual TLS HTTP channel. It is ignored if ``channel`` is provided. quota_project_id (Optional[str]): An optional project to use for billing and quota. client_info (google.api_core.gapic_v1.client_info.ClientInfo): The client info used to send a user-agent string along with API requests. If ``None``, then default info will be used. Generally, you only need to set this if you are developing your own client library. always_use_jwt_access (Optional[bool]): Whether self signed JWT should be used for service account credentials. url_scheme: the protocol scheme for the API endpoint. Normally "https", but for testing or local servers, "http" can be specified. """ # Run the base constructor # TODO(yon-mg): resolve other ctor params i.e. scopes, quota, etc. # TODO: When custom host (api_endpoint) is set, `scopes` must *also* be set on the # credentials object super().__init__( host=host, credentials=credentials, client_info=client_info, always_use_jwt_access=always_use_jwt_access, ) self._session = AuthorizedSession(self._credentials, default_host=self.DEFAULT_HOST) if client_cert_source_for_mtls: self._session.configure_mtls_channel(client_cert_source_for_mtls) self._prep_wrapped_messages(client_info) class _Delete(RoutesRestStub): def __hash__(self): return hash("Delete") __REQUIRED_FIELDS_DEFAULT_VALUES = {} @classmethod def _get_unset_required_fields(cls, message_dict): return { k: v for k, v in cls.__REQUIRED_FIELDS_DEFAULT_VALUES.items() if k not in message_dict } def __call__( self, request: compute.DeleteRouteRequest, *, retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Operation: r"""Call the delete method over HTTP. Args: request (~.compute.DeleteRouteRequest): The request object. A request message for Routes.Delete. See the method description for details. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Operation: Represents an Operation resource. Google Compute Engine has three Operation resources: \* `Global </compute/docs/reference/rest/v1/globalOperations>`__ \* `Regional </compute/docs/reference/rest/v1/regionOperations>`__ \* `Zonal </compute/docs/reference/rest/v1/zoneOperations>`__ You can use an operation resource to manage asynchronous API requests. For more information, read Handling API responses. Operations can be global, regional or zonal. - For global operations, use the ``globalOperations`` resource. - For regional operations, use the ``regionOperations`` resource. - For zonal operations, use the ``zonalOperations`` resource. For more information, read Global, Regional, and Zonal Resources. """ http_options = [ { "method": "delete", "uri": "/compute/v1/projects/{project}/global/routes/{route}", }, ] request_kwargs = compute.DeleteRouteRequest.to_dict(request) transcoded_request = path_template.transcode( http_options, **request_kwargs) uri = transcoded_request["uri"] method = transcoded_request["method"] # Jsonify the query params query_params = json.loads( compute.DeleteRouteRequest.to_json( compute.DeleteRouteRequest( transcoded_request["query_params"]), including_default_value_fields=False, use_integers_for_enums=False, )) query_params.update(self._get_unset_required_fields(query_params)) # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = getattr(self._session, method)( # Replace with proper schema configuration (http/https) logic "https://{host}{uri}".format(host=self._host, uri=uri), timeout=timeout, headers=headers, params=rest_helpers.flatten_query_params(query_params), ) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Operation.from_json(response.content, ignore_unknown_fields=True) class _Get(RoutesRestStub): def __hash__(self): return hash("Get") __REQUIRED_FIELDS_DEFAULT_VALUES = {} @classmethod def _get_unset_required_fields(cls, message_dict): return { k: v for k, v in cls.__REQUIRED_FIELDS_DEFAULT_VALUES.items() if k not in message_dict } def __call__( self, request: compute.GetRouteRequest, *, retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Route: r"""Call the get method over HTTP. Args: request (~.compute.GetRouteRequest): The request object. A request message for Routes.Get. See the method description for details. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Route: Represents a Route resource. A route defines a path from VM instances in the VPC network to a specific destination. This destination can be inside or outside the VPC network. For more information, read the Routes overview. """ http_options = [ { "method": "get", "uri": "/compute/v1/projects/{project}/global/routes/{route}", }, ] request_kwargs = compute.GetRouteRequest.to_dict(request) transcoded_request = path_template.transcode( http_options, **request_kwargs) uri = transcoded_request["uri"] method = transcoded_request["method"] # Jsonify the query params query_params = json.loads( compute.GetRouteRequest.to_json( compute.GetRouteRequest( transcoded_request["query_params"]), including_default_value_fields=False, use_integers_for_enums=False, )) query_params.update(self._get_unset_required_fields(query_params)) # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = getattr(self._session, method)( # Replace with proper schema configuration (http/https) logic "https://{host}{uri}".format(host=self._host, uri=uri), timeout=timeout, headers=headers, params=rest_helpers.flatten_query_params(query_params), ) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Route.from_json(response.content, ignore_unknown_fields=True) class _Insert(RoutesRestStub): def __hash__(self): return hash("Insert") __REQUIRED_FIELDS_DEFAULT_VALUES = {} @classmethod def _get_unset_required_fields(cls, message_dict): return { k: v for k, v in cls.__REQUIRED_FIELDS_DEFAULT_VALUES.items() if k not in message_dict } def __call__( self, request: compute.InsertRouteRequest, *, retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Operation: r"""Call the insert method over HTTP. Args: request (~.compute.InsertRouteRequest): The request object. A request message for Routes.Insert. See the method description for details. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Operation: Represents an Operation resource. Google Compute Engine has three Operation resources: \* `Global </compute/docs/reference/rest/v1/globalOperations>`__ \* `Regional </compute/docs/reference/rest/v1/regionOperations>`__ \* `Zonal </compute/docs/reference/rest/v1/zoneOperations>`__ You can use an operation resource to manage asynchronous API requests. For more information, read Handling API responses. Operations can be global, regional or zonal. - For global operations, use the ``globalOperations`` resource. - For regional operations, use the ``regionOperations`` resource. - For zonal operations, use the ``zonalOperations`` resource. For more information, read Global, Regional, and Zonal Resources. """ http_options = [ { "method": "post", "uri": "/compute/v1/projects/{project}/global/routes", "body": "route_resource", }, ] request_kwargs = compute.InsertRouteRequest.to_dict(request) transcoded_request = path_template.transcode( http_options, **request_kwargs) # Jsonify the request body body = compute.Route.to_json( compute.Route(transcoded_request["body"]), including_default_value_fields=False, use_integers_for_enums=False, ) uri = transcoded_request["uri"] method = transcoded_request["method"] # Jsonify the query params query_params = json.loads( compute.InsertRouteRequest.to_json( compute.InsertRouteRequest( transcoded_request["query_params"]), including_default_value_fields=False, use_integers_for_enums=False, )) query_params.update(self._get_unset_required_fields(query_params)) # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = getattr(self._session, method)( # Replace with proper schema configuration (http/https) logic "https://{host}{uri}".format(host=self._host, uri=uri), timeout=timeout, headers=headers, params=rest_helpers.flatten_query_params(query_params), data=body, ) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Operation.from_json(response.content, ignore_unknown_fields=True) class _List(RoutesRestStub): def __hash__(self): return hash("List") __REQUIRED_FIELDS_DEFAULT_VALUES = {} @classmethod def _get_unset_required_fields(cls, message_dict): return { k: v for k, v in cls.__REQUIRED_FIELDS_DEFAULT_VALUES.items() if k not in message_dict } def __call__( self, request: compute.ListRoutesRequest, *, retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.RouteList: r"""Call the list method over HTTP. Args: request (~.compute.ListRoutesRequest): The request object. A request message for Routes.List. See the method description for details. retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.RouteList: Contains a list of Route resources. """ http_options = [ { "method": "get", "uri": "/compute/v1/projects/{project}/global/routes", }, ] request_kwargs = compute.ListRoutesRequest.to_dict(request) transcoded_request = path_template.transcode( http_options, **request_kwargs) uri = transcoded_request["uri"] method = transcoded_request["method"] # Jsonify the query params query_params = json.loads( compute.ListRoutesRequest.to_json( compute.ListRoutesRequest( transcoded_request["query_params"]), including_default_value_fields=False, use_integers_for_enums=False, )) query_params.update(self._get_unset_required_fields(query_params)) # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = getattr(self._session, method)( # Replace with proper schema configuration (http/https) logic "https://{host}{uri}".format(host=self._host, uri=uri), timeout=timeout, headers=headers, params=rest_helpers.flatten_query_params(query_params), ) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.RouteList.from_json(response.content, ignore_unknown_fields=True) @property def delete( self) -> Callable[[compute.DeleteRouteRequest], compute.Operation]: stub = self._STUBS.get("delete") if not stub: stub = self._STUBS["delete"] = self._Delete( self._session, self._host) return stub @property def get(self) -> Callable[[compute.GetRouteRequest], compute.Route]: stub = self._STUBS.get("get") if not stub: stub = self._STUBS["get"] = self._Get(self._session, self._host) return stub @property def insert( self) -> Callable[[compute.InsertRouteRequest], compute.Operation]: stub = self._STUBS.get("insert") if not stub: stub = self._STUBS["insert"] = self._Insert( self._session, self._host) return stub @property def list(self) -> Callable[[compute.ListRoutesRequest], compute.RouteList]: stub = self._STUBS.get("list") if not stub: stub = self._STUBS["list"] = self._List(self._session, self._host) return stub def close(self): self._session.close()
def _connect_cloud(self): credentials = gauth.compute_engine.Credentials() self.session = AuthorizedSession(credentials)
def _connect_cache(self): project, access = self.project, self.access if (project, access) in self.tokens: credentials = self.tokens[(project, access)] self.session = AuthorizedSession(credentials)
def _connect_google_default(self): credentials, project = gauth.default(scopes=[self.scope]) self.project = project self.session = AuthorizedSession(credentials)
class GCSFileSystem(fsspec.AbstractFileSystem): """ Connect to Google Cloud Storage. The following modes of authentication are supported: - ``token=None``, GCSFS will attempt to guess your credentials in the following order: gcloud CLI default, gcsfs cached token, google compute metadata service, anonymous. - ``token='google_default'``, your default gcloud credentials will be used, which are typically established by doing ``gcloud login`` in a terminal. - ``token=='cache'``, credentials from previously successful gcsfs authentication will be used (use this after "browser" auth succeeded) - ``token='anon'``, no authentication is preformed, and you can only access data which is accessible to allUsers (in this case, the project and access level parameters are meaningless) - ``token='browser'``, you get an access code with which you can authenticate via a specially provided URL - if ``token='cloud'``, we assume we are running within google compute or google container engine, and query the internal metadata directly for a token. - you may supply a token generated by the [gcloud](https://cloud.google.com/sdk/docs/) utility; this is either a python dictionary, the name of a file containing the JSON returned by logging in with the gcloud CLI tool, or a Credentials object. gcloud typically stores its tokens in locations such as ``~/.config/gcloud/application_default_credentials.json``, `` ~/.config/gcloud/credentials``, or ``~\AppData\Roaming\gcloud\credentials``, etc. Specific methods, (eg. `ls`, `info`, ...) may return object details from GCS. These detailed listings include the [object resource](https://cloud.google.com/storage/docs/json_api/v1/objects#resource) GCS *does not* include "directory" objects but instead generates directories by splitting [object names](https://cloud.google.com/storage/docs/key-terms). This means that, for example, a directory does not need to exist for an object to be created within it. Creating an object implicitly creates it's parent directories, and removing all objects from a directory implicitly deletes the empty directory. `GCSFileSystem` generates listing entries for these implied directories in listing apis with the object properies: - "name" : string The "{bucket}/{name}" path of the dir, used in calls to GCSFileSystem or GCSFile. - "bucket" : string The name of the bucket containing this object. - "kind" : 'storage#object' - "size" : 0 - "storageClass" : 'DIRECTORY' - type: 'directory' (fsspec compat) GCSFileSystem maintains a per-implied-directory cache of object listings and fulfills all object information and listing requests from cache. This implied, for example, that objects created via other processes *will not* be visible to the GCSFileSystem until the cache refreshed. Calls to GCSFileSystem.open and calls to GCSFile are not effected by this cache. In the default case the cache is never expired. This may be controlled via the `cache_timeout` GCSFileSystem parameter or via explicit calls to `GCSFileSystem.invalidate_cache`. Parameters ---------- project : string project_id to work under. Note that this is not the same as, but ofter very similar to, the project name. This is required in order to list all the buckets you have access to within a project and to create/delete buckets, or update their access policies. If ``token='google_default'``, the value is overriden by the default, if ``token='anon'``, the value is ignored. access : one of {'read_only', 'read_write', 'full_control'} Full control implies read/write as well as modifying metadata, e.g., access control. token: None, dict or string (see description of authentication methods, above) consistency: 'none', 'size', 'md5' Check method when writing files. Can be overridden in open(). cache_timeout: float, seconds Cache expiration time in seconds for object metadata cache. Set cache_timeout <= 0 for no caching, None for no cache expiration. secure_serialize: bool If True, instances re-establish auth upon deserialization; if False, token is passed directly, which may be a security risk if passed across an insecure network. check_connection: bool When token=None, gcsfs will attempt various methods of establishing credentials, falling back to anon. It is possible for a methoc to find credentials in the system that turn out not to be valid. Setting this parameter to True will ensure that an actual operation is attempted before deciding that credentials are valid. """ scopes = {'read_only', 'read_write', 'full_control'} retries = 6 # number of retries on http failure base = "https://www.googleapis.com/storage/v1/" _singleton = [None] _singleton_pars = [None] default_block_size = DEFAULT_BLOCK_SIZE protocol = 'gcs', 'gs' def __init__(self, project=DEFAULT_PROJECT, access='full_control', token=None, block_size=None, consistency='none', cache_timeout=None, secure_serialize=True, check_connection=True, requests_timeout=None, **kwargs): super().__init__(self, **kwargs) pars = (project, access, token, block_size, consistency, cache_timeout) if access not in self.scopes: raise ValueError('access must be one of {}', self.scopes) if project is None: warnings.warn('GCS project not set - cannot list or create buckets') if block_size is not None: self.default_block_size = block_size self.project = project self.access = access self.scope = "https://www.googleapis.com/auth/devstorage." + access self.consistency = consistency self.token = token self.cache_timeout = cache_timeout self.requests_timeout = requests_timeout self.check_credentials = check_connection self._listing_cache = {} self.session = None self.connect(method=token) if not secure_serialize: self.token = self.session.credentials @staticmethod def load_tokens(): """Get "browser" tokens from disc""" try: with open(tfile, 'rb') as f: tokens = pickle.load(f) # backwards compatability tokens = {k: (GCSFileSystem._dict_to_credentials(v) if isinstance(v, dict) else v) for k, v in tokens.items()} except Exception: tokens = {} GCSFileSystem.tokens = tokens def _connect_google_default(self): credentials, project = gauth.default(scopes=[self.scope]) self.project = project self.session = AuthorizedSession(credentials) def _connect_cloud(self): credentials = gauth.compute_engine.Credentials() self.session = AuthorizedSession(credentials) def _connect_cache(self): project, access = self.project, self.access if (project, access) in self.tokens: credentials = self.tokens[(project, access)] self.session = AuthorizedSession(credentials) def _dict_to_credentials(self, token): """ Convert old dict-style token. Does not preserve access token itself, assumes refresh required. """ try: token = service_account.Credentials.from_service_account_info( token, scopes=[self.scope]) except: token = Credentials( None, refresh_token=token['refresh_token'], client_secret=token['client_secret'], client_id=token['client_id'], token_uri='https://www.googleapis.com/oauth2/v4/token', scopes=[self.scope] ) return token def _connect_token(self, token): """ Connect using a concrete token Parameters ---------- token: str, dict or Credentials If a str, try to load as a Service file, or next as a JSON; if dict, try to interpret as credentials; if Credentials, use directly. """ if isinstance(token, str): if not os.path.exists(token): raise FileNotFoundError(token) try: # is this a "service" token? self._connect_service(token) return except: # some other kind of token file # will raise exception if is not json token = json.load(open(token)) if isinstance(token, dict): credentials = self._dict_to_credentials(token) elif isinstance(token, google.auth.credentials.Credentials): credentials = token else: raise ValueError('Token format not understood') self.session = AuthorizedSession(credentials) def _connect_service(self, fn): # raises exception if file does not match expectation credentials = service_account.Credentials.from_service_account_file( fn, scopes=[self.scope]) self.session = AuthorizedSession(credentials) def _connect_anon(self): self.session = requests.Session() def _connect_browser(self): flow = InstalledAppFlow.from_client_config(client_config, [self.scope]) credentials = flow.run_console() self.tokens[(self.project, self.access)] = credentials self._save_tokens() self.session = AuthorizedSession(credentials) def connect(self, method=None): """ Establish session token. A new token will be requested if the current one is within 100s of expiry. Parameters ---------- method: str (google_default|cache|cloud|token|anon|browser) or None Type of authorisation to implement - calls `_connect_*` methods. If None, will try sequence of methods. """ if method not in ['google_default', 'cache', 'cloud', 'token', 'anon', 'browser', None]: self._connect_token(method) elif method is None: for meth in ['google_default', 'cache', 'anon']: try: self.connect(method=meth) if self.check_credentials and meth != 'anon': self.ls('anaconda-public-data') except: self.session = None logger.debug('Connection with method "%s" failed' % meth) if self.session: break else: self.__getattribute__('_connect_' + method)() self.method = method if self.session is None: if method is None: msg = ("Automatic authentication failed, you should try " "specifying a method with the token= kwarg") else: msg = ("Auth failed with method '%s'. See the docstrings for " "further details about your auth mechanism, also " "available at https://gcsfs.readthedocs.io/en/latest/" "api.html#gcsfs.core.GCSFileSystem" % method) raise RuntimeError(msg) @staticmethod def _save_tokens(): try: with open(tfile, 'wb') as f: pickle.dump(GCSFileSystem.tokens, f, 2) except Exception as e: warnings.warn('Saving token cache failed: ' + str(e)) @_tracemethod def _call(self, method, path, *args, **kwargs): for k, v in list(kwargs.items()): if v is None: del kwargs[k] json = kwargs.pop('json', None) headers = kwargs.pop('headers', None) data = kwargs.pop('data', None) r = None if not path.startswith('http'): path = self.base + path if args: path = path.format(*[quote_plus(p) for p in args]) for retry in range(self.retries): try: if retry > 0: time.sleep(min(random.random() + 2**(retry-1), 32)) r = self.session.request(method, path, params=kwargs, json=json, headers=headers, data=data, timeout=self.requests_timeout) validate_response(r, path) break except (HttpError, RequestException, RateLimitException, GoogleAuthError) as e: if retry == self.retries - 1: logger.exception("_call out of retries on exception: %s", e) raise e if is_retriable(e): logger.debug("_call retrying after exception: %s", e) continue logger.exception("_call non-retriable exception: %s", e) raise e return r @property def buckets(self): """Return list of available project buckets.""" return [b["name"] for b in self._list_buckets()] @staticmethod def _process_object(bucket, object_metadata): """Process object resource into gcsfs object information format. Process GCS object resource via type casting and attribute updates to the cache-able gcsfs object information format. Returns an updated copy of the object resource. (See https://cloud.google.com/storage/docs/json_api/v1/objects#resource) """ result = dict(object_metadata) result["size"] = int(object_metadata.get("size", 0)) result["name"] = posixpath.join(bucket, object_metadata["name"]) result['type'] = 'file' return result @_tracemethod def _get_object(self, path): """Return object information at the given path.""" bucket, key = split_path(path) # Check if parent dir is in listing cache parent = "/".join([bucket, posixpath.dirname(key.rstrip("/"))]) + "/" parent_cache = self._maybe_get_cached_listing(parent) if parent_cache: cached_obj = [o for o in parent_cache["items"] if o["name"] == key] if cached_obj: logger.debug("found cached object: %s", cached_obj) return cached_obj[0] else: logger.debug("object not found cached parent listing") raise FileNotFoundError(path) if not key: # Attempt to "get" the bucket root, return error instead of # listing. raise FileNotFoundError(path) result = self._process_object(bucket, self._call('GET', 'b/{}/o/{}', bucket, key).json()) return result @_tracemethod def _maybe_get_cached_listing(self, path): logger.debug("_maybe_get_cached_listing: %s", path) if path in self._listing_cache: retrieved_time, listing = self._listing_cache[path] cache_age = time.time() - retrieved_time if self.cache_timeout is not None and cache_age > self.cache_timeout: logger.debug( "expired cache path: %s retrieved_time: %.3f cache_age: " "%.3f cache_timeout: %.3f", path, retrieved_time, cache_age, self.cache_timeout ) del self._listing_cache[path] return None return listing return None @_tracemethod def _list_objects(self, path): path = norm_path(path) clisting = self._maybe_get_cached_listing(path) if clisting: return clisting listing = self._do_list_objects(path) retrieved_time = time.time() self._listing_cache[path] = (retrieved_time, listing) return listing @_tracemethod def _do_list_objects(self, path, max_results=None): """Object listing for the given {bucket}/{prefix}/ path.""" bucket, prefix = split_path(path) if not prefix: prefix = None prefixes = [] items = [] page = self._call('GET', 'b/{}/o/', bucket, delimiter="/", prefix=prefix, maxResults=max_results ).json() assert page["kind"] == "storage#objects" prefixes.extend(page.get("prefixes", [])) items.extend([i for i in page.get("items", []) if prefix is None or i['name'].rstrip('/') == prefix.rstrip('/') or i['name'].startswith(prefix.rstrip('/') + '/')]) next_page_token = page.get('nextPageToken', None) while next_page_token is not None: page = self._call('GET', 'b/{}/o/', bucket, delimiter="/", prefix=prefix, maxResults=max_results, pageToken=next_page_token ).json() assert page["kind"] == "storage#objects" prefixes.extend(page.get("prefixes", [])) items.extend([ i for i in page.get("items", []) ]) next_page_token = page.get('nextPageToken', None) result = { "kind": "storage#objects", "prefixes": prefixes, "items": [self._process_object(bucket, i) for i in items], } return result @_tracemethod def _list_buckets(self): """Return list of all buckets under the current project.""" items = [] page = self._call('GET', 'b/', project=self.project).json() assert page["kind"] == "storage#buckets" items.extend(page.get("items", [])) next_page_token = page.get('nextPageToken', None) while next_page_token is not None: page = self._call( 'GET', 'b/', project=self.project, pageToken=next_page_token).json() assert page["kind"] == "storage#buckets" items.extend(page.get("items", [])) next_page_token = page.get('nextPageToken', None) return [{'name': i['name'] + '/', 'size': 0, 'type': "directory"} for i in items] @_tracemethod def invalidate_cache(self, path=None): """ Invalidate listing cache for given path, it is reloaded on next use. Parameters ---------- path: string or None If None, clear all listings cached else listings at or under given path. """ if not path: logger.debug("invalidate_cache clearing cache") self._listing_cache.clear() else: path = norm_path(path) invalid_keys = [k for k in self._listing_cache if k.startswith(path)] for k in invalid_keys: self._listing_cache.pop(k, None) @_tracemethod def mkdir(self, bucket, acl='projectPrivate', default_acl='bucketOwnerFullControl'): """ New bucket Parameters ---------- bucket: str bucket name. If contains '/' (i.e., looks like subdir), will have no effect because GCS doesn't have real directories. acl: string, one of bACLs access for the bucket itself default_acl: str, one of ACLs default ACL for objects created in this bucket """ if bucket in ['', '/']: raise ValueError('Cannot create root bucket') if '/' in bucket: return self._call('post', 'b/', predefinedAcl=acl, project=self.project, predefinedDefaultObjectAcl=default_acl, json={"name": bucket}) self.invalidate_cache(bucket) @_tracemethod def rmdir(self, bucket): """Delete an empty bucket Parameters ---------- bucket: str bucket name. If contains '/' (i.e., looks like subdir), will have no effect because GCS doesn't have real directories. """ if '/' in bucket: return self._call('delete', 'b/' + bucket) self.invalidate_cache(bucket) @_tracemethod def ls(self, path, detail=False): """List objects under the given '/{bucket}/{prefix} path.""" path = norm_path(path) if path in ['/', '']: if detail: return self._list_buckets() else: return self.buckets elif path.endswith("/"): return self._ls(path, detail) else: combined_listing = self._ls(path, detail) + self._ls(path + "/", detail) if detail: combined_entries = dict( (l["name"], l) for l in combined_listing) combined_entries.pop(path + "/", None) return list(combined_entries.values()) else: return list(set(combined_listing) - {path + "/"}) @_tracemethod def _ls(self, path, detail=False): listing = self._list_objects(path) bucket, key = split_path(path) item_details = listing["items"] pseudodirs = [{ 'bucket': bucket, 'name': bucket + "/" + prefix, 'kind': 'storage#object', 'size': 0, 'storageClass': 'DIRECTORY', 'type': 'directory' } for prefix in listing["prefixes"] ] out = item_details + pseudodirs if detail: return out else: return sorted([o['name'] for o in out]) @staticmethod def url(path): """ Get HTTP URL of the given path """ u = 'https://www.googleapis.com/download/storage/v1/b/{}/o/{}?alt=media' bucket, object = split_path(path) object = quote_plus(object) return u.format(bucket, object) @_tracemethod def cat(self, path): """ Simple one-shot get of file data """ u2 = self.url(path) r = self.session.get(u2) r.raise_for_status() if 'X-Goog-Hash' in r.headers: # if header includes md5 hash, check that data matches bits = r.headers['X-Goog-Hash'].split(',') for bit in bits: key, val = bit.split('=', 1) if key == 'md5': md = b64decode(val) assert md5(r.content).digest() == md, "Checksum failure" return r.content def getxattr(self, path, attr): """Get user-defined metadata attribute""" meta = self.info(path).get('metadata', {}) return meta[attr] def setxattrs(self, path, content_type=None, content_encoding=None, **kwargs): """ Set/delete/add writable metadata attributes Parameters --------- content_type: str If not None, set the content-type to this value content_encoding: str If not None, set the content-encoding. See https://cloud.google.com/storage/docs/transcoding kw_args: key-value pairs like field="value" or field=None value must be string to add or modify, or None to delete Returns ------- Entire metadata after update (even if only path is passed) """ i_json = {'metadata': kwargs} if content_type is not None: i_json['contentType'] = content_type if content_encoding is not None: i_json['contentEncoding'] = content_encoding bucket, key = split_path(path) o_json = self._call('PATCH', "b/{}/o/{}", bucket, key, fields='metadata', json=i_json ).json() self.info(path)['metadata'] = o_json.get('metadata', {}) return o_json.get('metadata', {}) @_tracemethod def merge(self, path, paths, acl=None): """Concatenate objects within a single bucket""" bucket, key = split_path(path) source = [{'name': split_path(p)[1]} for p in paths] self._call('POST', 'b/{}/o/{}/compose', bucket, key, destinationPredefinedAcl=acl, json={'sourceObjects': source, "kind": "storage#composeRequest", 'destination': {'name': key, 'bucket': bucket}}) @_tracemethod def copy(self, path1, path2, acl=None): """Duplicate remote file """ b1, k1 = split_path(path1) b2, k2 = split_path(path2) out = self._call('POST', 'b/{}/o/{}/rewriteTo/b/{}/o/{}', b1, k1, b2, k2, destinationPredefinedAcl=acl) while out.json()['done'] is not True: out = self._call( 'POST', 'b/{}/o/{}/rewriteTo/b/{}/o/{}', b1, k1, b2, k2, rewriteToken=out['rewriteToken'], destinationPredefinedAcl=acl) @_tracemethod def rm(self, path, recursive=False): """Delete keys. If a list, batch-delete all keys in one go (can span buckets) Returns whether operation succeeded (a list if input was a list) If recursive, delete all keys given by find(path) """ if isinstance(path, (tuple, list)): template = ('\n--===============7330845974216740156==\n' 'Content-Type: application/http\n' 'Content-Transfer-Encoding: binary\n' 'Content-ID: <b29c5de2-0db4-490b-b421-6a51b598bd11+{i}>' '\n\nDELETE /storage/v1/b/{bucket}/o/{key} HTTP/1.1\n' 'Content-Type: application/json\n' 'accept: application/json\ncontent-length: 0\n') body = "".join([template.format(i=i+1, bucket=p.split('/', 1)[0], key=quote_plus(p.split('/', 1)[1])) for i, p in enumerate(path)]) r = self._call( 'POST', 'https://www.googleapis.com/batch', headers={'Content-Type': 'multipart/mixed; boundary="==========' '=====7330845974216740156=="'}, data=body + "\n--===============7330845974216740156==--") boundary = r.headers['Content-Type'].split('=', 1)[1] parents = {posixpath.dirname(norm_path(p)) for p in path} [self.invalidate_cache(parent) for parent in parents] return ['200 OK' in c or '204 No Content' in c for c in r.text.split(boundary)][1:-1] elif recursive: return self.rm(self.find(path)) else: bucket, key = split_path(path) self._call('DELETE', "b/{}/o/{}", bucket, key) self.invalidate_cache(posixpath.dirname(norm_path(path))) return True @_tracemethod def _open(self, path, mode='rb', block_size=None, acl=None, consistency=None, metadata=None, autocommit=True, **kwargs): """ See ``GCSFile``. consistency: None or str If None, use default for this instance """ if block_size is None: block_size = self.default_block_size const = consistency or self.consistency return GCSFile(self, path, mode, block_size, consistency=const, metadata=metadata, acl=acl, autocommit=autocommit, **kwargs) def __setstate__(self, state): self.__dict__.update(state) self.dircache = {} self.connect(self.token)
class GooglePlayService(InAppService): scopes = ['https://www.googleapis.com/auth/androidpublisher'] base_uri = 'https://www.googleapis.com/androidpublisher/v3/applications' products_uri = '/%packageName%/inappproducts' product_purchase_uri = '/%packageName%/purchases/products/%productId%/tokens/%token%' subscription_purchase_uri = '/%packageName%/purchases/subscriptions/%subscriptionId%/'+\ 'tokens/%token%' service_account_info = None service_account_file = None credentials = None authed_session = None package_name = None def __init__(self, service_account_info=None, service_account_file=None, package_name=None, scopes=None, base_uri=None, products_uri=None, product_purchase_uri=None, subscription_purchase_uri=None): if scopes is not None: self.scopes = scopes if base_uri is not None: self.base_uri = base_uri if products_uri is not None: self.products_uri = products_uri if product_purchase_uri is not None: self.product_purchase_uri = product_purchase_uri if subscription_purchase_uri is not None: self.subscription_purchase_uri = subscription_purchase_uri if service_account_info is not None: self.service_account_info = service_account_info if service_account_file is not None: self.service_account_file = service_account_file if package_name is not None: self.package_name = package_name if self.service_account_info is not None or self.service_account_file is not None: self.authed_session = self.create_session() def set_service_account_info(self, service_account_info): self.service_account_info = service_account_info return self def set_service_account_file(self, service_account_file): self.service_account_file = service_account_file return self def generate_credentials(self): if self.service_account_info is not None: self.credentials = service_account.Credentials.from_service_account_info( self.service_account_info, scopes=self.scopes) elif self.service_account_file is not None: self.credentials = service_account.Credentials.from_service_account_file( self.service_account_file, scopes=self.scopes) return self.credentials def create_session(self): if self.credentials is None: self.credentials = self.generate_credentials() if self.credentials is not None: self.authed_session = AuthorizedSession(self.credentials) return self.authed_session if not self.authed_session: self.authed_session = None return self.authed_session def refresh_session(self): self.authed_session = self.create_session() def close_session(self): return self.authed_session.close() def get_products(self, package_name=None): package_name = package_name if package_name is not None else self.package_name response = self.request( self.base_uri + self.products_uri.replace('%packageName%', package_name)) return self.get_products_response(response) def get_product_purchase(self, product_id, token, package_name=None): package_name = package_name if package_name is not None else self.package_name response = self.request( self.base_uri + self.product_purchase_uri.replace('%packageName%', package_name). replace('%productId%', product_id).replace('%token%', token)) return self.get_product_response(response) def get_subscription_purchase(self, subscription_id, token, package_name=None): package_name = package_name if package_name is not None else self.package_name response = self.request( self.base_uri + self.subscription_purchase_uri. replace('%packageName%', package_name).replace( '%subscriptionId%', subscription_id).replace('%token%', token)) return self.get_subscription_response(response, additional_data={ 'package_name': package_name, 'subscription_id': subscription_id }) def request(self, query): if self.authed_session is not None: response = self.authed_session.get(query) return response else: raise BaseException("authed_session is not defined") def get_products_response(self, response, additional_data=None): data = None if response.ok: data = response.json()['inappproduct'] return super(GooglePlayService, self).get_products_response(response, data) def get_product_response(self, response, additional_data=None): data = None if response.ok: data = response.json() return super(GooglePlayService, self).get_product_response(response, data) def get_subscription_response(self, response, additional_data=None): data = None try: if response.ok: response_data = response.json() cancellation_date_ms = ( int(response_data['userCancellationTimeMillis']) if 'userCancellationTimeMillis' in response_data else None) cancellation_date = (int(cancellation_date_ms / 1000) if cancellation_date_ms is not None else None) expires_date_ms = (int(response_data['expiryTimeMillis']) if 'expiryTimeMillis' in response_data else None) expires_date = (int(expires_date_ms / 1000) if expires_date_ms is not None else None) cancellation_reason = (response_data['cancelReason'] if 'cancelReason' in response_data else None) payment_state = (response_data['paymentState'] if 'paymentState' in response_data else None) payment_state = (response_data['paymentState'] if 'paymentState' in response_data else None) if cancellation_reason is None: status = SubscriptionStatus.ACTIVE elif cancellation_reason is not None: status = SubscriptionStatus.CANCELLED is_active = status == SubscriptionStatus.ACTIVE data = { 'purchase_id': response_data['orderId'], 'original_purchase_id': response_data['orderId'], 'purchase_date_ms': int(response_data['startTimeMillis']), 'purchase_date': int(int(response_data['startTimeMillis']) / 1000), 'original_purchase_date_ms': int(response_data['startTimeMillis']), 'original_purchase_date': int(int(response_data['startTimeMillis']) / 1000), 'auto_renewing': response_data['autoRenewing'], 'expires_date_ms': expires_date_ms, 'expires_date': expires_date, 'country_code': response_data['countryCode'], 'price_currency_code': response_data['priceCurrencyCode'], 'price_amount': float( int(int(response_data['priceAmountMicros']) / 1000) / 100), 'cancellation_date_ms': cancellation_date_ms, 'cancellation_date': cancellation_date, 'cancellation_reason': cancellation_reason, 'payment_state': payment_state, 'status': status, 'is_active': is_active, 'is_trial_period': None, 'expiration_intent': None } if additional_data is not None: if 'package_name' in additional_data: data.update( {'bundle_id': additional_data['package_name']}) if 'subscription_id' in additional_data: data.update({ 'subscription_id': additional_data['subscription_id'] }) return super(GooglePlayService, self).get_subscription_response(response, data) except Exception: return super(GooglePlayService, self).get_subscription_response(response)
import google.auth from google.auth.transport.requests import AuthorizedSession import google.oauth2.credentials from google.oauth2 import service_account from google.auth import impersonated_credentials urllib3.disable_warnings() credentials = service_account.Credentials.from_service_account_file( './app_controllers/secrets/oauth2.json') scoped_credentials = credentials.with_scopes([ 'https://www.googleapis.com/auth/userinfo.profile', 'https://www.googleapis.com/auth/userinfo.email' ]) print("CREDENTIALS:", scoped_credentials) authed_session = AuthorizedSession(scoped_credentials) print("AUTHORIZED:", authed_session) """ 1. deploy,service,ingress - done 2. install plugin/extension via cli/script 3. connect to gitlab 3. connect to kubernetes host 4. trigger from gitlab push 5. deploy to kubernetes """ # Installs Gitlab & Git Plugin def jenkinsInstallPlugin(clusterName, userName): url = "http://jenkins-" + userName + "." + clusterName + ".securethebox.us/pluginManager/install"
def _connect_browser(self): flow = InstalledAppFlow.from_client_config(client_config, [self.scope]) credentials = flow.run_console() self.tokens[(self.project, self.access)] = credentials self._save_tokens() self.session = AuthorizedSession(credentials)
def get_auth_session(self, cred): return AuthorizedSession(cred)
def __init__( self, *, host: str = "compute.googleapis.com", credentials: ga_credentials.Credentials = None, credentials_file: str = None, scopes: Sequence[str] = None, client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, quota_project_id: Optional[str] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, always_use_jwt_access: Optional[bool] = False, url_scheme: str = "https", ) -> None: """Instantiate the transport. Args: host (Optional[str]): The hostname to connect to. credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none are specified, the client will attempt to ascertain the credentials from the environment. credentials_file (Optional[str]): A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. This argument is ignored if ``channel`` is provided. scopes (Optional(Sequence[str])): A list of scopes. This argument is ignored if ``channel`` is provided. client_cert_source_for_mtls (Callable[[], Tuple[bytes, bytes]]): Client certificate to configure mutual TLS HTTP channel. It is ignored if ``channel`` is provided. quota_project_id (Optional[str]): An optional project to use for billing and quota. client_info (google.api_core.gapic_v1.client_info.ClientInfo): The client info used to send a user-agent string along with API requests. If ``None``, then default info will be used. Generally, you only need to set this if you are developing your own client library. always_use_jwt_access (Optional[bool]): Whether self signed JWT should be used for service account credentials. url_scheme: the protocol scheme for the API endpoint. Normally "https", but for testing or local servers, "http" can be specified. """ # Run the base constructor # TODO(yon-mg): resolve other ctor params i.e. scopes, quota, etc. # TODO: When custom host (api_endpoint) is set, `scopes` must *also* be set on the # credentials object super().__init__( host=host, credentials=credentials, client_info=client_info, always_use_jwt_access=always_use_jwt_access, ) self._session = AuthorizedSession( self._credentials, default_host=self.DEFAULT_HOST ) if client_cert_source_for_mtls: self._session.configure_mtls_channel(client_cert_source_for_mtls) self._prep_wrapped_messages(client_info)
def exec_google_copy(fi, ignored_dict, global_config): """ copy a file to google bucket. Args: fi(dict): a dictionary of a copying file global_config(dict): a configuration { "chunk_size_download": 1024, "chunk_size_upload": 1024 } Returns: DataFlowLog """ if fi["size"] == 0: msg = "can not copy {} to GOOGLE bucket since it is empty file".format(fi["id"]) return DataFlowLog(message=msg) indexd_client = IndexClient( INDEXD["host"], INDEXD["version"], (INDEXD["auth"]["username"], INDEXD["auth"]["password"]), ) if not ignored_dict: raise UserError( "Expecting non-empty IGNORED_FILES. Please check if ignored_files_manifest.py is configured correctly!!!" ) try: bucket_name = utils.get_google_bucket_name(fi, PROJECT_ACL) except UserError as e: msg = "can not copy {} to GOOGLE bucket. Detail {}. {}".format( fi["id"], e, PROJECT_ACL ) logger.error(msg) return DataFlowLog(message=msg) if not bucket_exists(bucket_name): msg = "There is no bucket with provided name {}\n".format(bucket_name) logger.error(msg) return DataFlowLog(message=msg) if fi["id"] in ignored_dict: logger.info( "{} is ignored. Start to check indexd for u5aa objects".format(fi["id"]) ) _update_indexd_for_5aa_object(fi, bucket_name, ignored_dict, indexd_client) return DataFlowLog(message="{} is in the ignored list".format(fi["id"])) client = storage.Client() sess = AuthorizedSession(client._credentials) blob_name = fi.get("id") + "/" + fi.get("file_name") _check_and_handle_changed_acl_object(fi) if blob_exists(bucket_name, blob_name): logger.info("{} is already copied".format(fi["id"])) else: try: logger.info( "Start to stream {}. Size {} (MB)".format( fi["id"], fi["size"] * 1.0 / 1000 / 1000 ) ) tries = 0 while tries < NUM_STREAMING_TRIES: try: resumable_streaming_copy( fi, client, bucket_name, blob_name, global_config ) if fail_resumable_copy_blob(sess, bucket_name, blob_name, fi): delete_object(sess, bucket_name, blob_name) else: break except Exception as e: logger.warning(e) tries += 1 if tries == NUM_STREAMING_TRIES: logger.error( "Can not stream {} after multiple attemps".format(fi.get("id")) ) else: logger.info( "Finish streaming {}. Size {} (MB)".format( fi["id"], fi["size"] * 1.0 / 1000 / 1000 ) ) except APIError as e: logger.error(str(e)) return DataFlowLog(message=str(e)) except Exception as e: # Don't break (Not expected) logger.error(str(e)) return DataFlowLog(message=str(e)) # Confirm that the object was copied if blob_exists(bucket_name, blob_name): try: if indexd_utils.update_url(fi, indexd_client, provider="gs"): logger.info("Successfully update indexd for {}".format(fi["id"])) else: logger.info("Can not update indexd for {}".format(fi["id"])) except APIError as e: logger.error(e) return DataFlowLog(copy_success=True, message=e) else: msg = "can not copy {} to GOOGLE bucket after multiple attempts. Check the error detail in logs".format( blob_name ) logger.error(msg) return DataFlowLog(message=msg) return DataFlowLog( copy_success=True, index_success=True, message="object {} successfully copied ".format(blob_name), )
def list_retrohunts( http_session: requests.AuthorizedSession, version_id: str, retrohunt_state: str = "", page_size: int = 0, page_token: str = "") -> Tuple[Sequence[Mapping[str, Any]], str]: """Lists retrohunts. Args: http_session: Authorized session for HTTP requests. version_id: Unique ID of the detection rule to list retrohunts for. Valid version ID formats: Retrohunts for a specific rule version: "ru_<UUID>@v_<seconds>_<nanoseconds>" Retrohunts for the latest version of a rule: "ru_<UUID>" Retrohunts across all versions of a rule: "ru_<UUID>@-" Retrohunts across all rules and all versions: "-" retrohunt_state: The status of the retrohunt to filter by (i.e. 'RUNNING') (default = no filter on retrohunt state). Optional - retrohunts of all states are returned. page_size: Maximum number of retrohunts to return. Must be non-negative, and is capped at a server-side limit of 1000. Optional - a server-side default of 100 is used if the size is 0 or a None value. page_token: Page token from a previous ListRetrohunts call used for pagination. Optional - the first page is retrieved if the token is the empty string or a None value. Returns: All the retrohunts (within the defined page) ordered by descending retrohunt start time, as well as a Base64 token for getting the retrohunts of the next page (any empty token string means the currently retrieved page is the last one). Raises: requests.exceptions.HTTPError: HTTP request resulted in an error (response.status_code >= 400). """ url = f"{CHRONICLE_API_BASE_URL}/v2/detect/rules/{version_id}/retrohunts" params_list = [("state", retrohunt_state), ("page_size", page_size), ("page_token", page_token)] params = {k: v for k, v in params_list if v} response = http_session.request("GET", url, params=params) # Expected server response: # { # "retrohunts": [ # { # "retrohuntId": "oh_<UUID>", # "ruleId": "ru_<UUID>", # "versionId": "ru_<UUID>@v_<seconds>_<nanoseconds>", # "eventStartTime": "yyyy-mm-ddThh:mm:ss.ssssssZ", # "eventEndTime": "yyyy-mm-ddThh:mm:ss.ssssssZ", # "retrohuntStartTime": "yyyy-mm-ddThh:mm:ss.ssssssZ", # "retrohuntEndTime": "yyyy-mm-ddThh:mm:ss.ssssssZ", # "state": "RUNNING"/"DONE"/"CANCELLED", # "progressPercentage": "<value from 0.00 to 100.00>" # }, # ... # ] # "nextPageToken": "<next_page_token>" # } if response.status_code >= 400: print(response.text) response.raise_for_status() json = response.json() return json.get("retrohunts", []), json.get("nextPageToken", "")
def get_detection(http_session: requests.AuthorizedSession, version_id: str, detection_id: str) -> Mapping[str, Any]: """Get detection. Args: http_session: Authorized session for HTTP requests. version_id: Unique ID of the detection rule to retrieve errors for ("ru_<UUID>" or "ru_<UUID>@v_<seconds>_<nanoseconds>"). If a version suffix isn't specified we use the rule's latest version. detection_id: Id of the detection to get information for. Returns: Detection information. Raises: requests.exceptions.HTTPError: HTTP request resulted in an error (response.status_code >= 400). """ url = (f"{CHRONICLE_API_BASE_URL}/v2/detect/rules/{version_id}/" + f"detections/{detection_id}") response = http_session.request("GET", url) # Expected server response: # { # "id": "de_<UUID>", # "type": "RULE_DETECTION", # "createdTime": "yyyy-mm-ddThh:mm:ssZ", # "detectionTime": "yyyy-mm-ddThh:mm:ssZ", # "timeWindow": { # "startTime": "yyyy-mm-ddThh:mm:ssZ", # "endTime": "yyyy-mm-ddThh:mm:ssZ", # } # "collectionElements": [ # { # "label": "e1", # "references": [ # { # "event": <UDM keys and values / sub-dictionaries>... # }, # ... # ], # }, # { # "label": "e2", # ... # }, # ... # ], # "detection": [ # { # "ruleId": "ru_<UUID>", # "ruleName": "<rule_name>", # "ruleVersion": "ru_<UUID>@v_<seconds>_<nanoseconds>", # "urlBackToProduct": "<URL>", # "alertState": "ALERTING"/"NOT_ALERTING", # "ruleType": "SINGLE_EVENT"/"MULTI_EVENT", # "detectionFields": [ # { # "key": "<field name>", # "value": "<field value>" # } # ] # }, # ], # } if response.status_code >= 400: print(response.text) response.raise_for_status() return response.json()
class BackendBucketsRestTransport(BackendBucketsTransport): """REST backend transport for BackendBuckets. The BackendBuckets API. This class defines the same methods as the primary client, so the primary client can load the underlying transport implementation and call it. It sends JSON representations of protocol buffers over HTTP/1.1 """ def __init__( self, *, host: str = "compute.googleapis.com", credentials: ga_credentials.Credentials = None, credentials_file: str = None, scopes: Sequence[str] = None, client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, quota_project_id: Optional[str] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, ) -> None: """Instantiate the transport. Args: host (Optional[str]): The hostname to connect to. credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none are specified, the client will attempt to ascertain the credentials from the environment. credentials_file (Optional[str]): A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. This argument is ignored if ``channel`` is provided. scopes (Optional(Sequence[str])): A list of scopes. This argument is ignored if ``channel`` is provided. client_cert_source_for_mtls (Callable[[], Tuple[bytes, bytes]]): Client certificate to configure mutual TLS HTTP channel. It is ignored if ``channel`` is provided. quota_project_id (Optional[str]): An optional project to use for billing and quota. client_info (google.api_core.gapic_v1.client_info.ClientInfo): The client info used to send a user-agent string along with API requests. If ``None``, then default info will be used. Generally, you only need to set this if you're developing your own client library. """ # Run the base constructor # TODO(yon-mg): resolve other ctor params i.e. scopes, quota, etc. # TODO: When custom host (api_endpoint) is set, `scopes` must *also* be set on the # credentials object super().__init__( host=host, credentials=credentials, client_info=client_info, ) self._session = AuthorizedSession( self._credentials, default_host=self.DEFAULT_HOST ) if client_cert_source_for_mtls: self._session.configure_mtls_channel(client_cert_source_for_mtls) self._prep_wrapped_messages(client_info) def add_signed_url_key( self, request: compute.AddSignedUrlKeyBackendBucketRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Operation: r"""Call the add signed url key method over HTTP. Args: request (~.compute.AddSignedUrlKeyBackendBucketRequest): The request object. A request message for BackendBuckets.AddSignedUrlKey. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Operation: Represents an Operation resource. Google Compute Engine has three Operation resources: - `Global </compute/docs/reference/rest/{$api_version}/globalOperations>`__ \* `Regional </compute/docs/reference/rest/{$api_version}/regionOperations>`__ \* `Zonal </compute/docs/reference/rest/{$api_version}/zoneOperations>`__ You can use an operation resource to manage asynchronous API requests. For more information, read Handling API responses. Operations can be global, regional or zonal. - For global operations, use the ``globalOperations`` resource. - For regional operations, use the ``regionOperations`` resource. - For zonal operations, use the ``zonalOperations`` resource. For more information, read Global, Regional, and Zonal Resources. (== resource_for {$api_version}.globalOperations ==) (== resource_for {$api_version}.regionOperations ==) (== resource_for {$api_version}.zoneOperations ==) """ # Jsonify the request body body = compute.SignedUrlKey.to_json( request.signed_url_key_resource, including_default_value_fields=False, use_integers_for_enums=False, ) # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/backendBuckets/{backend_bucket}/addSignedUrlKey".format( host=self._host, project=request.project, backend_bucket=request.backend_bucket, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.AddSignedUrlKeyBackendBucketRequest.request_id in request: query_params["requestId"] = request.request_id # TODO(yon-mg): further discussion needed whether 'python truthiness' is appropriate here # discards default values # TODO(yon-mg): add test for proper url encoded strings query_params = ["{k}={v}".format(k=k, v=v) for k, v in query_params.items()] url += "?{}".format("&".join(query_params)).replace(" ", "+") # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.post(url, headers=headers, data=body,) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Operation.from_json(response.content, ignore_unknown_fields=True) def delete( self, request: compute.DeleteBackendBucketRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Operation: r"""Call the delete method over HTTP. Args: request (~.compute.DeleteBackendBucketRequest): The request object. A request message for BackendBuckets.Delete. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Operation: Represents an Operation resource. Google Compute Engine has three Operation resources: - `Global </compute/docs/reference/rest/{$api_version}/globalOperations>`__ \* `Regional </compute/docs/reference/rest/{$api_version}/regionOperations>`__ \* `Zonal </compute/docs/reference/rest/{$api_version}/zoneOperations>`__ You can use an operation resource to manage asynchronous API requests. For more information, read Handling API responses. Operations can be global, regional or zonal. - For global operations, use the ``globalOperations`` resource. - For regional operations, use the ``regionOperations`` resource. - For zonal operations, use the ``zonalOperations`` resource. For more information, read Global, Regional, and Zonal Resources. (== resource_for {$api_version}.globalOperations ==) (== resource_for {$api_version}.regionOperations ==) (== resource_for {$api_version}.zoneOperations ==) """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/backendBuckets/{backend_bucket}".format( host=self._host, project=request.project, backend_bucket=request.backend_bucket, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.DeleteBackendBucketRequest.request_id in request: query_params["requestId"] = request.request_id # TODO(yon-mg): further discussion needed whether 'python truthiness' is appropriate here # discards default values # TODO(yon-mg): add test for proper url encoded strings query_params = ["{k}={v}".format(k=k, v=v) for k, v in query_params.items()] url += "?{}".format("&".join(query_params)).replace(" ", "+") # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.delete(url, headers=headers,) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Operation.from_json(response.content, ignore_unknown_fields=True) def delete_signed_url_key( self, request: compute.DeleteSignedUrlKeyBackendBucketRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Operation: r"""Call the delete signed url key method over HTTP. Args: request (~.compute.DeleteSignedUrlKeyBackendBucketRequest): The request object. A request message for BackendBuckets.DeleteSignedUrlKey. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Operation: Represents an Operation resource. Google Compute Engine has three Operation resources: - `Global </compute/docs/reference/rest/{$api_version}/globalOperations>`__ \* `Regional </compute/docs/reference/rest/{$api_version}/regionOperations>`__ \* `Zonal </compute/docs/reference/rest/{$api_version}/zoneOperations>`__ You can use an operation resource to manage asynchronous API requests. For more information, read Handling API responses. Operations can be global, regional or zonal. - For global operations, use the ``globalOperations`` resource. - For regional operations, use the ``regionOperations`` resource. - For zonal operations, use the ``zonalOperations`` resource. For more information, read Global, Regional, and Zonal Resources. (== resource_for {$api_version}.globalOperations ==) (== resource_for {$api_version}.regionOperations ==) (== resource_for {$api_version}.zoneOperations ==) """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/backendBuckets/{backend_bucket}/deleteSignedUrlKey".format( host=self._host, project=request.project, backend_bucket=request.backend_bucket, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if request.key_name: query_params["keyName"] = request.key_name if compute.DeleteSignedUrlKeyBackendBucketRequest.request_id in request: query_params["requestId"] = request.request_id # TODO(yon-mg): further discussion needed whether 'python truthiness' is appropriate here # discards default values # TODO(yon-mg): add test for proper url encoded strings query_params = ["{k}={v}".format(k=k, v=v) for k, v in query_params.items()] url += "?{}".format("&".join(query_params)).replace(" ", "+") # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.post(url, headers=headers,) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Operation.from_json(response.content, ignore_unknown_fields=True) def get( self, request: compute.GetBackendBucketRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.BackendBucket: r"""Call the get method over HTTP. Args: request (~.compute.GetBackendBucketRequest): The request object. A request message for BackendBuckets.Get. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.BackendBucket: Represents a Cloud Storage Bucket resource. This Cloud Storage bucket resource is referenced by a URL map of a load balancer. For more information, read Backend Buckets. """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/backendBuckets/{backend_bucket}".format( host=self._host, project=request.project, backend_bucket=request.backend_bucket, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} # TODO(yon-mg): further discussion needed whether 'python truthiness' is appropriate here # discards default values # TODO(yon-mg): add test for proper url encoded strings query_params = ["{k}={v}".format(k=k, v=v) for k, v in query_params.items()] url += "?{}".format("&".join(query_params)).replace(" ", "+") # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.get(url, headers=headers,) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.BackendBucket.from_json( response.content, ignore_unknown_fields=True ) def insert( self, request: compute.InsertBackendBucketRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Operation: r"""Call the insert method over HTTP. Args: request (~.compute.InsertBackendBucketRequest): The request object. A request message for BackendBuckets.Insert. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Operation: Represents an Operation resource. Google Compute Engine has three Operation resources: - `Global </compute/docs/reference/rest/{$api_version}/globalOperations>`__ \* `Regional </compute/docs/reference/rest/{$api_version}/regionOperations>`__ \* `Zonal </compute/docs/reference/rest/{$api_version}/zoneOperations>`__ You can use an operation resource to manage asynchronous API requests. For more information, read Handling API responses. Operations can be global, regional or zonal. - For global operations, use the ``globalOperations`` resource. - For regional operations, use the ``regionOperations`` resource. - For zonal operations, use the ``zonalOperations`` resource. For more information, read Global, Regional, and Zonal Resources. (== resource_for {$api_version}.globalOperations ==) (== resource_for {$api_version}.regionOperations ==) (== resource_for {$api_version}.zoneOperations ==) """ # Jsonify the request body body = compute.BackendBucket.to_json( request.backend_bucket_resource, including_default_value_fields=False, use_integers_for_enums=False, ) # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/backendBuckets".format( host=self._host, project=request.project, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.InsertBackendBucketRequest.request_id in request: query_params["requestId"] = request.request_id # TODO(yon-mg): further discussion needed whether 'python truthiness' is appropriate here # discards default values # TODO(yon-mg): add test for proper url encoded strings query_params = ["{k}={v}".format(k=k, v=v) for k, v in query_params.items()] url += "?{}".format("&".join(query_params)).replace(" ", "+") # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.post(url, headers=headers, data=body,) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Operation.from_json(response.content, ignore_unknown_fields=True) def list( self, request: compute.ListBackendBucketsRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.BackendBucketList: r"""Call the list method over HTTP. Args: request (~.compute.ListBackendBucketsRequest): The request object. A request message for BackendBuckets.List. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.BackendBucketList: Contains a list of BackendBucket resources. """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/backendBuckets".format( host=self._host, project=request.project, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.ListBackendBucketsRequest.filter in request: query_params["filter"] = request.filter if compute.ListBackendBucketsRequest.max_results in request: query_params["maxResults"] = request.max_results if compute.ListBackendBucketsRequest.order_by in request: query_params["orderBy"] = request.order_by if compute.ListBackendBucketsRequest.page_token in request: query_params["pageToken"] = request.page_token if compute.ListBackendBucketsRequest.return_partial_success in request: query_params["returnPartialSuccess"] = request.return_partial_success # TODO(yon-mg): further discussion needed whether 'python truthiness' is appropriate here # discards default values # TODO(yon-mg): add test for proper url encoded strings query_params = ["{k}={v}".format(k=k, v=v) for k, v in query_params.items()] url += "?{}".format("&".join(query_params)).replace(" ", "+") # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.get(url, headers=headers,) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.BackendBucketList.from_json( response.content, ignore_unknown_fields=True ) def patch( self, request: compute.PatchBackendBucketRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Operation: r"""Call the patch method over HTTP. Args: request (~.compute.PatchBackendBucketRequest): The request object. A request message for BackendBuckets.Patch. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Operation: Represents an Operation resource. Google Compute Engine has three Operation resources: - `Global </compute/docs/reference/rest/{$api_version}/globalOperations>`__ \* `Regional </compute/docs/reference/rest/{$api_version}/regionOperations>`__ \* `Zonal </compute/docs/reference/rest/{$api_version}/zoneOperations>`__ You can use an operation resource to manage asynchronous API requests. For more information, read Handling API responses. Operations can be global, regional or zonal. - For global operations, use the ``globalOperations`` resource. - For regional operations, use the ``regionOperations`` resource. - For zonal operations, use the ``zonalOperations`` resource. For more information, read Global, Regional, and Zonal Resources. (== resource_for {$api_version}.globalOperations ==) (== resource_for {$api_version}.regionOperations ==) (== resource_for {$api_version}.zoneOperations ==) """ # Jsonify the request body body = compute.BackendBucket.to_json( request.backend_bucket_resource, including_default_value_fields=False, use_integers_for_enums=False, ) # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/backendBuckets/{backend_bucket}".format( host=self._host, project=request.project, backend_bucket=request.backend_bucket, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.PatchBackendBucketRequest.request_id in request: query_params["requestId"] = request.request_id # TODO(yon-mg): further discussion needed whether 'python truthiness' is appropriate here # discards default values # TODO(yon-mg): add test for proper url encoded strings query_params = ["{k}={v}".format(k=k, v=v) for k, v in query_params.items()] url += "?{}".format("&".join(query_params)).replace(" ", "+") # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.patch(url, headers=headers, data=body,) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Operation.from_json(response.content, ignore_unknown_fields=True) def update( self, request: compute.UpdateBackendBucketRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Operation: r"""Call the update method over HTTP. Args: request (~.compute.UpdateBackendBucketRequest): The request object. A request message for BackendBuckets.Update. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Operation: Represents an Operation resource. Google Compute Engine has three Operation resources: - `Global </compute/docs/reference/rest/{$api_version}/globalOperations>`__ \* `Regional </compute/docs/reference/rest/{$api_version}/regionOperations>`__ \* `Zonal </compute/docs/reference/rest/{$api_version}/zoneOperations>`__ You can use an operation resource to manage asynchronous API requests. For more information, read Handling API responses. Operations can be global, regional or zonal. - For global operations, use the ``globalOperations`` resource. - For regional operations, use the ``regionOperations`` resource. - For zonal operations, use the ``zonalOperations`` resource. For more information, read Global, Regional, and Zonal Resources. (== resource_for {$api_version}.globalOperations ==) (== resource_for {$api_version}.regionOperations ==) (== resource_for {$api_version}.zoneOperations ==) """ # Jsonify the request body body = compute.BackendBucket.to_json( request.backend_bucket_resource, including_default_value_fields=False, use_integers_for_enums=False, ) # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/backendBuckets/{backend_bucket}".format( host=self._host, project=request.project, backend_bucket=request.backend_bucket, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.UpdateBackendBucketRequest.request_id in request: query_params["requestId"] = request.request_id # TODO(yon-mg): further discussion needed whether 'python truthiness' is appropriate here # discards default values # TODO(yon-mg): add test for proper url encoded strings query_params = ["{k}={v}".format(k=k, v=v) for k, v in query_params.items()] url += "?{}".format("&".join(query_params)).replace(" ", "+") # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.put(url, headers=headers, data=body,) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Operation.from_json(response.content, ignore_unknown_fields=True)
def DownloadDiscoveryDocument(api_name, version, path=_DEFAULT_DISCOVERY_PATH, label=None): """Downloads a discovery document for the given api_name and version. This utility assumes that the API for which a discovery document is being retrieved is publicly accessible. However, you may access whitelisted resources for a public API if you are added to its whitelist and specify the associated label. Args: api_name: a str indicating the name of the API for which a discovery document is to be downloaded. version: a str indicating the version number of the API. path: a str indicating the path to which you want to save the downloaded discovery document. label: a str indicating a label to be applied to the discovery service request. This is not applicable when downloading the discovery document of a legacy API. For non-legacy APIs, this may be used as a means of programmatically retrieving a copy of a discovery document containing whitelisted content. Raises: ValueError: if labels are specified for a legacy API, which is incompatible with labels. """ credentials = _GetCredentials() auth_session = AuthorizedSession(credentials) discovery_service = build('discovery', 'v1') discovery_rest_url = None discovery_response = discovery_service.apis().list( name=api_name).execute() if 'items' in discovery_response: for api in discovery_response['items']: if api['version'] == version: discovery_rest_url = api['discoveryRestUrl'] break if discovery_rest_url: is_legacy = _IsLegacy(discovery_rest_url) else: raise ValueError(f'API with name {api_name} and version {version} was not ' 'found.') if all((is_legacy, label)): raise ValueError('The discovery URL associated with the api_name ' f'{api_name} and version {version} is for a legacy API. ' 'These are not compatible with labels.') if label: # Apply the label query parameter if it exists. path_params = '&labels=%s' % label discovery_rest_url += path_params discovery_response = auth_session.get(discovery_rest_url) if discovery_response.status_code == 200: with open(path, 'wb') as handler: handler.write(discovery_response.text) else: raise ValueError('Unable to retrieve discovery document for api name ' f'{api_name} and version {version} via discovery URL: ' f'{discovery_rest_url}')
class ImagesRestTransport(ImagesTransport): """REST backend transport for Images. The Images API. This class defines the same methods as the primary client, so the primary client can load the underlying transport implementation and call it. It sends JSON representations of protocol buffers over HTTP/1.1 """ def __init__( self, *, host: str = "compute.googleapis.com", credentials: ga_credentials.Credentials = None, credentials_file: str = None, scopes: Sequence[str] = None, client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, quota_project_id: Optional[str] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, always_use_jwt_access: Optional[bool] = False, ) -> None: """Instantiate the transport. Args: host (Optional[str]): The hostname to connect to. credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none are specified, the client will attempt to ascertain the credentials from the environment. credentials_file (Optional[str]): A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. This argument is ignored if ``channel`` is provided. scopes (Optional(Sequence[str])): A list of scopes. This argument is ignored if ``channel`` is provided. client_cert_source_for_mtls (Callable[[], Tuple[bytes, bytes]]): Client certificate to configure mutual TLS HTTP channel. It is ignored if ``channel`` is provided. quota_project_id (Optional[str]): An optional project to use for billing and quota. client_info (google.api_core.gapic_v1.client_info.ClientInfo): The client info used to send a user-agent string along with API requests. If ``None``, then default info will be used. Generally, you only need to set this if you're developing your own client library. """ # Run the base constructor # TODO(yon-mg): resolve other ctor params i.e. scopes, quota, etc. # TODO: When custom host (api_endpoint) is set, `scopes` must *also* be set on the # credentials object super().__init__( host=host, credentials=credentials, client_info=client_info, always_use_jwt_access=always_use_jwt_access, ) self._session = AuthorizedSession( self._credentials, default_host=self.DEFAULT_HOST ) if client_cert_source_for_mtls: self._session.configure_mtls_channel(client_cert_source_for_mtls) self._prep_wrapped_messages(client_info) def delete( self, request: compute.DeleteImageRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Operation: r"""Call the delete method over HTTP. Args: request (~.compute.DeleteImageRequest): The request object. A request message for Images.Delete. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Operation: Represents an Operation resource. Google Compute Engine has three Operation resources: - `Global </compute/docs/reference/rest/{$api_version}/globalOperations>`__ \* `Regional </compute/docs/reference/rest/{$api_version}/regionOperations>`__ \* `Zonal </compute/docs/reference/rest/{$api_version}/zoneOperations>`__ You can use an operation resource to manage asynchronous API requests. For more information, read Handling API responses. Operations can be global, regional or zonal. - For global operations, use the ``globalOperations`` resource. - For regional operations, use the ``regionOperations`` resource. - For zonal operations, use the ``zonalOperations`` resource. For more information, read Global, Regional, and Zonal Resources. (== resource_for {$api_version}.globalOperations ==) (== resource_for {$api_version}.regionOperations ==) (== resource_for {$api_version}.zoneOperations ==) """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/images/{image}".format( host=self._host, project=request.project, image=request.image, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.DeleteImageRequest.request_id in request: query_params["requestId"] = request.request_id # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.delete(url, headers=headers, params=query_params,) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Operation.from_json(response.content, ignore_unknown_fields=True) def deprecate( self, request: compute.DeprecateImageRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Operation: r"""Call the deprecate method over HTTP. Args: request (~.compute.DeprecateImageRequest): The request object. A request message for Images.Deprecate. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Operation: Represents an Operation resource. Google Compute Engine has three Operation resources: - `Global </compute/docs/reference/rest/{$api_version}/globalOperations>`__ \* `Regional </compute/docs/reference/rest/{$api_version}/regionOperations>`__ \* `Zonal </compute/docs/reference/rest/{$api_version}/zoneOperations>`__ You can use an operation resource to manage asynchronous API requests. For more information, read Handling API responses. Operations can be global, regional or zonal. - For global operations, use the ``globalOperations`` resource. - For regional operations, use the ``regionOperations`` resource. - For zonal operations, use the ``zonalOperations`` resource. For more information, read Global, Regional, and Zonal Resources. (== resource_for {$api_version}.globalOperations ==) (== resource_for {$api_version}.regionOperations ==) (== resource_for {$api_version}.zoneOperations ==) """ # Jsonify the request body body = compute.DeprecationStatus.to_json( request.deprecation_status_resource, including_default_value_fields=False, use_integers_for_enums=False, ) # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/images/{image}/deprecate".format( host=self._host, project=request.project, image=request.image, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.DeprecateImageRequest.request_id in request: query_params["requestId"] = request.request_id # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.post( url, headers=headers, params=query_params, data=body, ) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Operation.from_json(response.content, ignore_unknown_fields=True) def get( self, request: compute.GetImageRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Image: r"""Call the get method over HTTP. Args: request (~.compute.GetImageRequest): The request object. A request message for Images.Get. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Image: Represents an Image resource. You can use images to create boot disks for your VM instances. For more information, read Images. (== resource_for {$api_version}.images ==) """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/images/{image}".format( host=self._host, project=request.project, image=request.image, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.get(url, headers=headers, params=query_params,) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Image.from_json(response.content, ignore_unknown_fields=True) def get_from_family( self, request: compute.GetFromFamilyImageRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Image: r"""Call the get from family method over HTTP. Args: request (~.compute.GetFromFamilyImageRequest): The request object. A request message for Images.GetFromFamily. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Image: Represents an Image resource. You can use images to create boot disks for your VM instances. For more information, read Images. (== resource_for {$api_version}.images ==) """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/images/family/{family}".format( host=self._host, project=request.project, family=request.family, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.get(url, headers=headers, params=query_params,) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Image.from_json(response.content, ignore_unknown_fields=True) def get_iam_policy( self, request: compute.GetIamPolicyImageRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Policy: r"""Call the get iam policy method over HTTP. Args: request (~.compute.GetIamPolicyImageRequest): The request object. A request message for Images.GetIamPolicy. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Policy: An Identity and Access Management (IAM) policy, which specifies access controls for Google Cloud resources. A ``Policy`` is a collection of ``bindings``. A ``binding`` binds one or more ``members`` to a single ``role``. Members can be user accounts, service accounts, Google groups, and domains (such as G Suite). A ``role`` is a named list of permissions; each ``role`` can be an IAM predefined role or a user-created custom role. For some types of Google Cloud resources, a ``binding`` can also specify a ``condition``, which is a logical expression that allows access to a resource only if the expression evaluates to ``true``. A condition can add constraints based on attributes of the request, the resource, or both. To learn which resources support conditions in their IAM policies, see the `IAM documentation <https://cloud.google.com/iam/help/conditions/resource-policies>`__. **JSON example:** { "bindings": [ { "role": "roles/resourcemanager.organizationAdmin", "members": [ "user:[email protected]", "group:[email protected]", "domain:google.com", "serviceAccount:[email protected]" ] }, { "role": "roles/resourcemanager.organizationViewer", "members": [ "user:[email protected]" ], "condition": { "title": "expirable access", "description": "Does not grant access after Sep 2020", "expression": "request.time < timestamp('2020-10-01T00:00:00.000Z')", } } ], "etag": "BwWWja0YfJA=", "version": 3 } **YAML example:** bindings: - members: - user:[email protected] - group:[email protected] - domain:google.com - serviceAccount:[email protected] role: roles/resourcemanager.organizationAdmin - members: - user:[email protected] role: roles/resourcemanager.organizationViewer condition: title: expirable access description: Does not grant access after Sep 2020 expression: request.time < timestamp('2020-10-01T00:00:00.000Z') - etag: BwWWja0YfJA= - version: 3 For a description of IAM and its features, see the `IAM documentation <https://cloud.google.com/iam/docs/>`__. """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/images/{resource}/getIamPolicy".format( host=self._host, project=request.project, resource=request.resource, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.GetIamPolicyImageRequest.options_requested_policy_version in request: query_params[ "optionsRequestedPolicyVersion" ] = request.options_requested_policy_version # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.get(url, headers=headers, params=query_params,) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Policy.from_json(response.content, ignore_unknown_fields=True) def insert( self, request: compute.InsertImageRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Operation: r"""Call the insert method over HTTP. Args: request (~.compute.InsertImageRequest): The request object. A request message for Images.Insert. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Operation: Represents an Operation resource. Google Compute Engine has three Operation resources: - `Global </compute/docs/reference/rest/{$api_version}/globalOperations>`__ \* `Regional </compute/docs/reference/rest/{$api_version}/regionOperations>`__ \* `Zonal </compute/docs/reference/rest/{$api_version}/zoneOperations>`__ You can use an operation resource to manage asynchronous API requests. For more information, read Handling API responses. Operations can be global, regional or zonal. - For global operations, use the ``globalOperations`` resource. - For regional operations, use the ``regionOperations`` resource. - For zonal operations, use the ``zonalOperations`` resource. For more information, read Global, Regional, and Zonal Resources. (== resource_for {$api_version}.globalOperations ==) (== resource_for {$api_version}.regionOperations ==) (== resource_for {$api_version}.zoneOperations ==) """ # Jsonify the request body body = compute.Image.to_json( request.image_resource, including_default_value_fields=False, use_integers_for_enums=False, ) # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/images".format( host=self._host, project=request.project, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.InsertImageRequest.force_create in request: query_params["forceCreate"] = request.force_create if compute.InsertImageRequest.request_id in request: query_params["requestId"] = request.request_id # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.post( url, headers=headers, params=query_params, data=body, ) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Operation.from_json(response.content, ignore_unknown_fields=True) def list( self, request: compute.ListImagesRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.ImageList: r"""Call the list method over HTTP. Args: request (~.compute.ListImagesRequest): The request object. A request message for Images.List. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.ImageList: Contains a list of images. """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/images".format( host=self._host, project=request.project, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.ListImagesRequest.filter in request: query_params["filter"] = request.filter if compute.ListImagesRequest.max_results in request: query_params["maxResults"] = request.max_results if compute.ListImagesRequest.order_by in request: query_params["orderBy"] = request.order_by if compute.ListImagesRequest.page_token in request: query_params["pageToken"] = request.page_token if compute.ListImagesRequest.return_partial_success in request: query_params["returnPartialSuccess"] = request.return_partial_success # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.get(url, headers=headers, params=query_params,) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.ImageList.from_json(response.content, ignore_unknown_fields=True) def patch( self, request: compute.PatchImageRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Operation: r"""Call the patch method over HTTP. Args: request (~.compute.PatchImageRequest): The request object. A request message for Images.Patch. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Operation: Represents an Operation resource. Google Compute Engine has three Operation resources: - `Global </compute/docs/reference/rest/{$api_version}/globalOperations>`__ \* `Regional </compute/docs/reference/rest/{$api_version}/regionOperations>`__ \* `Zonal </compute/docs/reference/rest/{$api_version}/zoneOperations>`__ You can use an operation resource to manage asynchronous API requests. For more information, read Handling API responses. Operations can be global, regional or zonal. - For global operations, use the ``globalOperations`` resource. - For regional operations, use the ``regionOperations`` resource. - For zonal operations, use the ``zonalOperations`` resource. For more information, read Global, Regional, and Zonal Resources. (== resource_for {$api_version}.globalOperations ==) (== resource_for {$api_version}.regionOperations ==) (== resource_for {$api_version}.zoneOperations ==) """ # Jsonify the request body body = compute.Image.to_json( request.image_resource, including_default_value_fields=False, use_integers_for_enums=False, ) # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/images/{image}".format( host=self._host, project=request.project, image=request.image, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.PatchImageRequest.request_id in request: query_params["requestId"] = request.request_id # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.patch( url, headers=headers, params=query_params, data=body, ) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Operation.from_json(response.content, ignore_unknown_fields=True) def set_iam_policy( self, request: compute.SetIamPolicyImageRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Policy: r"""Call the set iam policy method over HTTP. Args: request (~.compute.SetIamPolicyImageRequest): The request object. A request message for Images.SetIamPolicy. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Policy: An Identity and Access Management (IAM) policy, which specifies access controls for Google Cloud resources. A ``Policy`` is a collection of ``bindings``. A ``binding`` binds one or more ``members`` to a single ``role``. Members can be user accounts, service accounts, Google groups, and domains (such as G Suite). A ``role`` is a named list of permissions; each ``role`` can be an IAM predefined role or a user-created custom role. For some types of Google Cloud resources, a ``binding`` can also specify a ``condition``, which is a logical expression that allows access to a resource only if the expression evaluates to ``true``. A condition can add constraints based on attributes of the request, the resource, or both. To learn which resources support conditions in their IAM policies, see the `IAM documentation <https://cloud.google.com/iam/help/conditions/resource-policies>`__. **JSON example:** { "bindings": [ { "role": "roles/resourcemanager.organizationAdmin", "members": [ "user:[email protected]", "group:[email protected]", "domain:google.com", "serviceAccount:[email protected]" ] }, { "role": "roles/resourcemanager.organizationViewer", "members": [ "user:[email protected]" ], "condition": { "title": "expirable access", "description": "Does not grant access after Sep 2020", "expression": "request.time < timestamp('2020-10-01T00:00:00.000Z')", } } ], "etag": "BwWWja0YfJA=", "version": 3 } **YAML example:** bindings: - members: - user:[email protected] - group:[email protected] - domain:google.com - serviceAccount:[email protected] role: roles/resourcemanager.organizationAdmin - members: - user:[email protected] role: roles/resourcemanager.organizationViewer condition: title: expirable access description: Does not grant access after Sep 2020 expression: request.time < timestamp('2020-10-01T00:00:00.000Z') - etag: BwWWja0YfJA= - version: 3 For a description of IAM and its features, see the `IAM documentation <https://cloud.google.com/iam/docs/>`__. """ # Jsonify the request body body = compute.GlobalSetPolicyRequest.to_json( request.global_set_policy_request_resource, including_default_value_fields=False, use_integers_for_enums=False, ) # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/images/{resource}/setIamPolicy".format( host=self._host, project=request.project, resource=request.resource, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.post( url, headers=headers, params=query_params, data=body, ) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Policy.from_json(response.content, ignore_unknown_fields=True) def set_labels( self, request: compute.SetLabelsImageRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Operation: r"""Call the set labels method over HTTP. Args: request (~.compute.SetLabelsImageRequest): The request object. A request message for Images.SetLabels. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Operation: Represents an Operation resource. Google Compute Engine has three Operation resources: - `Global </compute/docs/reference/rest/{$api_version}/globalOperations>`__ \* `Regional </compute/docs/reference/rest/{$api_version}/regionOperations>`__ \* `Zonal </compute/docs/reference/rest/{$api_version}/zoneOperations>`__ You can use an operation resource to manage asynchronous API requests. For more information, read Handling API responses. Operations can be global, regional or zonal. - For global operations, use the ``globalOperations`` resource. - For regional operations, use the ``regionOperations`` resource. - For zonal operations, use the ``zonalOperations`` resource. For more information, read Global, Regional, and Zonal Resources. (== resource_for {$api_version}.globalOperations ==) (== resource_for {$api_version}.regionOperations ==) (== resource_for {$api_version}.zoneOperations ==) """ # Jsonify the request body body = compute.GlobalSetLabelsRequest.to_json( request.global_set_labels_request_resource, including_default_value_fields=False, use_integers_for_enums=False, ) # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/images/{resource}/setLabels".format( host=self._host, project=request.project, resource=request.resource, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.post( url, headers=headers, params=query_params, data=body, ) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Operation.from_json(response.content, ignore_unknown_fields=True) def test_iam_permissions( self, request: compute.TestIamPermissionsImageRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.TestPermissionsResponse: r"""Call the test iam permissions method over HTTP. Args: request (~.compute.TestIamPermissionsImageRequest): The request object. A request message for Images.TestIamPermissions. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.TestPermissionsResponse: """ # Jsonify the request body body = compute.TestPermissionsRequest.to_json( request.test_permissions_request_resource, including_default_value_fields=False, use_integers_for_enums=False, ) # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/images/{resource}/testIamPermissions".format( host=self._host, project=request.project, resource=request.resource, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.post( url, headers=headers, params=query_params, data=body, ) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.TestPermissionsResponse.from_json( response.content, ignore_unknown_fields=True )
def request(method, url, target_audience=None, service_account_file=None, data=None, headers=None, **kwargs): """Obtains a Google-issued ID token and uses it to make a HTTP request. Args: method (str): The HTTP request method to use ('GET', 'OPTIONS', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE') url: The URL where the HTTP request will be sent. target_audience (str): Optional, the value to use in the audience ('aud') claim of the ID token. Defaults to the value of 'url' if not provided. service_account_file (str): Optional, the full path to the service account credentials JSON file. Defaults to '<working directory>/service-account.json'. data: Optional dictionary, list of tuples, bytes, or file-like object to send in the body of the request. headers (dict): Optional dictionary of HTTP headers to send with the request. **kwargs: Any of the parameters defined for the request function: https://github.com/requests/requests/blob/master/requests/api.py If no timeout is provided, it is set to 90 seconds. Returns: The page body, or raises an exception if the HTTP request failed. """ # Set target_audience, if missing if not target_audience: target_audience = url # Set service_account_file, if missing if not service_account_file: service_account_file = os.path.join(os.getcwd(), 'service-account.json') # Set the default timeout, if missing if 'timeout' not in kwargs: kwargs['timeout'] = 90 # seconds # Obtain ID token credentials for the specified audience creds = service_account.IDTokenCredentials.from_service_account_file( service_account_file, target_audience=target_audience) # Create a session for sending requests with the ID token credentials session = AuthorizedSession(creds) # Send a HTTP request to the provided URL using the Google-issued ID token resp = session.request(method, url, data=data, headers=headers, **kwargs) if resp.status_code == 403: raise Exception('Service account {} does not have permission to ' 'access the application.'.format( creds.service_account_email)) elif resp.status_code != 200: raise Exception( 'Bad response from application: {!r} / {!r} / {!r}'.format( resp.status_code, resp.headers, resp.text)) else: return resp.text
def __init__(self, credentials): self._service = build('drive', 'v3', credentials=credentials) self._authed_session = AuthorizedSession(credentials)
class Google(object): """docstring for Google""" def __init__(self, scopes, service_account_file = None, subject = '*****@*****.**'): super(Google, self).__init__() self.service_account_file = service_account_file self.scopes = scopes self.subject = subject """http://google-auth.readthedocs.io/en/latest/reference/google.oauth2.service_account.html""" if service_account_file != None: delegated_credentials = service_account.Credentials.from_service_account_file(service_account_file, scopes = scopes, subject = subject) self.session = AuthorizedSession(delegated_credentials) basic_credentials = service_account.Credentials.from_service_account_file(service_account_file, scopes = scopes) self.basic_session = AuthorizedSession(basic_credentials) def get_users(self): ouput = {} ouput['users'] = [] initial_request = 'https://www.googleapis.com/admin/directory/v1/users?domain=punch.vn&projection=basic&orderBy=email' request = 'https://www.googleapis.com/admin/directory/v1/users?domain=punch.vn&projection=basic&orderBy=email' has_next_page = True while has_next_page : try: response = self.session.request('GET', request) except RefreshError as e: ouput['return_code'] = 500 ouput['message'] = str(e) return ouput result = vars(response) users = json.loads(result['_content']) if 'error' in users: ouput['return_code'] = users['error']['code'] ouput['message'] = users['error']['message'] return ouput else: ouput['return_code'] = 200 ouput['users'] = ouput['users'] + users['users'] if not 'nextPageToken' in users: has_next_page = False else: # print("Next Page Token: " + users['nextPageToken']) request = initial_request + "&pageToken=" + users['nextPageToken'] return ouput def get_tokens(self, email): ouput = {} try: response = self.session.request('GET', 'https://www.googleapis.com/admin/directory/v1/users/' + email + '/tokens') except RefreshError as e: ouput['return_code'] = 500 ouput['message'] = str(e) return ouput result = vars(response) tokens = json.loads(result['_content']) if 'error' in tokens: ouput['return_code'] = tokens['error']['code'] ouput['message'] = tokens['error']['message'] return ouput else: ouput['return_code'] = 200 ouput['tokens'] = tokens['items'] return ouput def readSheet(self, sheetId, sheetRange): ouput = {} try: response = self.basic_session.request('GET', 'https://sheets.googleapis.com/v4/spreadsheets/'+ sheetId +'/values/'+ sheetRange) except RefreshError as e: ouput['return_code'] = 500 ouput['message'] = str(e) return ouput result = vars(response) data = json.loads(result['_content']) if 'error' in data: ouput['return_code'] = data['error']['code'] ouput['message'] = data['error']['message'] return ouput else: ouput['return_code'] = 200 ouput['values'] = data['values'] return ouput def get_devices(self): ouput = {} ouput['mobiledevices'] = [] initial_request = 'https://www.googleapis.com/admin/directory/v1/customer/C02bgpnb8/devices/mobile?orderBy=email&projection=BASIC' request = 'https://www.googleapis.com/admin/directory/v1/customer/C02bgpnb8/devices/mobile?orderBy=email&projection=BASIC' has_next_page = True while has_next_page : try: response = self.session.request('GET', request) except RefreshError as e: ouput['return_code'] = 500 ouput['message'] = str(e) return ouput result = vars(response) data = json.loads(result['_content']) if 'error' in data: ouput['return_code'] = data['error']['code'] ouput['message'] = data['error']['message'] return ouput else: ouput['return_code'] = 200 ouput['mobiledevices'] = ouput['mobiledevices'] + data['mobiledevices'] if not 'nextPageToken' in data: has_next_page = False else: # print("Next Page Token: " + data['nextPageToken']) request = initial_request + "&pageToken=" + data['nextPageToken'] return ouput
class GoogleDriveV2Accessor(object): """ A list of APIs to access Google Drive. Refer to https://developers.google.com/drive/api/v3/reference/drives to get information on how this class is set up. """ logger = logging.getLogger('intent_parser_google_drive_accessor') def __init__(self, credentials): self._service = build('drive', 'v2', credentials=credentials) self._authed_session = AuthorizedSession(credentials) def get_document_metadata(self, document_id): return self._service.files().get(fileId=document_id).execute() def get_document_parents(self, document_id): return self._service.parents().list(fileId=document_id).execute() def get_document_revisions(self, document_id): """ Returns the list of revisions for the given document_id """ return self._service.revisions().list(fileId=document_id).execute() def get_head_revision(self, document_id): revisions = self.get_document_revisions(document_id=document_id) revision_ids = [int(revision['id']) for revision in revisions['items']] if len(revision_ids) < 1: raise ValueError('Revision not found.') return str(max(revision_ids)) def get_file_with_revision(self, file_id, revision_id, mime_type): """Download a Google Doc base on a Doc's id and its revision. Args: fild_id: Google Doc ID revision_id: Google Doc revision ID mime_type: format to download the Google Doc in. Visit https://developers.google.com/drive/api/v3/ref-export-formats to get a list of file formats that Google can export to Returns: An HTML response of the requested Google Doc revision. """ revisions = self.get_document_revisions(document_id=file_id) filter_by_revision = [ revision for revision in revisions['items'] if revision['id'] == revision_id ] if len(filter_by_revision) < 1: raise ValueError('Revision not found.') url = filter_by_revision[0]['exportLinks'][mime_type] response = self._authed_session.request('GET', url) return response def insert_comment_box(self, file_id, comment_message, quoted_text=None): """Insert a comment box to the desired document. Args: file_id: ID of a document. comment_message: message to display in comment box quoted_text: text in document that comment box refers to. """ new_comment = {'content': comment_message} if quoted_text: new_comment['context'] = {'value': quoted_text} try: response = self._service.comments().insert( fileId=file_id, body=new_comment).execute() except errors.HttpError as error: self.logger.warning( 'Unable to insert comment box to file: %s due to %s' % (file_id, error)) def retrieve_comments(self, file_id): """Retrieve a list of comments. Args: service: Drive API service instance. file_id: ID of the file to retrieve comments for. Returns: List of comments. """ try: comments = self._service.comments().list(fileId=file_id).execute() return comments.get('items', []) except errors.HttpError as error: self.logger.warning( 'Unable to retrieve comments from file: %s due to %s' % (file_id, error)) def remove_comment(self, file_id, comment_id): """Remove a comment. Args: service: Drive API service instance. file_id: ID of the file to remove the comment for. comment_id: ID of the comment to remove. """ try: self._service.comments().delete(fileId=file_id, commentId=comment_id).execute() except errors.HttpError as error: self.logger.warning( 'Unable to remove comment box from file: %s due to %s' % (file_id, error)) def upload_revision(self, document_name, document, folder_id, original_format, title='Untitled', target_format='*/*'): """Upload file to a Google Drive folder. Args: document_name: Name of the document document: content of the document to upload. folder_id: id of the Google Drive folder original_format: file format of the document content title: document title target_format: file format that that the uploaded file will transform into. Returns: A string to represent the uploaded file's id. """ file_metadata = { 'name': document_name, 'title': title, 'parents': [folder_id], 'mimeType': target_format } fh = BytesIO(document) media = MediaIoBaseUpload(fh, mimetype=original_format, resumable=True) file = self._service.files().insert(body=file_metadata, media_body=media, convert=True, fields='id').execute() print('File ID: ' + file.get('id')) return file.get('id') def delete_file(self, file_id: str): """Delete an existing file Args: file_id - the file to delete Returns: A boolean value. True if the file has been deleted successfully and False, otherwise. """ response = self._service.files().delete(fileId=file_id).execute() return not response def move_file_to_folder(self, folder_id, file_id): return self._service.files().update(fileId=file_id, addParents=folder_id, removeParents='root').execute()
class RegionNetworkEndpointGroupsRestTransport(RegionNetworkEndpointGroupsTransport): """REST backend transport for RegionNetworkEndpointGroups. The RegionNetworkEndpointGroups API. This class defines the same methods as the primary client, so the primary client can load the underlying transport implementation and call it. It sends JSON representations of protocol buffers over HTTP/1.1 """ def __init__( self, *, host: str = "compute.googleapis.com", credentials: ga_credentials.Credentials = None, credentials_file: str = None, scopes: Sequence[str] = None, client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, quota_project_id: Optional[str] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, ) -> None: """Instantiate the transport. Args: host (Optional[str]): The hostname to connect to. credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none are specified, the client will attempt to ascertain the credentials from the environment. credentials_file (Optional[str]): A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. This argument is ignored if ``channel`` is provided. scopes (Optional(Sequence[str])): A list of scopes. This argument is ignored if ``channel`` is provided. client_cert_source_for_mtls (Callable[[], Tuple[bytes, bytes]]): Client certificate to configure mutual TLS HTTP channel. It is ignored if ``channel`` is provided. quota_project_id (Optional[str]): An optional project to use for billing and quota. client_info (google.api_core.gapic_v1.client_info.ClientInfo): The client info used to send a user-agent string along with API requests. If ``None``, then default info will be used. Generally, you only need to set this if you're developing your own client library. """ # Run the base constructor # TODO(yon-mg): resolve other ctor params i.e. scopes, quota, etc. # TODO: When custom host (api_endpoint) is set, `scopes` must *also* be set on the # credentials object super().__init__( host=host, credentials=credentials, client_info=client_info, ) self._session = AuthorizedSession( self._credentials, default_host=self.DEFAULT_HOST ) if client_cert_source_for_mtls: self._session.configure_mtls_channel(client_cert_source_for_mtls) self._prep_wrapped_messages(client_info) def delete( self, request: compute.DeleteRegionNetworkEndpointGroupRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Operation: r"""Call the delete method over HTTP. Args: request (~.compute.DeleteRegionNetworkEndpointGroupRequest): The request object. A request message for RegionNetworkEndpointGroups.Delete. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Operation: Represents an Operation resource. Google Compute Engine has three Operation resources: - `Global </compute/docs/reference/rest/{$api_version}/globalOperations>`__ \* `Regional </compute/docs/reference/rest/{$api_version}/regionOperations>`__ \* `Zonal </compute/docs/reference/rest/{$api_version}/zoneOperations>`__ You can use an operation resource to manage asynchronous API requests. For more information, read Handling API responses. Operations can be global, regional or zonal. - For global operations, use the ``globalOperations`` resource. - For regional operations, use the ``regionOperations`` resource. - For zonal operations, use the ``zonalOperations`` resource. For more information, read Global, Regional, and Zonal Resources. (== resource_for {$api_version}.globalOperations ==) (== resource_for {$api_version}.regionOperations ==) (== resource_for {$api_version}.zoneOperations ==) """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/regions/{region}/networkEndpointGroups/{network_endpoint_group}".format( host=self._host, project=request.project, region=request.region, network_endpoint_group=request.network_endpoint_group, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.DeleteRegionNetworkEndpointGroupRequest.request_id in request: query_params["requestId"] = request.request_id # TODO(yon-mg): further discussion needed whether 'python truthiness' is appropriate here # discards default values # TODO(yon-mg): add test for proper url encoded strings query_params = ["{k}={v}".format(k=k, v=v) for k, v in query_params.items()] url += "?{}".format("&".join(query_params)).replace(" ", "+") # Send the request response = self._session.delete(url,) # Raise requests.exceptions.HTTPError if the status code is >= 400 response.raise_for_status() # Return the response return compute.Operation.from_json(response.content, ignore_unknown_fields=True) def get( self, request: compute.GetRegionNetworkEndpointGroupRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.NetworkEndpointGroup: r"""Call the get method over HTTP. Args: request (~.compute.GetRegionNetworkEndpointGroupRequest): The request object. A request message for RegionNetworkEndpointGroups.Get. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.NetworkEndpointGroup: Represents a collection of network endpoints. A network endpoint group (NEG) defines how a set of endpoints should be reached, whether they are reachable, and where they are located. For more information about using NEGs, see Setting up external HTTP(S) Load Balancing with internet NEGs, Setting up zonal NEGs, or Setting up external HTTP(S) Load Balancing with serverless NEGs. (== resource_for {$api_version}.networkEndpointGroups ==) (== resource_for {$api_version}.globalNetworkEndpointGroups ==) (== resource_for {$api_version}.regionNetworkEndpointGroups ==) """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/regions/{region}/networkEndpointGroups/{network_endpoint_group}".format( host=self._host, project=request.project, region=request.region, network_endpoint_group=request.network_endpoint_group, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} # TODO(yon-mg): further discussion needed whether 'python truthiness' is appropriate here # discards default values # TODO(yon-mg): add test for proper url encoded strings query_params = ["{k}={v}".format(k=k, v=v) for k, v in query_params.items()] url += "?{}".format("&".join(query_params)).replace(" ", "+") # Send the request response = self._session.get(url,) # Raise requests.exceptions.HTTPError if the status code is >= 400 response.raise_for_status() # Return the response return compute.NetworkEndpointGroup.from_json( response.content, ignore_unknown_fields=True ) def insert( self, request: compute.InsertRegionNetworkEndpointGroupRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Operation: r"""Call the insert method over HTTP. Args: request (~.compute.InsertRegionNetworkEndpointGroupRequest): The request object. A request message for RegionNetworkEndpointGroups.Insert. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Operation: Represents an Operation resource. Google Compute Engine has three Operation resources: - `Global </compute/docs/reference/rest/{$api_version}/globalOperations>`__ \* `Regional </compute/docs/reference/rest/{$api_version}/regionOperations>`__ \* `Zonal </compute/docs/reference/rest/{$api_version}/zoneOperations>`__ You can use an operation resource to manage asynchronous API requests. For more information, read Handling API responses. Operations can be global, regional or zonal. - For global operations, use the ``globalOperations`` resource. - For regional operations, use the ``regionOperations`` resource. - For zonal operations, use the ``zonalOperations`` resource. For more information, read Global, Regional, and Zonal Resources. (== resource_for {$api_version}.globalOperations ==) (== resource_for {$api_version}.regionOperations ==) (== resource_for {$api_version}.zoneOperations ==) """ # Jsonify the request body body = compute.NetworkEndpointGroup.to_json( request.network_endpoint_group_resource, including_default_value_fields=False, use_integers_for_enums=False, ) # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/regions/{region}/networkEndpointGroups".format( host=self._host, project=request.project, region=request.region, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.InsertRegionNetworkEndpointGroupRequest.request_id in request: query_params["requestId"] = request.request_id # TODO(yon-mg): further discussion needed whether 'python truthiness' is appropriate here # discards default values # TODO(yon-mg): add test for proper url encoded strings query_params = ["{k}={v}".format(k=k, v=v) for k, v in query_params.items()] url += "?{}".format("&".join(query_params)).replace(" ", "+") # Send the request response = self._session.post(url, data=body,) # Raise requests.exceptions.HTTPError if the status code is >= 400 response.raise_for_status() # Return the response return compute.Operation.from_json(response.content, ignore_unknown_fields=True) def list( self, request: compute.ListRegionNetworkEndpointGroupsRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.NetworkEndpointGroupList: r"""Call the list method over HTTP. Args: request (~.compute.ListRegionNetworkEndpointGroupsRequest): The request object. A request message for RegionNetworkEndpointGroups.List. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.NetworkEndpointGroupList: """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/regions/{region}/networkEndpointGroups".format( host=self._host, project=request.project, region=request.region, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.ListRegionNetworkEndpointGroupsRequest.filter in request: query_params["filter"] = request.filter if compute.ListRegionNetworkEndpointGroupsRequest.max_results in request: query_params["maxResults"] = request.max_results if compute.ListRegionNetworkEndpointGroupsRequest.order_by in request: query_params["orderBy"] = request.order_by if compute.ListRegionNetworkEndpointGroupsRequest.page_token in request: query_params["pageToken"] = request.page_token if ( compute.ListRegionNetworkEndpointGroupsRequest.return_partial_success in request ): query_params["returnPartialSuccess"] = request.return_partial_success # TODO(yon-mg): further discussion needed whether 'python truthiness' is appropriate here # discards default values # TODO(yon-mg): add test for proper url encoded strings query_params = ["{k}={v}".format(k=k, v=v) for k, v in query_params.items()] url += "?{}".format("&".join(query_params)).replace(" ", "+") # Send the request response = self._session.get(url,) # Raise requests.exceptions.HTTPError if the status code is >= 400 response.raise_for_status() # Return the response return compute.NetworkEndpointGroupList.from_json( response.content, ignore_unknown_fields=True )
# format the json we're sending it data = {} details = {} details['weight'] = arguments['<weight>'] details['message'] = arguments['<message>'] details['bmi'] = bmi details['bmr'] = bmr details['tdee'] = tdee data[key] = details; json_data = json.dumps(data); # Define the required scopes scopes = [ "https://www.googleapis.com/auth/userinfo.email", "https://www.googleapis.com/auth/firebase.database" ] # Authenticate a credential with the service account credentials = service_account.Credentials.from_service_account_file( "./secrets/metrac-io-firebase-adminsdk-3o30f-0ea8c2fa69.json", scopes=scopes) # Use the credentials object to authenticate a Requests session. authed_session = AuthorizedSession(credentials) response = authed_session.put(url+ext, json_data) print response response = authed_session.get(url+ext) print response.content
class GlobalAddressesRestTransport(GlobalAddressesTransport): """REST backend transport for GlobalAddresses. The GlobalAddresses API. This class defines the same methods as the primary client, so the primary client can load the underlying transport implementation and call it. It sends JSON representations of protocol buffers over HTTP/1.1 """ def __init__( self, *, host: str = "compute.googleapis.com", credentials: ga_credentials.Credentials = None, credentials_file: str = None, scopes: Sequence[str] = None, client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, quota_project_id: Optional[str] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, always_use_jwt_access: Optional[bool] = False, ) -> None: """Instantiate the transport. Args: host (Optional[str]): The hostname to connect to. credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none are specified, the client will attempt to ascertain the credentials from the environment. credentials_file (Optional[str]): A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. This argument is ignored if ``channel`` is provided. scopes (Optional(Sequence[str])): A list of scopes. This argument is ignored if ``channel`` is provided. client_cert_source_for_mtls (Callable[[], Tuple[bytes, bytes]]): Client certificate to configure mutual TLS HTTP channel. It is ignored if ``channel`` is provided. quota_project_id (Optional[str]): An optional project to use for billing and quota. client_info (google.api_core.gapic_v1.client_info.ClientInfo): The client info used to send a user-agent string along with API requests. If ``None``, then default info will be used. Generally, you only need to set this if you're developing your own client library. """ # Run the base constructor # TODO(yon-mg): resolve other ctor params i.e. scopes, quota, etc. # TODO: When custom host (api_endpoint) is set, `scopes` must *also* be set on the # credentials object super().__init__( host=host, credentials=credentials, client_info=client_info, always_use_jwt_access=always_use_jwt_access, ) self._session = AuthorizedSession(self._credentials, default_host=self.DEFAULT_HOST) if client_cert_source_for_mtls: self._session.configure_mtls_channel(client_cert_source_for_mtls) self._prep_wrapped_messages(client_info) def delete( self, request: compute.DeleteGlobalAddressRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Operation: r"""Call the delete method over HTTP. Args: request (~.compute.DeleteGlobalAddressRequest): The request object. A request message for GlobalAddresses.Delete. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Operation: Represents an Operation resource. Google Compute Engine has three Operation resources: - `Global </compute/docs/reference/rest/{$api_version}/globalOperations>`__ \* `Regional </compute/docs/reference/rest/{$api_version}/regionOperations>`__ \* `Zonal </compute/docs/reference/rest/{$api_version}/zoneOperations>`__ You can use an operation resource to manage asynchronous API requests. For more information, read Handling API responses. Operations can be global, regional or zonal. - For global operations, use the ``globalOperations`` resource. - For regional operations, use the ``regionOperations`` resource. - For zonal operations, use the ``zonalOperations`` resource. For more information, read Global, Regional, and Zonal Resources. (== resource_for {$api_version}.globalOperations ==) (== resource_for {$api_version}.regionOperations ==) (== resource_for {$api_version}.zoneOperations ==) """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/addresses/{address}".format( host=self._host, project=request.project, address=request.address, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.DeleteGlobalAddressRequest.request_id in request: query_params["requestId"] = request.request_id # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.delete( url, headers=headers, params=query_params, ) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Operation.from_json(response.content, ignore_unknown_fields=True) def get( self, request: compute.GetGlobalAddressRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Address: r"""Call the get method over HTTP. Args: request (~.compute.GetGlobalAddressRequest): The request object. A request message for GlobalAddresses.Get. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Address: Use global external addresses for GFE-based external HTTP(S) load balancers in Premium Tier. Use global internal addresses for reserved peering network range. Use regional external addresses for the following resources: - External IP addresses for VM instances - Regional external forwarding rules - Cloud NAT external IP addresses - GFE based LBs in Standard Tier - Network LBs in Premium or Standard Tier - Cloud VPN gateways (both Classic and HA) Use regional internal IP addresses for subnet IP ranges (primary and secondary). This includes: - Internal IP addresses for VM instances - Alias IP ranges of VM instances (/32 only) - Regional internal forwarding rules - Internal TCP/UDP load balancer addresses - Internal HTTP(S) load balancer addresses - Cloud DNS inbound forwarding IP addresses For more information, read reserved IP address. (== resource_for {$api_version}.addresses ==) (== resource_for {$api_version}.globalAddresses ==) """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/addresses/{address}".format( host=self._host, project=request.project, address=request.address, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.get( url, headers=headers, params=query_params, ) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Address.from_json(response.content, ignore_unknown_fields=True) def insert( self, request: compute.InsertGlobalAddressRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.Operation: r"""Call the insert method over HTTP. Args: request (~.compute.InsertGlobalAddressRequest): The request object. A request message for GlobalAddresses.Insert. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.Operation: Represents an Operation resource. Google Compute Engine has three Operation resources: - `Global </compute/docs/reference/rest/{$api_version}/globalOperations>`__ \* `Regional </compute/docs/reference/rest/{$api_version}/regionOperations>`__ \* `Zonal </compute/docs/reference/rest/{$api_version}/zoneOperations>`__ You can use an operation resource to manage asynchronous API requests. For more information, read Handling API responses. Operations can be global, regional or zonal. - For global operations, use the ``globalOperations`` resource. - For regional operations, use the ``regionOperations`` resource. - For zonal operations, use the ``zonalOperations`` resource. For more information, read Global, Regional, and Zonal Resources. (== resource_for {$api_version}.globalOperations ==) (== resource_for {$api_version}.regionOperations ==) (== resource_for {$api_version}.zoneOperations ==) """ # Jsonify the request body body = compute.Address.to_json( request.address_resource, including_default_value_fields=False, use_integers_for_enums=False, ) # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/addresses".format( host=self._host, project=request.project, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.InsertGlobalAddressRequest.request_id in request: query_params["requestId"] = request.request_id # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.post( url, headers=headers, params=query_params, data=body, ) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.Operation.from_json(response.content, ignore_unknown_fields=True) def list( self, request: compute.ListGlobalAddressesRequest, *, metadata: Sequence[Tuple[str, str]] = (), ) -> compute.AddressList: r"""Call the list method over HTTP. Args: request (~.compute.ListGlobalAddressesRequest): The request object. A request message for GlobalAddresses.List. See the method description for details. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. Returns: ~.compute.AddressList: Contains a list of addresses. """ # TODO(yon-mg): need to handle grpc transcoding and parse url correctly # current impl assumes basic case of grpc transcoding url = "https://{host}/compute/v1/projects/{project}/global/addresses".format( host=self._host, project=request.project, ) # TODO(yon-mg): handle nested fields corerctly rather than using only top level fields # not required for GCE query_params = {} if compute.ListGlobalAddressesRequest.filter in request: query_params["filter"] = request.filter if compute.ListGlobalAddressesRequest.max_results in request: query_params["maxResults"] = request.max_results if compute.ListGlobalAddressesRequest.order_by in request: query_params["orderBy"] = request.order_by if compute.ListGlobalAddressesRequest.page_token in request: query_params["pageToken"] = request.page_token if compute.ListGlobalAddressesRequest.return_partial_success in request: query_params[ "returnPartialSuccess"] = request.return_partial_success # Send the request headers = dict(metadata) headers["Content-Type"] = "application/json" response = self._session.get( url, headers=headers, params=query_params, ) # In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception # subclass. if response.status_code >= 400: raise core_exceptions.from_http_response(response) # Return the response return compute.AddressList.from_json(response.content, ignore_unknown_fields=True)
class SheriffConfigClient(object): """Wrapping of sheriff-config HTTP API.""" def __init__(self): """Make the Cloud Endpoints request from this handler.""" credentials, _ = google.auth.default( scopes=['https://www.googleapis.com/auth/userinfo.email']) jwt_credentials = jwt.Credentials.from_signing_credentials( credentials, 'sheriff-config-dot-chromeperf.appspot.com') self._session = AuthorizedSession(jwt_credentials) @staticmethod def _ParseSubscription(revision, subscription): return Subscription( revision=revision, name=subscription.name, rotation_url=subscription.rotation_url, notification_email=subscription.notification_email, bug_labels=list(subscription.bug_labels), bug_components=list(subscription.bug_components), bug_cc_emails=list(subscription.bug_cc_emails), visibility=subscription.visibility, auto_triage_enable=subscription.auto_triage.enable, ) def Match(self, path, check=False): response = self._session.post( 'https://sheriff-config-dot-chromeperf.appspot.com/subscriptions/match', json={'path': path}) if response.status_code == 404: # If no subscription matched return [], None if not response.ok: err_msg = '%r\n%s' % (response, response.text) if check: raise InternalServerError(err_msg) return None, err_msg match_resp = json_format.Parse(response.text, sheriff_config_pb2.MatchResponse()) return [ self._ParseSubscription(s.revision, s.subscription) for s in match_resp.subscriptions ], None def List(self, check=False): response = self._session.post( 'https://sheriff-config-dot-chromeperf.appspot.com/subscriptions/list', json={'identity_email': GetEmail()}) if not response.ok: err_msg = '%r\n%s' % (response, response.text) if check: raise InternalServerError(err_msg) return None, err_msg list_resp = json_format.Parse(response.text, sheriff_config_pb2.ListResponse()) return [ self._ParseSubscription(s.revision, s.subscription) for s in list_resp.subscriptions ], None def Update(self, check=False): response = self._session.get( 'https://sheriff-config-dot-chromeperf.appspot.com/configs/update') if response.ok: return True, None err_msg = '%r\n%s' % (response, response.text) if check: raise InternalServerError(err_msg) return False, err_msg