def DiagnoseNetworkConnection(urls=None): """Diagnose and fix local network connection issues. Checks reachability of passed in urls. If any are unreachable, asks user to change or update Cloud SDK proxy properties. If user makes proxy settings changes, then reachability check is run one more time. Args: urls: iterable(str), The list of urls to check connection to. Defaults to DEFAULT_URLS (above). Returns: bool: Whether the network connection check ultimately passed. """ if not urls: urls = _DefaultUrls() if _CheckNetworkConnection(urls, http.Http(auth=False), first_run=True): return True log.status.Print( 'Network connection problems may be due to proxy or firewall settings.' ) while True: if not _ChangeGcloudProxySettings(): return False if _CheckNetworkConnection(urls, http.Http(auth=False), first_run=False): return True
def testServiceAccountImpersonation(self): http.Http().request('http://foo.com', 'GET', None, {}) access_token = self.request_mock.call_args[1]['headers'][ b'authorization'] self.assertEqual(access_token, b'Bearer ' + self.impersonation_token.encode('utf-8'))
def testServiceAccountImpersonation(self): # This is the logged in credential, but it will not be used to make the call # because of the impersonation. store.Store(self.fake_cred) properties.VALUES.auth.impersonate_service_account.Set('*****@*****.**') # Expect a call to get a temp access token for impersonation. fake_token = 'impersonation-token' self.mock_client.projects_serviceAccounts.GenerateAccessToken.Expect( self.gen_request_msg( name='projects/-/serviceAccounts/[email protected]', generateAccessTokenRequest=self.messages.GenerateAccessTokenRequest( scope=config.CLOUDSDK_SCOPES)), self.messages.GenerateAccessTokenResponse( accessToken=fake_token, expireTime='2016-01-08T00:00:00Z') ) request_mock = self.StartObjectPatch( httplib2.Http, 'request', return_value=(httplib2.Response({'status': 200}), b'')) try: store.IMPERSONATION_TOKEN_PROVIDER = ( util.ImpersonationAccessTokenProvider()) http.Http().request('http://foo.com', 'GET', None, {}) access_token = request_mock.call_args[0][3][b'Authorization'] self.assertEqual(access_token, b'Bearer ' + fake_token.encode('utf8')) finally: store.IMPERSONATION_TOKEN_PROVIDER = None
def GenerateIdToken(service_account_id, audience, include_email=False): """Generates an id token for the given service account.""" service_account_ref = resources.REGISTRY.Parse( service_account_id, collection='iamcredentials.serviceAccounts', params={ 'projectsId': '-', 'serviceAccountsId': service_account_id }) # pylint: disable=protected-access http_client = http_creds.Http(enable_resource_quota=False, response_encoding=http_creds.ENCODING, allow_account_impersonation=False) iam_client = apis_internal._GetClientInstance('iamcredentials', 'v1', http_client=http_client) response = iam_client.projects_serviceAccounts.GenerateIdToken( iam_client.MESSAGES_MODULE. IamcredentialsProjectsServiceAccountsGenerateIdTokenRequest( name=service_account_ref.RelativeName(), generateIdTokenRequest=iam_client.MESSAGES_MODULE. GenerateIdTokenRequest(audience=audience, includeEmail=include_email))) return response.token
def GetClientInstance(api_name, api_version, no_http=False): """Returns an instance of the API client specified in the args. Args: api_name: str, The API name (or the command surface name, if different). api_version: str, The version of the API. no_http: bool, True to not create an http object for this client. Returns: base_api.BaseApiClient, An instance of the specified API client. """ # pylint: disable=g-import-not-at-top if no_http: http_client = None else: # Import http only when needed, as it depends on credential infrastructure # which is not needed in all cases. from googlecloudsdk.core.credentials import http http_client = http.Http() client_class = GetClientClass(api_name, api_version) return client_class(url=GetEffectiveApiEndpoint(api_name, api_version, client_class), get_credentials=False, http=http_client)
def CheckReachability(urls, http_client=None): """Checks whether the hosts of given urls are reachable. Args: urls: iterable(str), The list of urls to check connection to. http_client: httplib2.Http, an object used by gcloud to make http and https connections. Defaults to an non-authenticated Http object from the googlecloudsdk.core.credentials.http module. Returns: list(Failure): Reasons for why any urls were unreachable. The list will be empty if all urls are reachable, or no urls are passed in. """ if not urls: return [] if not http_client: http_client = http.Http(auth=False) failures = [] for url in urls: try: _, _ = http_client.request(url, method='GET') # TODO(b/29218762): Investigate other possible exceptions. except (httplib.HTTPException, socket.error, ssl.SSLError, httplib2.HttpLib2Error) as err: message = 'Cannot reach {0} ({1})'.format(url, type(err).__name__) failures.append(Failure(message=message, exception=err)) return failures
def testIAMAuthorizationTokenHeaderGoogleAuth(self): authorization_token = 'A very interesting authorization token' authorization_token_file = self.Touch(self.temp_path, 'auth_token_file', contents=authorization_token) properties.VALUES.auth.authorization_token_file.Set( authorization_token_file) expect_headers = { 'user-agent': self.expected_user_agent, 'x-goog-iam-authorization-token': authorization_token, 'authorization': 'Bearer ' + self.FakeAuthAccessToken() } expect_headers = EncodeHeaders(expect_headers) http.Http(use_google_auth=True).request(self.url, 'GET', None, {}, redirections=0, connection_type=None) self.request_mock.assert_called_once_with(self.url, 'GET', body=None, headers=expect_headers, redirections=0, connection_type=None)
def testResourceProjectOverrideCustomProject(self): properties.VALUES.billing.quota_project.Set('bar') http.Http(enable_resource_quota=True).request('http://foo.com', 'GET', None, {}) expect_headers = self._EncodeHeaders({'X-Goog-User-Project': 'bar'}) self.assertDictContainsSubset(expect_headers, self.request_mock.call_args[0][3])
def _GetClientInstance(api_name, api_version, no_http=False, check_response_func=None): """Returns an instance of the API client specified in the args. Args: api_name: str, The API name (or the command surface name, if different). api_version: str, The version of the API. no_http: bool, True to not create an http object for this client. check_response_func: error handling callback to give to apitools. Returns: base_api.BaseApiClient, An instance of the specified API client. """ # pylint: disable=g-import-not-at-top if no_http: http_client = None else: # Import http only when needed, as it depends on credential infrastructure # which is not needed in all cases. from googlecloudsdk.core.credentials import http http_client = http.Http() client_class = _GetClientClass(api_name, api_version) client_instance = client_class( url=_GetEffectiveApiEndpoint(api_name, api_version, client_class), get_credentials=False, http=http_client) if check_response_func is not None: client_instance.check_response_func = check_response_func api_key = properties.VALUES.core.api_key.Get() if api_key: client_instance.AddGlobalParam('key', api_key) header = 'X-Google-Project-Override' client_instance.additional_http_headers[header] = 'apikey' return client_instance
def GetApiClient(default_version='v1beta4'): """Initializes an AppengineApiClient using the specified API version. Uses the api_client_overrides/appengine property to determine which client version to use. Additionally uses the api_endpoint_overrides/appengine property to determine the server endpoint for the App Engine API. Args: default_version: Default client version to use if the api_client_overrides/appengine property was not set. Returns: An AppengineApiClient used by gcloud to communicate with the App Engine API. Raises: ValueError: If default_version does not correspond to a supported version of the API. """ api_version = properties.VALUES.api_client_overrides.appengine.Get() if not api_version: api_version = default_version client = KNOWN_APIS.get(api_version) if not client: raise ValueError('Invalid API version: [{0}]'.format(api_version)) endpoint_override = properties.VALUES.api_endpoint_overrides.appengine.Get( ) appengine_client = client(url=endpoint_override, get_credentials=False, http=http.Http()) return AppengineApiClient(appengine_client, api_version)
def CreateBackup(backup_ref, args, encryption_type=None, kms_key=None): """Create a new backup.""" client = apis.GetClientInstance('spanner', 'v1') msgs = apis.GetMessagesModule('spanner', 'v1') query_params = {'alt': 'json', 'backupId': args.backup} if encryption_type: query_params['encryptionConfig.encryptionType'] = encryption_type if kms_key: query_params['encryptionConfig.kmsKeyName'] = kms_key parent = backup_ref.Parent().RelativeName() url = '{}v1/{}/backups?{}'.format(client.url, parent, urllib.parse.urlencode(query_params)) backup = msgs.Backup( database=parent + '/databases/' + args.database, expireTime=CheckAndGetExpireTime(args)) if args.IsSpecified('version_time'): backup.versionTime = args.version_time # Workaround since gcloud cannot handle HttpBody properly (b/31403673). response_encoding = None if six.PY2 else 'utf-8' # Make an http request directly instead of using the apitools client which # does not support '.' characters in query parameters (b/31244944). response, response_body = http.Http( response_encoding=response_encoding).request( uri=url, method='POST', body=client.SerializeMessage(backup)) if int(response.get('status')) != httplib.OK: raise HttpRequestFailedError('HTTP request failed. Response: ' + response_body) message_type = getattr(msgs, 'Operation') return client.DeserializeMessage(message_type, response_body)
def LookupMembershipName(version, group_id, member_email): """Lookup membership name for a specific pair of member key id and group email. Args: version: Release track information group_id: str, group id (e.g. groups/03qco8b4452k99t) member_email: str, member email Returns: LookupMembershipNameResponse: Response message for LookupMembershipName operation which is containing a resource name of the membership in the format: 'name: members/{member_id}' """ client = GetClient(version) # Following part is added to resolve the gcloud known issue described # in this bug: b/141658179 query_params = [('parent', group_id), ('memberKey.id', member_email)] base_url = client.url url = '{}{}/{}/memberships:lookup?{}'.format( base_url, version, group_id, urllib.parse.urlencode(query_params)) unused_response, raw_content = http.Http().request(uri=url) return json.loads(raw_content.decode('utf8'))
def testNoCredentialsMode(self): properties.VALUES.billing.quota_project.Set('bar') properties.VALUES.auth.disable_credentials.Set(True) http.Http(use_google_auth=True).request(self.url, 'GET', None, {}) del self.common_headers[b'authorization'] self.request_mock.assert_called_once_with(self.url, 'GET', None, self.common_headers)
def testCurrentProjectWithFallbackMode_WithoutPermission(self): properties.VALUES.billing.quota_project.Set( properties.VALUES.billing.CURRENT_PROJECT_WITH_FALLBACK) properties.VALUES.core.project.Set('foo') self.request_mock.side_effect = (self.permission_denied_response, mock.DEFAULT) http.Http(use_google_auth=True).request(self.url, 'GET', None, {}) headers_with_quota = self.common_headers.copy() headers_with_quota[b'X-Goog-User-Project'] = b'foo' self.assertEqual(self.request_mock.call_count, 2) calls = [ mock.call(self.url, 'GET', body=None, headers=headers_with_quota, redirections=5, connection_type=None), mock.call(self.url, 'GET', body=None, headers=self.common_headers, redirections=5, connection_type=None), ] self.request_mock.assert_has_calls(calls)
def GenerateConnectAgentManifest(self, option): """Generate the YAML manifest to deploy the GKE Connect agent. Args: option: an instance of ConnectAgentOption. Returns: A slice of connect agent manifest resources. Raises: Error: if the API call to generate connect agent manifest failed. """ project = properties.VALUES.core.project.GetOrFail() # Can't directly use the generated API client given that it currently # doesn't support nested messages. See the discussion here: # https://groups.google.com/a/google.com/forum/#!msg/cloud-sdk-eng/hwdwUTEmvlw/fRdrvK26AAAJ query_params = [('connectAgent.name', option.name), ('connectAgent.namespace', option.namespace), ('connectAgent.proxy', option.proxy), ('isUpgrade', option.is_upgrade), ('version', option.version), ('registry', option.registry), ('image_pull_secret_content', option.image_pull_secret_content)] base_url = self.client.url url = '{}/v1beta1/projects/{}/locations/global/connectAgents:generateManifest?{}'.format( base_url, project, urllib.parse.urlencode(query_params)) response, raw_content = http.Http().request(uri=url) content = core_encoding.Decode(raw_content) status_code = response.get('status') if status_code != '200': msg = self._HTTP_ERROR_FORMAT.format(status_code, content) raise exceptions.HttpException(msg) return json.loads(content).get('manifest')
def ReadObject(self, object_ref): """Read a file from the given Cloud Storage bucket. Args: object_ref: storage_util.ObjectReference, The object to read from. Raises: BadFileException if the file read is not successful. Returns: file-like object containing the data read. """ data = io.BytesIO() chunksize = self._GetChunkSize() download = transfer.Download.FromStream(data, chunksize=chunksize) download.bytes_http = http.Http(response_encoding=None) get_req = self.messages.StorageObjectsGetRequest( bucket=object_ref.bucket, object=object_ref.object) log.info('Reading [%s]', object_ref) try: self.client.objects.Get(get_req, download=download) except api_exceptions.HttpError as err: raise exceptions.BadFileException( 'Could not read [{object_}]. Please retry: {err}'.format( object_=object_ref, err=http_exc.HttpException(err))) data.seek(0) return data
def testDisabledAuth(self): properties.VALUES.auth.disable_credentials.Set(True) expect_headers = {'user-agent': self.expected_user_agent} expect_headers = EncodeHeaders(expect_headers) http_client = http.Http() http_client.request(self.url, 'GET', None, None, 0, None) self.request_mock.assert_called_once_with(self.url, 'GET', None, expect_headers, 0, None)
def _CheckURL(self, url): try: http.Http(auth=False).request(url, method='GET') # TODO(b/29218762): Investigate other possible exceptions. except (httplib.HTTPException, socket.error, ssl.SSLError, httplib2.HttpLib2Error, socks.HTTPError) as err: msg = 'Cannot reach {0} ({1})'.format(url, type(err).__name__) return check_base.Failure(message=msg, exception=err)
def testResourceProjectOverrideUnsetDefault(self): properties.VALUES.billing.quota_project.Set(None) http.Http(enable_resource_quota=False).request(self.url, 'GET', None, {}) self.AssertHeaderNotContains('X-Goog-User-Project', self.request_mock.call_args) self.AssertHeaderNotContains(b'X-Goog-User-Project', self.request_mock.call_args)
def testResourceProjectOverrideForceResourceQuota(self): properties.VALUES.billing.quota_project.Set( properties.VALUES.billing.LEGACY) properties.VALUES.core.project.Set('foo') http.Http(enable_resource_quota=True, force_resource_quota=True).request(self.url, 'GET', None, {}) expect_headers = EncodeHeaders({'X-Goog-User-Project': 'foo'}) self.AssertHeaderContains(expect_headers, self.request_mock.call_args)
def testResourceProjectOverrideUnsetDefault(self): properties.VALUES.billing.quota_project.Set(None) http.Http(enable_resource_quota=False).request('http://foo.com', 'GET', None, {}) self.assertNotIn('X-Goog-User-Project', self.request_mock.call_args[0][3]) self.assertNotIn(b'X-Goog-User-Project', self.request_mock.call_args[0][3])
def __init__(self, bucket, obj, out=log.status, url_pattern=None): self.http = http.Http() url_pattern = url_pattern or self.GCS_URL_PATTERN self.url = url_pattern.format(bucket=bucket, obj=obj) log.debug('GCS logfile url is ' + self.url) # position in the file being read self.cursor = 0 self.out = out
def __init__(self, batch_url, compute, project, resources): """Sets fields expected by ScopePrompter.""" self.batch_url = batch_url self.compute = compute self.project = project self.resources = resources self.resource_type = None self.http = http.Http()
def HttpClient(self): # Import http only when needed, as it depends on credential infrastructure # which is not needed in all cases. assert self.active from googlecloudsdk.core.credentials import http as http_creds # pylint: disable=g-import-not-at-top http_client = http_creds.Http(response_encoding=http_creds.ENCODING, ca_certs=self.ca_certs) return http_client
def testComputeServiceAccountGoogleAuth(self): # Don't add x-goog-user-project header for service accounts, # unless billing project is set. In that case, use the value for the header. http.Http(enable_resource_quota=True, use_google_auth=True).request(self.url, 'GET', None, {}) self.AssertHeaderNotContains('X-Goog-User-Project', self.request_mock.call_args) self.AssertHeaderNotContains(b'X-Goog-User-Project', self.request_mock.call_args)
def testTokenRefreshErrorGoogleAuth(self): refresh_mock = self.StartObjectPatch(credentials.Credentials, 'before_request') refresh_mock.side_effect = google_auth_exceptions.RefreshError http_client = http.Http(use_google_auth=True) with self.assertRaisesRegex( store.TokenRefreshError, 'There was a problem refreshing your current auth tokens'): http_client.request('http://foo.com')
def _GetClientInstance(api_name, api_version, no_http=False, check_response_func=None, enable_resource_quota=True, force_resource_quota=False, ca_certs=None, allow_account_impersonation=True): """Returns an instance of the API client specified in the args. Args: api_name: str, The API name (or the command surface name, if different). api_version: str, The version of the API. no_http: bool, True to not create an http object for this client. check_response_func: error handling callback to give to apitools. enable_resource_quota: bool, By default, we are going to tell APIs to use the quota of the project being operated on. For some APIs we want to use gcloud's quota, so you can explicitly disable that behavior by passing False here. force_resource_quota: bool, If true resource project quota will be used by this client regardless of the settings in gcloud. This should be used for newer APIs that cannot work with legacy project quota. ca_certs: str, absolute path of a ca_certs file to use instead of the default allow_account_impersonation: bool, True to allow use of impersonated service account credentials for calls made with this client. If False, the active user credentials will always be used. Returns: base_api.BaseApiClient, An instance of the specified API client. """ # TODO(b/77278279): Decide whether we should always set this or not. encoding = None if six.PY2 else 'utf8' # pylint: disable=g-import-not-at-top if no_http: http_client = None else: # Import http only when needed, as it depends on credential infrastructure # which is not needed in all cases. from googlecloudsdk.core.credentials import http http_client = http.Http( enable_resource_quota=enable_resource_quota, force_resource_quota=force_resource_quota, response_encoding=encoding, ca_certs=ca_certs, allow_account_impersonation=allow_account_impersonation) client_class = _GetClientClass(api_name, api_version) client_instance = client_class( url=_GetEffectiveApiEndpoint(api_name, api_version, client_class), get_credentials=False, http=http_client) if check_response_func is not None: client_instance.check_response_func = check_response_func api_key = properties.VALUES.core.api_key.Get() if api_key: client_instance.AddGlobalParam('key', api_key) header = 'X-Google-Project-Override' client_instance.additional_http_headers[header] = 'apikey' return client_instance
def MakeAnalyzeIamPolicyHttpRequests(args, api_version=V1P4ALPHA1_API_VERSION): """Manually make the get assets history request.""" http_client = http.Http() parent = asset_utils.GetParentNameForAnalyzeIamPolicy(args.organization) url_base = '{0}/{1}/{2}:{3}'.format(BASE_URL, api_version, parent, 'analyzeIamPolicy') params = [] if args.IsSpecified('full_resource_name'): params.extend([('resourceSelector.fullResourceName', args.full_resource_name)]) if args.IsSpecified('identity'): params.extend([('identitySelector.identity', args.identity)]) if args.IsSpecified('roles'): params.extend([('accessSelector.roles', r) for r in args.roles]) if args.IsSpecified('permissions'): params.extend([('accessSelector.permissions', p) for p in args.permissions]) if args.IsSpecified('expand_groups'): params.extend([('options.expandGroups', args.expand_groups)]) if args.IsSpecified('expand_resources'): params.extend([('options.expandResources', args.expand_resources)]) if args.IsSpecified('expand_roles'): params.extend([('options.expandRoles', args.expand_roles)]) if args.IsSpecified('output_resource_edges'): params.extend([('options.outputResourceEdges', args.output_resource_edges)]) if args.IsSpecified('output_group_edges'): params.extend([('options.outputGroupEdges', args.output_group_edges)]) if args.IsSpecified('output_partial_result_before_timeout'): params.extend([('options.outputPartialResultBeforeTimeout', args.output_partial_result_before_timeout)]) url_query = six.moves.urllib.parse.urlencode(params) url = '?'.join([url_base, url_query]) response, raw_content = http_client.request(uri=url, headers=_HEADERS) content = core_encoding.Decode(raw_content) if response['status'] != '200': http_error = api_exceptions.HttpError(response, content, url) raise exceptions.HttpException(http_error) response_message_class = GetMessages(api_version).AnalyzeIamPolicyResponse try: response = encoding.JsonToMessage(response_message_class, content) except ValueError as e: err_msg = ('Failed receiving proper response from server, cannot' 'parse received assets. Error details: ' + six.text_type(e)) raise MessageDecodeError(err_msg) return response
def testDisabledAuthGoogleAuth(self): properties.VALUES.auth.disable_credentials.Set(True) url = 'http://foo.com' expect_headers = {'user-agent': self.expected_user_agent} expect_headers = self._EncodeHeaders(expect_headers) http_client = http.Http(use_google_auth=True) http_client.request(url, 'GET', None, None, 0, None) self.request_mock.assert_called_once_with(url, 'GET', None, expect_headers, 0, None)
def testComputeServiceAccount(self): # Don't do it for service accounts. properties.VALUES.billing.quota_project.Set('bar') http.Http(enable_resource_quota=True).request('http://foo.com', 'GET', None, {}) self.assertNotIn('X-Goog-User-Project', self.request_mock.call_args[0][3]) self.assertNotIn(b'X-Goog-User-Project', self.request_mock.call_args[0][3])