def DiagnoseNetworkConnection(urls=None):
"""Diagnose and fix local network connection issues.
Checks reachability of passed in urls. If any are unreachable, asks user to
change or update Cloud SDK proxy properties. If user makes proxy settings
changes, then reachability check is run one more time.
Args:
urls: iterable(str), The list of urls to check connection to. Defaults to
DEFAULT_URLS (above).
Returns:
bool: Whether the network connection check ultimately passed.
"""
if not urls:
urls = _DefaultUrls()
if _CheckNetworkConnection(urls, http.Http(auth=False), first_run=True):
return True
log.status.Print(
'Network connection problems may be due to proxy or firewall settings.'
)
while True:
if not _ChangeGcloudProxySettings():
return False
if _CheckNetworkConnection(urls,
http.Http(auth=False),
first_run=False):
return True
def testServiceAccountImpersonation(self):
http.Http().request('http://foo.com', 'GET', None, {})
access_token = self.request_mock.call_args[1]['headers'][
b'authorization']
self.assertEqual(access_token,
b'Bearer ' + self.impersonation_token.encode('utf-8'))
def testServiceAccountImpersonation(self):
# This is the logged in credential, but it will not be used to make the call
# because of the impersonation.
store.Store(self.fake_cred)
properties.VALUES.auth.impersonate_service_account.Set('*****@*****.**')
# Expect a call to get a temp access token for impersonation.
fake_token = 'impersonation-token'
self.mock_client.projects_serviceAccounts.GenerateAccessToken.Expect(
self.gen_request_msg(
name='projects/-/serviceAccounts/[email protected]',
generateAccessTokenRequest=self.messages.GenerateAccessTokenRequest(
scope=config.CLOUDSDK_SCOPES)),
self.messages.GenerateAccessTokenResponse(
accessToken=fake_token, expireTime='2016-01-08T00:00:00Z')
)
request_mock = self.StartObjectPatch(
httplib2.Http, 'request',
return_value=(httplib2.Response({'status': 200}), b''))
try:
store.IMPERSONATION_TOKEN_PROVIDER = (
util.ImpersonationAccessTokenProvider())
http.Http().request('http://foo.com', 'GET', None, {})
access_token = request_mock.call_args[0][3][b'Authorization']
self.assertEqual(access_token, b'Bearer ' + fake_token.encode('utf8'))
finally:
store.IMPERSONATION_TOKEN_PROVIDER = None
def GenerateIdToken(service_account_id, audience, include_email=False):
"""Generates an id token for the given service account."""
service_account_ref = resources.REGISTRY.Parse(
service_account_id,
collection='iamcredentials.serviceAccounts',
params={
'projectsId': '-',
'serviceAccountsId': service_account_id
})
# pylint: disable=protected-access
http_client = http_creds.Http(enable_resource_quota=False,
response_encoding=http_creds.ENCODING,
allow_account_impersonation=False)
iam_client = apis_internal._GetClientInstance('iamcredentials',
'v1',
http_client=http_client)
response = iam_client.projects_serviceAccounts.GenerateIdToken(
iam_client.MESSAGES_MODULE.
IamcredentialsProjectsServiceAccountsGenerateIdTokenRequest(
name=service_account_ref.RelativeName(),
generateIdTokenRequest=iam_client.MESSAGES_MODULE.
GenerateIdTokenRequest(audience=audience,
includeEmail=include_email)))
return response.token
def GetClientInstance(api_name, api_version, no_http=False):
"""Returns an instance of the API client specified in the args.
Args:
api_name: str, The API name (or the command surface name, if different).
api_version: str, The version of the API.
no_http: bool, True to not create an http object for this client.
Returns:
base_api.BaseApiClient, An instance of the specified API client.
"""
# pylint: disable=g-import-not-at-top
if no_http:
http_client = None
else:
# Import http only when needed, as it depends on credential infrastructure
# which is not needed in all cases.
from googlecloudsdk.core.credentials import http
http_client = http.Http()
client_class = GetClientClass(api_name, api_version)
return client_class(url=GetEffectiveApiEndpoint(api_name, api_version,
client_class),
get_credentials=False,
http=http_client)
def CheckReachability(urls, http_client=None):
"""Checks whether the hosts of given urls are reachable.
Args:
urls: iterable(str), The list of urls to check connection to.
http_client: httplib2.Http, an object used by gcloud to make http and https
connections. Defaults to an non-authenticated Http object from the
googlecloudsdk.core.credentials.http module.
Returns:
list(Failure): Reasons for why any urls were unreachable. The list will be
empty if all urls are reachable, or no urls are passed in.
"""
if not urls:
return []
if not http_client:
http_client = http.Http(auth=False)
failures = []
for url in urls:
try:
_, _ = http_client.request(url, method='GET')
# TODO(b/29218762): Investigate other possible exceptions.
except (httplib.HTTPException, socket.error, ssl.SSLError,
httplib2.HttpLib2Error) as err:
message = 'Cannot reach {0} ({1})'.format(url, type(err).__name__)
failures.append(Failure(message=message, exception=err))
return failures
def testIAMAuthorizationTokenHeaderGoogleAuth(self):
authorization_token = 'A very interesting authorization token'
authorization_token_file = self.Touch(self.temp_path,
'auth_token_file',
contents=authorization_token)
properties.VALUES.auth.authorization_token_file.Set(
authorization_token_file)
expect_headers = {
'user-agent': self.expected_user_agent,
'x-goog-iam-authorization-token': authorization_token,
'authorization': 'Bearer ' + self.FakeAuthAccessToken()
}
expect_headers = EncodeHeaders(expect_headers)
http.Http(use_google_auth=True).request(self.url,
'GET',
None, {},
redirections=0,
connection_type=None)
self.request_mock.assert_called_once_with(self.url,
'GET',
body=None,
headers=expect_headers,
redirections=0,
connection_type=None)
def testResourceProjectOverrideCustomProject(self):
properties.VALUES.billing.quota_project.Set('bar')
http.Http(enable_resource_quota=True).request('http://foo.com', 'GET',
None, {})
expect_headers = self._EncodeHeaders({'X-Goog-User-Project': 'bar'})
self.assertDictContainsSubset(expect_headers,
self.request_mock.call_args[0][3])
def _GetClientInstance(api_name, api_version, no_http=False,
check_response_func=None):
"""Returns an instance of the API client specified in the args.
Args:
api_name: str, The API name (or the command surface name, if different).
api_version: str, The version of the API.
no_http: bool, True to not create an http object for this client.
check_response_func: error handling callback to give to apitools.
Returns:
base_api.BaseApiClient, An instance of the specified API client.
"""
# pylint: disable=g-import-not-at-top
if no_http:
http_client = None
else:
# Import http only when needed, as it depends on credential infrastructure
# which is not needed in all cases.
from googlecloudsdk.core.credentials import http
http_client = http.Http()
client_class = _GetClientClass(api_name, api_version)
client_instance = client_class(
url=_GetEffectiveApiEndpoint(api_name, api_version, client_class),
get_credentials=False,
http=http_client)
if check_response_func is not None:
client_instance.check_response_func = check_response_func
api_key = properties.VALUES.core.api_key.Get()
if api_key:
client_instance.AddGlobalParam('key', api_key)
header = 'X-Google-Project-Override'
client_instance.additional_http_headers[header] = 'apikey'
return client_instance
def GetApiClient(default_version='v1beta4'):
"""Initializes an AppengineApiClient using the specified API version.
Uses the api_client_overrides/appengine property to determine which client
version to use. Additionally uses the api_endpoint_overrides/appengine
property to determine the server endpoint for the App Engine API.
Args:
default_version: Default client version to use if the
api_client_overrides/appengine property was not set.
Returns:
An AppengineApiClient used by gcloud to communicate with the App Engine API.
Raises:
ValueError: If default_version does not correspond to a supported version of
the API.
"""
api_version = properties.VALUES.api_client_overrides.appengine.Get()
if not api_version:
api_version = default_version
client = KNOWN_APIS.get(api_version)
if not client:
raise ValueError('Invalid API version: [{0}]'.format(api_version))
endpoint_override = properties.VALUES.api_endpoint_overrides.appengine.Get(
)
appengine_client = client(url=endpoint_override,
get_credentials=False,
http=http.Http())
return AppengineApiClient(appengine_client, api_version)
def CreateBackup(backup_ref, args, encryption_type=None, kms_key=None):
"""Create a new backup."""
client = apis.GetClientInstance('spanner', 'v1')
msgs = apis.GetMessagesModule('spanner', 'v1')
query_params = {'alt': 'json', 'backupId': args.backup}
if encryption_type:
query_params['encryptionConfig.encryptionType'] = encryption_type
if kms_key:
query_params['encryptionConfig.kmsKeyName'] = kms_key
parent = backup_ref.Parent().RelativeName()
url = '{}v1/{}/backups?{}'.format(client.url, parent,
urllib.parse.urlencode(query_params))
backup = msgs.Backup(
database=parent + '/databases/' + args.database,
expireTime=CheckAndGetExpireTime(args))
if args.IsSpecified('version_time'):
backup.versionTime = args.version_time
# Workaround since gcloud cannot handle HttpBody properly (b/31403673).
response_encoding = None if six.PY2 else 'utf-8'
# Make an http request directly instead of using the apitools client which
# does not support '.' characters in query parameters (b/31244944).
response, response_body = http.Http(
response_encoding=response_encoding).request(
uri=url, method='POST', body=client.SerializeMessage(backup))
if int(response.get('status')) != httplib.OK:
raise HttpRequestFailedError('HTTP request failed. Response: ' +
response_body)
message_type = getattr(msgs, 'Operation')
return client.DeserializeMessage(message_type, response_body)
def LookupMembershipName(version, group_id, member_email):
"""Lookup membership name for a specific pair of member key id and group email.
Args:
version: Release track information
group_id: str, group id (e.g. groups/03qco8b4452k99t)
member_email: str, member email
Returns:
LookupMembershipNameResponse: Response message for LookupMembershipName
operation which is containing a resource name of the membership in the
format:
'name: members/{member_id}'
"""
client = GetClient(version)
# Following part is added to resolve the gcloud known issue described
# in this bug: b/141658179
query_params = [('parent', group_id), ('memberKey.id', member_email)]
base_url = client.url
url = '{}{}/{}/memberships:lookup?{}'.format(
base_url, version, group_id, urllib.parse.urlencode(query_params))
unused_response, raw_content = http.Http().request(uri=url)
return json.loads(raw_content.decode('utf8'))
def testNoCredentialsMode(self):
properties.VALUES.billing.quota_project.Set('bar')
properties.VALUES.auth.disable_credentials.Set(True)
http.Http(use_google_auth=True).request(self.url, 'GET', None, {})
del self.common_headers[b'authorization']
self.request_mock.assert_called_once_with(self.url, 'GET', None,
self.common_headers)
def testCurrentProjectWithFallbackMode_WithoutPermission(self):
properties.VALUES.billing.quota_project.Set(
properties.VALUES.billing.CURRENT_PROJECT_WITH_FALLBACK)
properties.VALUES.core.project.Set('foo')
self.request_mock.side_effect = (self.permission_denied_response,
mock.DEFAULT)
http.Http(use_google_auth=True).request(self.url, 'GET', None, {})
headers_with_quota = self.common_headers.copy()
headers_with_quota[b'X-Goog-User-Project'] = b'foo'
self.assertEqual(self.request_mock.call_count, 2)
calls = [
mock.call(self.url,
'GET',
body=None,
headers=headers_with_quota,
redirections=5,
connection_type=None),
mock.call(self.url,
'GET',
body=None,
headers=self.common_headers,
redirections=5,
connection_type=None),
]
self.request_mock.assert_has_calls(calls)
def GenerateConnectAgentManifest(self, option):
"""Generate the YAML manifest to deploy the GKE Connect agent.
Args:
option: an instance of ConnectAgentOption.
Returns:
A slice of connect agent manifest resources.
Raises:
Error: if the API call to generate connect agent manifest failed.
"""
project = properties.VALUES.core.project.GetOrFail()
# Can't directly use the generated API client given that it currently
# doesn't support nested messages. See the discussion here:
# https://groups.google.com/a/google.com/forum/#!msg/cloud-sdk-eng/hwdwUTEmvlw/fRdrvK26AAAJ
query_params = [('connectAgent.name', option.name),
('connectAgent.namespace', option.namespace),
('connectAgent.proxy', option.proxy),
('isUpgrade', option.is_upgrade),
('version', option.version),
('registry', option.registry),
('image_pull_secret_content',
option.image_pull_secret_content)]
base_url = self.client.url
url = '{}/v1beta1/projects/{}/locations/global/connectAgents:generateManifest?{}'.format(
base_url, project, urllib.parse.urlencode(query_params))
response, raw_content = http.Http().request(uri=url)
content = core_encoding.Decode(raw_content)
status_code = response.get('status')
if status_code != '200':
msg = self._HTTP_ERROR_FORMAT.format(status_code, content)
raise exceptions.HttpException(msg)
return json.loads(content).get('manifest')
def ReadObject(self, object_ref):
"""Read a file from the given Cloud Storage bucket.
Args:
object_ref: storage_util.ObjectReference, The object to read from.
Raises:
BadFileException if the file read is not successful.
Returns:
file-like object containing the data read.
"""
data = io.BytesIO()
chunksize = self._GetChunkSize()
download = transfer.Download.FromStream(data, chunksize=chunksize)
download.bytes_http = http.Http(response_encoding=None)
get_req = self.messages.StorageObjectsGetRequest(
bucket=object_ref.bucket, object=object_ref.object)
log.info('Reading [%s]', object_ref)
try:
self.client.objects.Get(get_req, download=download)
except api_exceptions.HttpError as err:
raise exceptions.BadFileException(
'Could not read [{object_}]. Please retry: {err}'.format(
object_=object_ref, err=http_exc.HttpException(err)))
data.seek(0)
return data
def testDisabledAuth(self):
properties.VALUES.auth.disable_credentials.Set(True)
expect_headers = {'user-agent': self.expected_user_agent}
expect_headers = EncodeHeaders(expect_headers)
http_client = http.Http()
http_client.request(self.url, 'GET', None, None, 0, None)
self.request_mock.assert_called_once_with(self.url, 'GET', None,
expect_headers, 0, None)
def _CheckURL(self, url):
try:
http.Http(auth=False).request(url, method='GET')
# TODO(b/29218762): Investigate other possible exceptions.
except (httplib.HTTPException, socket.error, ssl.SSLError,
httplib2.HttpLib2Error, socks.HTTPError) as err:
msg = 'Cannot reach {0} ({1})'.format(url, type(err).__name__)
return check_base.Failure(message=msg, exception=err)
def testResourceProjectOverrideUnsetDefault(self):
properties.VALUES.billing.quota_project.Set(None)
http.Http(enable_resource_quota=False).request(self.url, 'GET', None,
{})
self.AssertHeaderNotContains('X-Goog-User-Project',
self.request_mock.call_args)
self.AssertHeaderNotContains(b'X-Goog-User-Project',
self.request_mock.call_args)
def testResourceProjectOverrideForceResourceQuota(self):
properties.VALUES.billing.quota_project.Set(
properties.VALUES.billing.LEGACY)
properties.VALUES.core.project.Set('foo')
http.Http(enable_resource_quota=True,
force_resource_quota=True).request(self.url, 'GET', None, {})
expect_headers = EncodeHeaders({'X-Goog-User-Project': 'foo'})
self.AssertHeaderContains(expect_headers, self.request_mock.call_args)
def testResourceProjectOverrideUnsetDefault(self):
properties.VALUES.billing.quota_project.Set(None)
http.Http(enable_resource_quota=False).request('http://foo.com', 'GET',
None, {})
self.assertNotIn('X-Goog-User-Project',
self.request_mock.call_args[0][3])
self.assertNotIn(b'X-Goog-User-Project',
self.request_mock.call_args[0][3])
def __init__(self, bucket, obj, out=log.status, url_pattern=None):
self.http = http.Http()
url_pattern = url_pattern or self.GCS_URL_PATTERN
self.url = url_pattern.format(bucket=bucket, obj=obj)
log.debug('GCS logfile url is ' + self.url)
# position in the file being read
self.cursor = 0
self.out = out
def __init__(self, batch_url, compute, project, resources):
"""Sets fields expected by ScopePrompter."""
self.batch_url = batch_url
self.compute = compute
self.project = project
self.resources = resources
self.resource_type = None
self.http = http.Http()
def HttpClient(self):
# Import http only when needed, as it depends on credential infrastructure
# which is not needed in all cases.
assert self.active
from googlecloudsdk.core.credentials import http as http_creds # pylint: disable=g-import-not-at-top
http_client = http_creds.Http(response_encoding=http_creds.ENCODING,
ca_certs=self.ca_certs)
return http_client
def testComputeServiceAccountGoogleAuth(self):
# Don't add x-goog-user-project header for service accounts,
# unless billing project is set. In that case, use the value for the header.
http.Http(enable_resource_quota=True,
use_google_auth=True).request(self.url, 'GET', None, {})
self.AssertHeaderNotContains('X-Goog-User-Project',
self.request_mock.call_args)
self.AssertHeaderNotContains(b'X-Goog-User-Project',
self.request_mock.call_args)
def testTokenRefreshErrorGoogleAuth(self):
refresh_mock = self.StartObjectPatch(credentials.Credentials,
'before_request')
refresh_mock.side_effect = google_auth_exceptions.RefreshError
http_client = http.Http(use_google_auth=True)
with self.assertRaisesRegex(
store.TokenRefreshError,
'There was a problem refreshing your current auth tokens'):
http_client.request('http://foo.com')
def _GetClientInstance(api_name, api_version, no_http=False,
check_response_func=None, enable_resource_quota=True,
force_resource_quota=False,
ca_certs=None, allow_account_impersonation=True):
"""Returns an instance of the API client specified in the args.
Args:
api_name: str, The API name (or the command surface name, if different).
api_version: str, The version of the API.
no_http: bool, True to not create an http object for this client.
check_response_func: error handling callback to give to apitools.
enable_resource_quota: bool, By default, we are going to tell APIs to use
the quota of the project being operated on. For some APIs we want to use
gcloud's quota, so you can explicitly disable that behavior by passing
False here.
force_resource_quota: bool, If true resource project quota will be used by
this client regardless of the settings in gcloud. This should be used for
newer APIs that cannot work with legacy project quota.
ca_certs: str, absolute path of a ca_certs file to use instead of the
default
allow_account_impersonation: bool, True to allow use of impersonated service
account credentials for calls made with this client. If False, the active
user credentials will always be used.
Returns:
base_api.BaseApiClient, An instance of the specified API client.
"""
# TODO(b/77278279): Decide whether we should always set this or not.
encoding = None if six.PY2 else 'utf8'
# pylint: disable=g-import-not-at-top
if no_http:
http_client = None
else:
# Import http only when needed, as it depends on credential infrastructure
# which is not needed in all cases.
from googlecloudsdk.core.credentials import http
http_client = http.Http(
enable_resource_quota=enable_resource_quota,
force_resource_quota=force_resource_quota,
response_encoding=encoding,
ca_certs=ca_certs,
allow_account_impersonation=allow_account_impersonation)
client_class = _GetClientClass(api_name, api_version)
client_instance = client_class(
url=_GetEffectiveApiEndpoint(api_name, api_version, client_class),
get_credentials=False,
http=http_client)
if check_response_func is not None:
client_instance.check_response_func = check_response_func
api_key = properties.VALUES.core.api_key.Get()
if api_key:
client_instance.AddGlobalParam('key', api_key)
header = 'X-Google-Project-Override'
client_instance.additional_http_headers[header] = 'apikey'
return client_instance
def MakeAnalyzeIamPolicyHttpRequests(args, api_version=V1P4ALPHA1_API_VERSION):
"""Manually make the get assets history request."""
http_client = http.Http()
parent = asset_utils.GetParentNameForAnalyzeIamPolicy(args.organization)
url_base = '{0}/{1}/{2}:{3}'.format(BASE_URL, api_version, parent,
'analyzeIamPolicy')
params = []
if args.IsSpecified('full_resource_name'):
params.extend([('resourceSelector.fullResourceName',
args.full_resource_name)])
if args.IsSpecified('identity'):
params.extend([('identitySelector.identity', args.identity)])
if args.IsSpecified('roles'):
params.extend([('accessSelector.roles', r) for r in args.roles])
if args.IsSpecified('permissions'):
params.extend([('accessSelector.permissions', p)
for p in args.permissions])
if args.IsSpecified('expand_groups'):
params.extend([('options.expandGroups', args.expand_groups)])
if args.IsSpecified('expand_resources'):
params.extend([('options.expandResources', args.expand_resources)])
if args.IsSpecified('expand_roles'):
params.extend([('options.expandRoles', args.expand_roles)])
if args.IsSpecified('output_resource_edges'):
params.extend([('options.outputResourceEdges',
args.output_resource_edges)])
if args.IsSpecified('output_group_edges'):
params.extend([('options.outputGroupEdges', args.output_group_edges)])
if args.IsSpecified('output_partial_result_before_timeout'):
params.extend([('options.outputPartialResultBeforeTimeout',
args.output_partial_result_before_timeout)])
url_query = six.moves.urllib.parse.urlencode(params)
url = '?'.join([url_base, url_query])
response, raw_content = http_client.request(uri=url, headers=_HEADERS)
content = core_encoding.Decode(raw_content)
if response['status'] != '200':
http_error = api_exceptions.HttpError(response, content, url)
raise exceptions.HttpException(http_error)
response_message_class = GetMessages(api_version).AnalyzeIamPolicyResponse
try:
response = encoding.JsonToMessage(response_message_class, content)
except ValueError as e:
err_msg = ('Failed receiving proper response from server, cannot'
'parse received assets. Error details: ' + six.text_type(e))
raise MessageDecodeError(err_msg)
return response
def testDisabledAuthGoogleAuth(self):
properties.VALUES.auth.disable_credentials.Set(True)
url = 'http://foo.com'
expect_headers = {'user-agent': self.expected_user_agent}
expect_headers = self._EncodeHeaders(expect_headers)
http_client = http.Http(use_google_auth=True)
http_client.request(url, 'GET', None, None, 0, None)
self.request_mock.assert_called_once_with(url, 'GET', None,
expect_headers, 0, None)
def testComputeServiceAccount(self):
# Don't do it for service accounts.
properties.VALUES.billing.quota_project.Set('bar')
http.Http(enable_resource_quota=True).request('http://foo.com', 'GET',
None, {})
self.assertNotIn('X-Goog-User-Project',
self.request_mock.call_args[0][3])
self.assertNotIn(b'X-Goog-User-Project',
self.request_mock.call_args[0][3])
