def Run(self, args):
holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
client = holder.client
request_protobufs = self.CreateRequests(holder, args)
requests = []
for request in request_protobufs:
requests.append(
(client.apitools_client.urlMaps, 'InvalidateCache', request))
if args. async:
resources, errors = batch_helper.MakeRequests(
requests=requests,
http=client.apitools_client.http,
batch_url=client.batch_url)
if not errors:
for invalidation_operation in resources:
log.status.write('Invalidation pending for [{0}]\n'.format(
invalidation_operation.targetLink))
log.status.write('Monitor its progress at [{0}]\n'.format(
invalidation_operation.selfLink))
else:
utils.RaiseToolException(errors)
else:
# We want to run through the generator that MakeRequests returns in order
# to actually make the requests.
resources = client.MakeRequests(requests)
return resources
def Run(self, args):
request_protobufs = self.CreateRequests(args)
requests = []
for request in request_protobufs:
requests.append((self.service, self.method, request))
errors = []
if args. async:
resources, new_errors = batch_helper.MakeRequests(
requests=requests, http=self.http, batch_url=self.batch_url)
if not new_errors:
for invalidation_operation in resources:
log.status.write('Invalidation pending for [{0}]\n'.format(
invalidation_operation.targetLink))
log.status.write('Monitor its progress at [{0}]\n'.format(
invalidation_operation.selfLink))
errors.extend(new_errors)
else:
# We want to run through the generator that MakeRequests returns in order
# to actually make the requests.
resources = list(
request_helper.MakeRequests(requests=requests,
http=self.http,
batch_url=self.batch_url,
errors=errors))
resources = lister.ProcessResults(
resources=resources,
field_selector=property_selector.PropertySelector(
properties=None, transformations=self.transformations))
if errors:
utils.RaiseToolException(errors)
return resources
def NoEnableApiBaseTest(self):
with mock.patch.object(http_wrapper, 'MakeRequest',
autospec=True) as mock_request:
# pylint: disable=unused-argument
def Perform(http, request):
return http_wrapper.Response(
{
'status': '200',
'content-type': 'multipart/mixed; boundary="boundary"',
},
textwrap.dedent("""\
--boundary
content-type: application/json; charset=UTF-8
content-id: <id+0>
HTTP/1.1 403 Forbidden
{{
"error": {{
"errors": [
{{
"domain": "usageLimits",
"reason": "accessNotConfigured",
"message": "{message}",
"extendedHelp": ""
}}
],
"code": 403,
"message": "{message}"
}}
}}
--boundary--""".format(message=self.message)), None)
mock_request.side_effect = Perform
_, errors = batch_helper.MakeRequests(
requests=[(self.compute_v1.instances, 'Get',
self.messages.ComputeInstancesGetRequest(
instance='my-instance-1',
project='my-project',
zone='my-zone'))],
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute')
error_code, error_msg = errors[0]
# Errors are thrown in a layer above this one. It is sufficient to check
# that the error matches what we expect and MakeRequest is only called
# once (which means no retries).
self.assertEqual(error_code, 403)
self.assertEqual(error_msg, self.message)
self.assertEqual(mock_request.call_count, 1)
def _ListCore(requests, http, batch_url, errors, response_handler):
"""Makes a series of list and/or aggregatedList batch requests.
Args:
requests: A list of requests to make. Each element must be a 3-element
tuple where the first element is the service, the second element is
the method ('List' or 'AggregatedList'), and the third element
is a protocol buffer representing either a list or aggregatedList
request.
http: An httplib2.Http-like object.
batch_url: The handler for making batch requests.
errors: A list for capturing errors. If any response contains an error,
it is added to this list.
response_handler: The function to extract information responses.
Yields:
Resources encapsulated in format chosen by response_handler as they are
received from the server.
"""
while requests:
responses, request_errors = batch_helper.MakeRequests(
requests=requests,
http=http,
batch_url=batch_url)
errors.extend(request_errors)
new_requests = []
for i, response in enumerate(responses):
if not response:
continue
service, method, request_protobuf = requests[i]
items, next_page_token = response_handler(response, service, method,
errors)
for item in items:
yield item
if next_page_token:
new_request_protobuf = copy.deepcopy(request_protobuf)
new_request_protobuf.pageToken = next_page_token
new_requests.append((service, method, new_request_protobuf))
requests = new_requests
def _MakeRequests(client, requests, is_async):
"""Helper for making asynchronous or synchronous peering creation requests."""
if is_async:
responses, errors = batch_helper.MakeRequests(
requests=requests,
http=client.apitools_client.http,
batch_url=client.batch_url)
if not errors:
for operation in responses:
log.status.write('Creating network peering for [{0}]\n'.format(
operation.targetLink))
log.status.write('Monitor its progress at [{0}]\n'.format(
operation.selfLink))
else:
utils.RaiseToolException(errors)
else:
# We want to run through the generator that MakeRequests returns in order
# to actually make the requests.
responses = client.MakeRequests(requests)
return responses
def _Run(args, holder, url_map_arg):
"""Issues requests necessary to invalidate a URL map cdn cache."""
client = holder.client
requests = _CreateRequests(holder, args, url_map_arg)
if args.async:
resources, errors = batch_helper.MakeRequests(
requests=requests,
http=client.apitools_client.http,
batch_url=client.batch_url)
if not errors:
for invalidation_operation in resources:
log.status.write('Invalidation pending for [{0}]\n'.format(
invalidation_operation.targetLink))
log.status.write('Monitor its progress at [{0}]\n'.format(
invalidation_operation.selfLink))
else:
utils.RaiseToolException(errors)
else:
# We want to run through the generator that MakeRequests returns in order
# to actually make the requests.
resources = client.MakeRequests(requests)
return resources
def MakeRequests(requests, http, batch_url, errors, progress_tracker=None):
"""Makes one or more requests to the API.
Each request can be either a synchronous API call or an asynchronous
one. For synchronous calls (e.g., get and list), the result from the
server is yielded immediately. For asynchronous calls (e.g., calls
that return operations like insert), this function waits until the
operation reaches the DONE state and fetches the corresponding
object and yields that object (nothing is yielded for deletions).
Currently, a heterogenous set of synchronous calls can be made
(e.g., get request to fetch a disk and instance), however, the
asynchronous requests must be homogenous (e.g., they must all be the
same verb on the same collection). In the future, heterogenous
asynchronous requests will be supported. For now, it is up to the
client to ensure that the asynchronous requests are
homogenous. Synchronous and asynchronous requests can be mixed.
Args:
requests: A list of requests to make. Each element must be a 3-element
tuple where the first element is the service, the second element is
the string name of the method on the service, and the last element
is a protocol buffer representing the request.
http: An httplib2.Http-like object.
batch_url: The handler for making batch requests.
errors: A list for capturing errors. If any response contains an error,
it is added to this list.
progress_tracker: progress tracker to be ticked while waiting for operations
to finish.
Yields:
A response for each request. For deletion requests, no corresponding
responses are returned.
"""
if _RequestsAreListRequests(requests):
for item in _List(requests=requests,
http=http,
batch_url=batch_url,
errors=errors):
yield item
return
responses, new_errors = batch_helper.MakeRequests(requests=requests,
http=http,
batch_url=batch_url)
errors.extend(new_errors)
operation_service = None
resource_service = None
project = None
# Collects all operation objects in a list so they can be waited on
# and yields all non-operation objects since non-operation responses
# cannot be waited on.
operations_data = []
for request, response in zip(requests, responses):
if response is None:
continue
service, _, request_body = request
if (isinstance(response, service.client.MESSAGES_MODULE.Operation)
and service.__class__.__name__
not in ('GlobalOperationsService', 'RegionOperationsService',
'ZoneOperationsService',
'GlobalAccountsOperationsService')):
resource_service = service
project = request_body.project
if response.zone:
operation_service = service.client.zoneOperations
elif response.region:
operation_service = service.client.regionOperations
else:
operation_service = service.client.globalOperations
operations_data.append(
waiters.OperationData(response, project, operation_service,
resource_service))
else:
yield response
if operations_data:
warnings = []
for response in waiters.WaitForOperations(
operations_data=operations_data,
http=http,
batch_url=batch_url,
warnings=warnings,
progress_tracker=progress_tracker,
errors=errors):
yield response
if warnings:
log.warning(
utils.ConstructList('Some requests generated warnings:',
warnings))
def MakeRequests(requests, http, batch_url, errors):
"""Makes one or more requests to the API.
Each request can be either a synchronous API call or an asynchronous
one. For synchronous calls (e.g., get and list), the result from the
server is yielded immediately. For asynchronous calls (e.g., calls
that return operations like insert), this function waits until the
operation reaches the DONE state and fetches the corresponding
object and yields that object (nothing is yielded for deletions).
Currently, a heterogenous set of synchronous calls can be made
(e.g., get request to fetch a disk and instance), however, the
asynchronous requests must be homogenous (e.g., they must all be the
same verb on the same collection). In the future, heterogenous
asynchronous requests will be supported. For now, it is up to the
client to ensure that the asynchronous requests are
homogenous. Synchronous and asynchronous requests can be mixed.
Args:
requests: A list of requests to make. Each element must be a 3-element
tuple where the first element is the service, the second element is
the string name of the method on the service, and the last element
is a protocol buffer representing the request.
http: An httplib2.Http-like object.
batch_url: The handler for making batch requests.
errors: A list for capturing errors. If any response contains an error,
it is added to this list.
Yields:
A response for each request. For deletion requests, no corresponding
responses are returned.
"""
if _RequestsAreListRequests(requests):
for item in _List(requests=requests,
http=http,
batch_url=batch_url,
errors=errors):
yield item
return
# TODO(b/36057059): Delete the batch_helper module and move its logic
# here. To do this, we also have to edit the lister module to depend
# on this module instead of batch_helper.
responses, new_errors = batch_helper.MakeRequests(requests=requests,
http=http,
batch_url=batch_url)
errors.extend(new_errors)
operation_service = None
resource_service = None
project = None
# Collects all operation objects in a list so they can be waited on
# and yields all non-operation objects since non-operation responses
# cannot be waited on.
operations = []
for request, response in zip(requests, responses):
if response is None:
continue
service, _, request_body = request
if (isinstance(response, service.client.MESSAGES_MODULE.Operation)
and service.__class__.__name__
not in ('GlobalOperationsService', 'RegionOperationsService',
'ZoneOperationsService',
'GlobalAccountsOperationsService')):
operations.append(response)
# This logic assumes that all requests are homogenous, i.e.,
# they all make calls to the same service and verb. This is
# temporary. In the future, we will push this logic into the
# function that does the actual waiting. For now, we have to
# deal with existing interfaces to keep the scopes of the
# refactoring CLs small.
# TODO(b/32276307)
if not operation_service:
resource_service = service
project = request_body.project
if response.kind == 'clouduseraccounts#operation':
operation_service = service.client.globalAccountsOperations
elif response.zone:
operation_service = service.client.zoneOperations
elif response.region:
operation_service = service.client.regionOperations
else:
operation_service = service.client.globalOperations
else:
yield response
if operations:
warnings = []
for response in waiters.WaitForOperations(
operations=operations,
project=project,
operation_service=operation_service,
resource_service=resource_service,
http=http,
batch_url=batch_url,
warnings=warnings,
errors=errors):
yield response
if warnings:
log.warn(
utils.ConstructList('Some requests generated warnings:',
warnings))
def _List(requests, http, batch_url, errors):
"""Makes a series of list and/or aggregatedList batch requests.
Args:
requests: A list of requests to make. Each element must be a 3-element
tuple where the first element is the service, the second element is
the method ('List' or 'AggregatedList'), and the third element
is a protocol buffer representing either a list or aggregatedList
request.
http: An httplib2.Http-like object.
batch_url: The handler for making batch requests.
errors: A list for capturing errors. If any response contains an error,
it is added to this list.
Yields:
Resources encapsulated as protocol buffers as they are received
from the server.
"""
while requests:
responses, request_errors = batch_helper.MakeRequests(
requests=requests, http=http, batch_url=batch_url)
errors.extend(request_errors)
new_requests = []
for i, response in enumerate(responses):
if not response:
continue
service, method, request_protobuf = requests[i]
# If the request is a list call, then yield the items directly.
if method == 'List':
for item in response.items:
yield item
# If the request is an aggregatedList call, then do all the
# magic necessary to get the actual resources because the
# aggregatedList responses are very complicated data
# structures...
else:
items_field_name = service.GetMethodConfig(
'AggregatedList').relative_path.split('/')[-1]
for scope_result in response.items.additionalProperties:
# If the given scope is unreachable, record the warning
# message in the errors list.
warning = scope_result.value.warning
if (warning and warning.code
== warning.CodeValueValuesEnum.UNREACHABLE):
errors.append((None, warning.message))
items = getattr(scope_result.value, items_field_name)
for item in items:
yield item
next_page_token = response.nextPageToken
if next_page_token:
new_request_protobuf = copy.deepcopy(request_protobuf)
new_request_protobuf.pageToken = next_page_token
new_requests.append((service, method, new_request_protobuf))
requests = new_requests
def testBatchHelperMakeRequest(self):
properties.VALUES.core.should_prompt_to_enable_api.Set(True)
service_enablement = {'enabled': False}
service_enabled_mock = self.StartPatch(
'googlecloudsdk.api_lib.services.enable_api.'
'IsServiceEnabled')
def IsServiceEnabled(enable_project, enable_service):
self.assertEqual(enable_project, self.project)
self.assertEqual(enable_service, self.service)
return service_enablement['enabled']
service_enabled_mock.side_effect = IsServiceEnabled
enable_mock = self.StartPatch(
'googlecloudsdk.api_lib.services.enable_api.'
'EnableService')
def Enable(enable_project, enable_service):
self.assertEqual(enable_project, self.project)
self.assertEqual(enable_service, self.service)
service_enablement['enabled'] = True
enable_mock.side_effect = Enable
with mock.patch.object(http_wrapper, 'MakeRequest',
autospec=True) as mock_request:
# pylint: disable=unused-argument
def Perform(http, request):
# We ignore the inputs and just return what we expect
if service_enablement['enabled']:
return http_wrapper.Response(
{
'status': '200',
'content-type':
'multipart/mixed; boundary="boundary"',
},
textwrap.dedent("""\
--boundary
content-type: application/json; charset=UTF-8
content-id: <id+0>
HTTP/1.1 200 OK
{}
--boundary--"""), None)
else:
return http_wrapper.Response(
{
'status': '200',
'content-type':
'multipart/mixed; boundary="boundary"',
},
textwrap.dedent("""\
--boundary
content-type: application/json; charset=UTF-8
content-id: <id+0>
HTTP/1.1 403 Forbidden
{{
"error": {{
"errors": [
{{
"domain": "usageLimits",
"reason": "accessNotConfigured",
"message": "{message}",
"extendedHelp": ""
}}
],
"code": 403,
"message": "{message}"
}}
}}
--boundary--""".format(message=self.message)), None)
mock_request.side_effect = Perform
self.WriteInput('y')
batch_helper.MakeRequests(
requests=[(self.compute_v1.instances, 'Get',
self.messages.ComputeInstancesGetRequest(
instance='my-instance-1',
project='my-project',
zone='my-zone'))],
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute')
# This ensures that it was called twice (once for failure, once for
# success)
self.assertTrue(service_enablement['enabled'])
self.assertEqual(2, mock_request.call_count)
enable_mock.assert_called_once_with(self.project, self.service)
def WaitForOperations(operations_data,
http,
batch_url,
warnings,
errors,
progress_tracker=None,
timeout=None,
log_result=True):
"""Blocks until the given operations are done or until a timeout is reached.
Args:
operations_data: A list of OperationData objects holding Operations to poll.
http: An HTTP object.
batch_url: The URL to which batch requests should be sent.
warnings: An output parameter for capturing warnings.
errors: An output parameter for capturing errors.
progress_tracker: progress tracker to tick while waiting for operations to
finish.
timeout: The maximum amount of time, in seconds, to wait for the
operations to reach the DONE state.
log_result: Whether the Operation Waiter should print the result in past
tense of each request.
Yields:
The resources pointed to by the operations' targetLink fields if
the operation type is not delete. Only resources whose
corresponding operations reach done are yielded.
"""
if not operations_data:
return
timeout = timeout or _POLLING_TIMEOUT_SEC
# Operation -> OperationData mapping will be used to reify operation_service
# and resource_service from operation_service.Get(operation) response.
# It is necessary because poll operation is returning only response, but we
# also need to get operation details to know the service to poll for all
# unprocessed_operations.
operation_details = {}
unprocessed_operations = []
for operation in operations_data:
operation_details[operation.operation.selfLink] = operation
unprocessed_operations.append(operation.operation)
start = time_util.CurrentTimeSec()
sleep_sec = 0
# There is only one type of operation in compute API.
# We pick the type of the first operation in the list.
operation_type = operations_data[0].operation_service.GetResponseType(
'Get')
while unprocessed_operations:
if progress_tracker:
progress_tracker.Tick()
resource_requests = []
operation_requests = []
log.debug('Operations to inspect: %s', unprocessed_operations)
for operation in unprocessed_operations:
# Reify operation
data = operation_details[operation.selfLink]
# Need to update the operation since old operation may not have all the
# required information.
data.SetOperation(operation)
operation_service = data.operation_service
resource_service = data.resource_service
if operation.status == operation_type.StatusValueValuesEnum.DONE:
# The operation has reached the DONE state, so we record any
# problems it contains (if any) and proceed to get the target
# resource if there were no problems and the operation is not
# a deletion.
_RecordProblems(operation, warnings, errors)
# We shouldn't attempt to get the target resource if there was
# anything wrong with the operation. Note that
# httpErrorStatusCode is set only when the operation is not
# successful.
if (operation.httpErrorStatusCode and
operation.httpErrorStatusCode != 200): # httplib.OK
continue
# Just in case the server did not set httpErrorStatusCode but
# the operation did fail, we check the "error" field.
if operation.error:
continue
# We shouldn't get the target resource if the operation type
# is delete because there will be no resource left.
if not _IsDeleteOp(
operation.operationType) and not data.no_followup:
request = data.ResourceGetRequest()
# Some operations do not have target and should not send get request.
if request:
resource_requests.append(
(resource_service, 'Get', request))
# Only log when there is target link in the operation.
if operation.targetLink and log_result:
log.status.write('{0} [{1}].\n'.format(
_HumanFriendlyNameForOpPastTense(
operation.operationType).capitalize(),
operation.targetLink))
else:
# The operation has not reached the DONE state, so we add a request
# to poll the operation.
# TODO(b/129413862): Global org operation service supports wait API.
if data.IsGlobalOrganizationOperation():
request = data.OperationGetRequest()
operation_requests.append(
(operation_service, 'Get', request))
else:
request = data.OperationWaitRequest()
operation_requests.append(
(operation_service, 'Wait', request))
requests = resource_requests + operation_requests
if not requests:
break
responses, request_errors = batch_helper.MakeRequests(
requests=requests, http=http, batch_url=batch_url)
errors.extend(request_errors)
all_done = True
unprocessed_operations = []
for response in responses:
if isinstance(response, operation_type):
unprocessed_operations.append(response)
if response.status != operation_type.StatusValueValuesEnum.DONE:
all_done = False
else:
yield response
# If there are no more operations, we are done.
if not unprocessed_operations:
break
# If all of the operations are done, we should ignore the timeout and ignore
# the sleep.
if all_done:
continue
# Did we time out? If so, record the operations that timed out so
# they can be reported to the user.
if time_util.CurrentTimeSec() - start > timeout:
log.debug('Timeout of %ss reached.', timeout)
_RecordUnfinishedOperations(unprocessed_operations, errors)
break
# Sleeps before trying to poll the operations again.
sleep_sec = min(sleep_sec + 1, _MAX_TIME_BETWEEN_POLLS_SEC)
log.debug('Sleeping for %ss.', sleep_sec)
time_util.Sleep(sleep_sec)
def WaitForOperations(
operations_data, http, batch_url, warnings, errors,
progress_tracker=None, timeout=None):
"""Blocks until the given operations are done or until a timeout is reached.
Args:
operations_data: A list of OperationData objects holding Operations to poll.
http: An HTTP object.
batch_url: The URL to which batch requests should be sent.
warnings: An output parameter for capturing warnings.
errors: An output parameter for capturing errors.
progress_tracker: progress tracker to tick while waiting for operations to
finish.
timeout: The maximum amount of time, in seconds, to wait for the
operations to reach the DONE state.
Yields:
The resources pointed to by the operations' targetLink fields if
the operation type is not delete. Only resources whose
corresponding operations reach done are yielded.
"""
timeout = timeout or _POLLING_TIMEOUT_SEC
# Operation -> OperationData mapping will be used to reify operation_service
# and resource_service from operation_service.Get(operation) response.
# It is necessary because poll operation is returning only response, but we
# also need to get operation details to know the service to poll for all
# unfinished_operations.
operation_details = {}
unfinished_operations = []
for operation in operations_data:
operation_details[operation.operation.selfLink] = operation
unfinished_operations.append(operation.operation)
responses = []
start = time_util.CurrentTimeSec()
sleep_sec = 0
while unfinished_operations:
if progress_tracker:
progress_tracker.Tick()
resource_requests = []
operation_requests = []
log.debug('Operations to inspect: %s', unfinished_operations)
for operation in unfinished_operations:
# Reify operation
data = operation_details[operation.selfLink]
project = data.project
operation_service = data.operation_service
resource_service = data.resource_service
operation_type = operation_service.GetResponseType('Get')
if operation.status == operation_type.StatusValueValuesEnum.DONE:
# The operation has reached the DONE state, so we record any
# problems it contains (if any) and proceed to get the target
# resource if there were no problems and the operation is not
# a deletion.
_RecordProblems(operation, warnings, errors)
# We shouldn't attempt to get the target resource if there was
# anything wrong with the operation. Note that
# httpErrorStatusCode is set only when the operation is not
# successful.
if (operation.httpErrorStatusCode and
operation.httpErrorStatusCode != 200): # httplib.OK
continue
# Just in case the server did not set httpErrorStatusCode but
# the operation did fail, we check the "error" field.
if operation.error:
continue
target_link = operation.targetLink
# We shouldn't get the target resource if the operation type
# is delete because there will be no resource left.
if not _IsDeleteOp(operation.operationType):
request = resource_service.GetRequestType('Get')(project=project)
if operation.zone:
request.zone = path_simplifier.Name(operation.zone)
elif operation.region:
request.region = path_simplifier.Name(operation.region)
name_field = resource_service.GetMethodConfig(
'Get').ordered_params[-1]
setattr(request, name_field,
path_simplifier.Name(operation.targetLink))
resource_requests.append((resource_service, 'Get', request))
log.status.write('{0} [{1}].\n'.format(
_HumanFrieldlyNameForOpPastTense(
operation.operationType).capitalize(),
target_link))
else:
# The operation has not reached the DONE state, so we add a
# get request to poll the operation.
request = operation_service.GetRequestType('Get')(
operation=operation.name,
project=project)
if operation.zone:
request.zone = path_simplifier.Name(operation.zone)
elif operation.region:
request.region = path_simplifier.Name(operation.region)
operation_requests.append((operation_service, 'Get', request))
requests = resource_requests + operation_requests
if not requests:
break
responses, request_errors = batch_helper.MakeRequests(
requests=requests,
http=http,
batch_url=batch_url)
errors.extend(request_errors)
unfinished_operations = []
for response in responses:
if isinstance(response, operation_type):
unfinished_operations.append(response)
else:
yield response
# If there are no more operations, we are done.
if not unfinished_operations:
break
# Did we time out? If so, record the operations that timed out so
# they can be reported to the user.
if time_util.CurrentTimeSec() - start > timeout:
log.debug('Timeout of %ss reached.', timeout)
_RecordUnfinishedOperations(unfinished_operations, errors)
break
# Sleeps before trying to poll the operations again.
sleep_sec += 1
# Don't re-use sleep_sec, since we want to keep the same time increment
sleep_time = min(sleep_sec, _MAX_TIME_BETWEEN_POLLS_SEC)
log.debug('Sleeping for %ss.', sleep_time)
time_util.Sleep(sleep_time)
def WaitForOperations(operations,
project,
operation_service,
resource_service,
http,
batch_url,
warnings,
errors,
custom_get_requests=None,
timeout=None):
"""Blocks until the given operations are done or until a timeout is reached.
Args:
operations: A list of Operation objects to poll.
project: The project to which the resources belog.
operation_service: The service that can be used to get operation
objects.
resource_service: The service of the collection being mutated by
the operations. If the operation type is not delete, this service
is used to fetch the mutated objects after the operations are done.
http: An HTTP object.
batch_url: The URL to which batch requests should be sent.
warnings: An output parameter for capturing warnings.
errors: An output parameter for capturing errors.
custom_get_requests: A mapping of resource names to requests. If
this is provided, when an operation is DONE, instead of performing
a get on the targetLink, this function will consult custom_get_requests
and perform the request dictated by custom_get_requests.
timeout: The maximum amount of time, in seconds, to wait for the
operations to reach the DONE state.
Yields:
The resources pointed to by the operations' targetLink fields if
the operation type is not delete. Only resources whose
corresponding operations reach done are yielded.
"""
timeout = timeout or _POLLING_TIMEOUT_SEC
operation_type = operation_service.GetResponseType('Get')
responses = []
start = time_util.CurrentTimeSec()
sleep_sec = 0
while operations:
resource_requests = []
operation_requests = []
log.debug('Operations to inspect: %s', operations)
for operation in operations:
if operation.status == operation_type.StatusValueValuesEnum.DONE:
# The operation has reached the DONE state, so we record any
# problems it contains (if any) and proceed to get the target
# resource if there were no problems and the operation is not
# a deletion.
_RecordProblems(operation, warnings, errors)
# We shouldn't attempt to get the target resource if there was
# anything wrong with the operation. Note that
# httpErrorStatusCode is set only when the operation is not
# successful.
if (operation.httpErrorStatusCode
and operation.httpErrorStatusCode != httplib.OK):
continue
# Just in case the server did not set httpErrorStatusCode but
# the operation did fail, we check the "error" field.
if operation.error:
continue
target_link = operation.targetLink
if custom_get_requests:
target_link, service, request_protobuf = (
custom_get_requests[operation.targetLink])
resource_requests.append(
(service, 'Get', request_protobuf))
# We shouldn't get the target resource if the operation type
# is delete because there will be no resource left.
elif not _IsDeleteOp(operation.operationType):
request = resource_service.GetRequestType('Get')(
project=project)
if operation.zone:
request.zone = path_simplifier.Name(operation.zone)
elif operation.region:
request.region = path_simplifier.Name(operation.region)
name_field = resource_service.GetMethodConfig(
'Get').ordered_params[-1]
setattr(request, name_field,
path_simplifier.Name(operation.targetLink))
resource_requests.append(
(resource_service, 'Get', request))
log.status.write('{0} [{1}].\n'.format(
_HumanFrieldlyNameForOpPastTense(
operation.operationType).capitalize(), target_link))
else:
# The operation has not reached the DONE state, so we add a
# get request to poll the operation.
request = operation_service.GetRequestType('Get')(
operation=operation.name, project=project)
if operation.zone:
request.zone = path_simplifier.Name(operation.zone)
elif operation.region:
request.region = path_simplifier.Name(operation.region)
operation_requests.append((operation_service, 'Get', request))
requests = resource_requests + operation_requests
if not requests:
break
responses, request_errors = batch_helper.MakeRequests(
requests=requests, http=http, batch_url=batch_url)
errors.extend(request_errors)
operations = []
for response in responses:
if isinstance(response, operation_type):
operations.append(response)
else:
yield response
# If there are no more operations, we are done.
if not operations:
break
# Did we time out? If so, record the operations that timed out so
# they can be reported to the user.
if time_util.CurrentTimeSec() - start > timeout:
log.debug('Timeout of %ss reached.', timeout)
_RecordUnfinishedOperations(operations, errors)
break
# Sleeps before trying to poll the operations again.
sleep_sec += 1
# Don't re-use sleep_sec, since we want to keep the same time increment
sleep_time = min(sleep_sec, _MAX_TIME_BETWEEN_POLLS_SEC)
log.debug('Sleeping for %ss.', sleep_time)
time_util.Sleep(sleep_time)
def MakeRequests(requests,
http,
batch_url,
errors,
progress_tracker=None,
no_followup=False,
always_return_operation=False,
followup_overrides=None,
log_result=True,
log_warnings=True,
timeout=None):
"""Makes one or more requests to the API.
Each request can be either a synchronous API call or an asynchronous
one. For synchronous calls (e.g., get and list), the result from the
server is yielded immediately. For asynchronous calls (e.g., calls
that return operations like insert), this function waits until the
operation reaches the DONE state and fetches the corresponding
object and yields that object (nothing is yielded for deletions).
Currently, a heterogenous set of synchronous calls can be made
(e.g., get request to fetch a disk and instance), however, the
asynchronous requests must be homogenous (e.g., they must all be the
same verb on the same collection). In the future, heterogenous
asynchronous requests will be supported. For now, it is up to the
client to ensure that the asynchronous requests are
homogenous. Synchronous and asynchronous requests can be mixed.
Args:
requests: A list of requests to make. Each element must be a 3-element tuple
where the first element is the service, the second element is the string
name of the method on the service, and the last element is a protocol
buffer representing the request.
http: An httplib2.Http-like object.
batch_url: The handler for making batch requests.
errors: A list for capturing errors. If any response contains an error, it
is added to this list.
progress_tracker: progress tracker to be ticked while waiting for operations
to finish.
no_followup: If True, do not followup operation with a GET request.
always_return_operation: If True, return operation object even if operation
fails.
followup_overrides: A list of new resource names to GET once the operation
finishes. Generally used in renaming calls.
log_result: Whether the Operation Waiter should print the result in past
tense of each request.
log_warnings: Whether warnings for completed operation should be printed.
timeout: The maximum amount of time, in seconds, to wait for the
operations to reach the DONE state.
Yields:
A response for each request. For deletion requests, no corresponding
responses are returned.
"""
if _RequestsAreListRequests(requests):
for item in _List(requests=requests,
http=http,
batch_url=batch_url,
errors=errors):
yield item
return
responses, new_errors = batch_helper.MakeRequests(requests=requests,
http=http,
batch_url=batch_url)
errors.extend(new_errors)
operation_service = None
resource_service = None
# Collects all operation objects in a list so they can be waited on
# and yields all non-operation objects since non-operation responses
# cannot be waited on.
operations_data = []
if not followup_overrides:
followup_overrides = [None for _ in requests]
for request, response, followup_override in zip(requests, responses,
followup_overrides):
if response is None:
continue
service, _, request_body = request
if (isinstance(response, service.client.MESSAGES_MODULE.Operation)
and not _IsEmptyOperation(response, service)
and service.__class__.__name__
not in ('GlobalOperationsService', 'RegionOperationsService',
'ZoneOperationsService',
'GlobalOrganizationOperationsService',
'GlobalAccountsOperationsService')):
resource_service = service
project = None
if hasattr(request_body, 'project'):
project = request_body.project
if response.zone:
operation_service = service.client.zoneOperations
elif response.region:
operation_service = service.client.regionOperations
else:
operation_service = service.client.globalOperations
else:
operation_service = service.client.globalOrganizationOperations
operations_data.append(
waiters.OperationData(
response,
operation_service,
resource_service,
project=project,
no_followup=no_followup,
followup_override=followup_override,
always_return_operation=always_return_operation))
else:
yield response
if operations_data:
warnings = []
for response in waiters.WaitForOperations(
operations_data=operations_data,
http=http,
batch_url=batch_url,
warnings=warnings,
progress_tracker=progress_tracker,
errors=errors,
log_result=log_result,
timeout=timeout):
yield response
if warnings and log_warnings:
log.warning(
utils.ConstructList('Some requests generated warnings:',
warnings))
