def testListingWithSingleRequestAndNoPaging(self):
items = [
self.messages.Operation(name='operation-1'),
self.messages.Operation(name='operation-2'),
self.messages.Operation(name='operation-3'),
]
self.batch_helper.side_effect = [
[[
encoding.MessageToJson(
self.messages.OperationList(items=items))
], []],
]
requests = [
(self.compute_v1.globalOperations, 'List',
self.messages.ComputeGlobalOperationsListRequest(
project='my-project')),
]
errors = []
res = request_helper.ListJson(
requests=requests,
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute',
errors=errors)
self.assertEqual(list(res), resource_projector.MakeSerializable(items))
self.assertFalse(errors)
self.batch_helper.assert_called_once_with(
requests=requests,
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute')
def __call__(self, frontend):
scope_set = frontend.scope_set
requests = []
if isinstance(scope_set, ZoneSet):
for project, zones in six.iteritems(
_GroupByProject(sorted(list(scope_set)))):
for zone_ref in zones:
requests.append((self.zonal_service, 'List',
self.zonal_service.GetRequestType('List')(
filter=frontend.filter,
maxResults=frontend.max_results,
project=project,
zone=zone_ref.zone)))
elif isinstance(scope_set, RegionSet):
for project, regions in six.iteritems(
_GroupByProject(sorted(list(scope_set)))):
for region_ref in regions:
requests.append((self.regional_service, 'List',
self.regional_service.GetRequestType('List')(
filter=frontend.filter,
maxResults=frontend.max_results,
project=project,
region=region_ref.region)))
elif isinstance(scope_set, GlobalScope):
for project_ref in sorted(list(scope_set)):
requests.append((self.global_service, 'List',
self.global_service.GetRequestType('List')(
filter=frontend.filter,
maxResults=frontend.max_results,
project=project_ref.project)))
else:
# scopeSet is AllScopes
# generate AggregatedList
for project_ref in sorted(list(scope_set.projects)):
requests.append(
(self.aggregation_service, 'AggregatedList',
self.aggregation_service.GetRequestType('AggregatedList')(
filter=frontend.filter,
maxResults=frontend.max_results,
project=project_ref.project)))
errors = []
response_count = 0
for item in request_helper.ListJson(
requests=requests,
http=self.client.apitools_client.http,
batch_url=self.client.batch_url,
errors=errors):
response_count += 1
yield item
if errors:
# If the command allows partial server errors, instead of raising an
# exception to show something went wrong, we show a warning message that
# contains the error messages instead.
if self.allow_partial_server_failure and response_count > 0:
utils.WarnIfPartialRequestFail(errors)
else:
utils.RaiseException(errors, ListException)
def GetResources(self, args, errors):
"""Yields images from (potentially) multiple projects."""
filter_expr = self.GetFilterExpr(args)
image_projects = [self.project]
if args.standard_images:
image_projects.extend(constants.PUBLIC_IMAGE_PROJECTS)
if args.preview_images:
image_projects.extend(constants.PREVIEW_IMAGE_PROJECTS)
requests = []
for project in image_projects:
requests.append((self.service, 'List',
self.messages.ComputeImagesListRequest(
filter=filter_expr,
maxResults=constants.MAX_RESULTS_PER_PAGE,
project=project)))
images = request_helper.ListJson(requests=requests,
http=self.http,
batch_url=self.batch_url,
errors=errors)
for image in images:
if not image.get('deprecated', False) or args.show_deprecated:
yield image
def testWithAggregatedListRequestAndNoPaging(self):
items = [
self.messages.Instance(name='instance-1'),
self.messages.Instance(name='instance-2'),
self.messages.Instance(name='instance-3'),
]
self.batch_helper.side_effect = [
[[
encoding.MessageToJson(
self.messages.InstanceAggregatedList(
items=self.messages.InstanceAggregatedList.ItemsValue(
additionalProperties=[
_KV(key='zones/zone-1',
value=self.messages.InstancesScopedList(
instances=[
self.messages.Instance(
name='instance-1'),
self.messages.Instance(
name='instance-2'),
])),
_KV(key='zones/zone-2',
value=self.messages.InstancesScopedList(
instances=[
self.messages.Instance(
name='instance-3'),
])),
])))
], []],
]
requests = [
(self.compute_v1.instances, 'AggregatedList',
self.messages.ComputeInstancesAggregatedListRequest(
project='my-project')),
]
errors = []
res = request_helper.ListJson(
requests=requests,
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute',
errors=errors)
self.assertEqual(set(instance['name'] for instance in res),
set(instance.name for instance in items))
self.assertFalse(errors)
self.batch_helper.assert_called_once_with(
requests=[(self.compute_v1.instances, 'AggregatedList',
self.messages.ComputeInstancesAggregatedListRequest(
project='my-project'))],
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute')
def __call__(self, frontend):
scope_set = frontend.scope_set
requests = []
if isinstance(scope_set, ZoneSet):
for project, zones in _GroupByProject(sorted(
list(scope_set))).iteritems():
for zone_ref in zones:
requests.append((self.zonal_service, 'List',
self.zonal_service.GetRequestType('List')(
filter=frontend.filter,
maxResults=frontend.max_results,
project=project,
zone=zone_ref.zone)))
elif isinstance(scope_set, RegionSet):
for project, regions in _GroupByProject(sorted(
list(scope_set))).iteritems():
for region_ref in regions:
requests.append(
(self.regional_service, 'List',
self.regional_service.GetRequestType('List')(
filter=frontend.filter,
maxResults=frontend.max_results,
project=project,
region=region_ref.region)))
elif isinstance(scope_set, GlobalScope):
for project_ref in sorted(list(scope_set)):
requests.append((self.global_service, 'List',
self.global_service.GetRequestType('List')(
filter=frontend.filter,
maxResults=frontend.max_results,
project=project_ref.project)))
else:
# scopeSet is AllScopes
# generate AggregatedList
for project_ref in sorted(list(scope_set.projects)):
requests.append(
(self.aggregation_service, 'AggregatedList',
self.aggregation_service.GetRequestType('AggregatedList')(
filter=frontend.filter,
maxResults=frontend.max_results,
project=project_ref.project)))
errors = []
for item in request_helper.ListJson(
requests=requests,
http=self.client.apitools_client.http,
batch_url=self.client.batch_url,
errors=errors):
yield item
if errors:
utils.RaiseException(errors, ListException)
def testWithAggregatedListRequestWithUnreachableScope(self):
items = [
self.messages.Instance(name='instance-3'),
]
warning = self.messages.InstancesScopedList.WarningValue(
code=(self.messages.InstancesScopedList.WarningValue.
CodeValueValuesEnum.UNREACHABLE),
message='Scope [zones/zone-1] is unreachable.',
)
self.batch_helper.side_effect = [
[[
encoding.MessageToJson(
self.messages.InstanceAggregatedList(
items=self.messages.InstanceAggregatedList.ItemsValue(
additionalProperties=[
_KV(key='zones/zone-1',
value=self.messages.InstancesScopedList(
warning=warning)),
_KV(key='zones/zone-2',
value=self.messages.InstancesScopedList(
instances=[
self.messages.Instance(
name='instance-3'),
])),
])))
], []],
]
requests = [
(self.compute_v1.instances, 'AggregatedList',
self.messages.ComputeInstancesAggregatedListRequest(
project='my-project')),
]
errors = []
res = request_helper.ListJson(
requests=requests,
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute',
errors=errors)
self.assertEqual(list(res), resource_projector.MakeSerializable(items))
self.assertEqual(errors,
[(None, 'Scope [zones/zone-1] is unreachable.')])
self.batch_helper.assert_called_once_with(
requests=[(self.compute_v1.instances, 'AggregatedList',
self.messages.ComputeInstancesAggregatedListRequest(
project='my-project'))],
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute')
def GetResources(self, args, errors):
"""Returns zonal, regional and/or global resources.
Args:
args: argparse.Namespace, Parsed arguments
errors: list, Errors will be returned in this list
Returns:
Zonal, regional and/or global resources.
"""
regions = getattr(args, 'regions', None)
zones = getattr(args, 'zones', None)
global_arg = getattr(args, 'global', None)
# This is true if the user provided no flags indicating scope
no_scope_flags = not regions and not zones and not global_arg
requests = []
filter_expr = self.GetFilterExpr(args)
if args.page_size is not None:
max_results = min(args.page_size, constants.MAX_RESULTS_PER_PAGE)
else:
max_results = constants.MAX_RESULTS_PER_PAGE
project = self.project
# If --regions is present with no arguments OR no scope flags are present
# then we have to do an aggregated list
# pylint:disable=g-explicit-bool-comparison
if no_scope_flags and self.aggregation_service:
requests.append(
(self.aggregation_service,
'AggregatedList',
self.aggregation_service.GetRequestType('AggregatedList')(
filter=filter_expr,
maxResults=max_results,
project=project)))
# Else if some regions were provided then only list within them
elif regions is not None:
region_names = set(
self.CreateGlobalReference(region, resource_type='regions').Name()
for region in regions)
for region_name in sorted(region_names):
requests.append(
(self.regional_service,
'List',
self.regional_service.GetRequestType('List')(
filter=filter_expr,
maxResults=max_results,
region=region_name,
project=project)))
# Else if some regions were provided then only list within them
elif zones is not None:
zone_names = set(
self.CreateGlobalReference(zone, resource_type='zones').Name()
for zone in zones)
for zone_name in sorted(zone_names):
requests.append(
(self.zonal_service,
'List',
self.zonal_service.GetRequestType('List')(
filter=filter_expr,
maxResults=max_results,
zone=zone_name,
project=project)))
else:
# Either --global was specified or we do not have aggregation service.
# Note that --global, --region and --zone are mutually exclusive.
requests.append(
(self.global_service,
'List',
self.global_service.GetRequestType('List')(
filter=filter_expr,
maxResults=max_results,
project=project)))
return request_helper.ListJson(
requests=requests,
http=self.http,
batch_url=self.batch_url,
errors=errors)
def testWithManyListRequestsAndPaging(self):
zonal_page_1 = [
self.messages.Operation(name='operation-1'),
self.messages.Operation(name='operation-2'),
]
zonal_page_2 = [
self.messages.Operation(name='operation-3'),
self.messages.Operation(name='operation-4'),
self.messages.Operation(name='operation-5'),
]
global_page_1 = [
self.messages.Operation(name='operation-6'),
self.messages.Operation(name='operation-7'),
]
global_page_2 = [
self.messages.Operation(name='operation-8'),
self.messages.Operation(name='operation-9'),
]
self.batch_helper.side_effect = [
[[
encoding.MessageToJson(
self.messages.OperationList(items=zonal_page_1,
nextPageToken='page-2-a')),
encoding.MessageToJson(
self.messages.OperationList(items=global_page_1,
nextPageToken='page-2-b'))
], []],
[[
encoding.MessageToJson(
self.messages.OperationList(items=zonal_page_2)),
encoding.MessageToJson(
self.messages.OperationList(items=global_page_2)),
], []],
]
requests = [
(self.compute_v1.zoneOperations, 'List',
self.messages.ComputeZoneOperationsListRequest(
zone='zone-1', project='my-project')),
(self.compute_v1.globalOperations, 'List',
self.messages.ComputeGlobalOperationsListRequest(
project='my-project')),
]
errors = []
res = request_helper.ListJson(
requests=requests,
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute',
errors=errors)
# Protocol messages in protorpc are mutable, so they cannot be
# hashed which means they cannot be stored in a set. To get around
# this, we create sets using just the name attribute of the
# protocol buffers before doing the equality comparison.
self.assertEqual(
set(operation['name'] for operation in res),
set(operation.name for operation in zonal_page_1 + zonal_page_2 +
global_page_1 + global_page_2))
self.assertFalse(errors)
self.assertEqual(self.batch_helper.call_args_list, [
mock.call(
requests=[(self.compute_v1.zoneOperations, 'List',
self.messages.ComputeZoneOperationsListRequest(
zone='zone-1', project='my-project')),
(self.compute_v1.globalOperations, 'List',
self.messages.ComputeGlobalOperationsListRequest(
project='my-project'))],
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute'),
mock.call(requests=[
(self.compute_v1.zoneOperations, 'List',
self.messages.ComputeZoneOperationsListRequest(
zone='zone-1', pageToken='page-2-a',
project='my-project')),
(self.compute_v1.globalOperations, 'List',
self.messages.ComputeGlobalOperationsListRequest(
pageToken='page-2-b', project='my-project'))
],
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute'),
])
def testListingWithSingleListRequestAndPaging(self):
page_1 = [
self.messages.Operation(name='operation-1'),
self.messages.Operation(name='operation-2'),
self.messages.Operation(name='operation-3'),
]
page_2 = [
self.messages.Operation(name='operation-4'),
self.messages.Operation(name='operation-5'),
self.messages.Operation(name='operation-6'),
]
page_3 = [
self.messages.Operation(name='operation-7'),
self.messages.Operation(name='operation-8'),
self.messages.Operation(name='operation-9'),
]
self.batch_helper.side_effect = [
[[
encoding.MessageToJson(
self.messages.OperationList(items=page_1,
nextPageToken='page-2'))
], []],
[[
encoding.MessageToJson(
self.messages.OperationList(items=page_2,
nextPageToken='page-3'))
], []],
[[
encoding.MessageToJson(
self.messages.OperationList(items=page_3))
], []],
]
requests = [
(self.compute_v1.globalOperations, 'List',
self.messages.ComputeGlobalOperationsListRequest(
project='my-project')),
]
errors = []
res = request_helper.ListJson(
requests=requests,
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute',
errors=errors)
self.assertEqual(
list(res),
resource_projector.MakeSerializable(page_1 + page_2 + page_3))
self.assertFalse(errors)
self.assertEqual(self.batch_helper.call_args_list, [
mock.call(
requests=[(self.compute_v1.globalOperations, 'List',
self.messages.ComputeGlobalOperationsListRequest(
project='my-project'))],
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute'),
mock.call(
requests=[(self.compute_v1.globalOperations, 'List',
self.messages.ComputeGlobalOperationsListRequest(
pageToken='page-2', project='my-project'))],
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute'),
mock.call(
requests=[(self.compute_v1.globalOperations, 'List',
self.messages.ComputeGlobalOperationsListRequest(
pageToken='page-3', project='my-project'))],
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute'),
])
def testWithAggregatedListRequestAndPaging(self):
page_1 = [
self.messages.Instance(name='instance-1'),
self.messages.Instance(name='instance-2'),
self.messages.Instance(name='instance-3'),
]
page_2 = [
self.messages.Instance(name='instance-4'),
self.messages.Instance(name='instance-5'),
self.messages.Instance(name='instance-6'),
]
self.batch_helper.side_effect = [
[[
encoding.MessageToJson(
self.messages.InstanceAggregatedList(
nextPageToken='page-2',
items=self.messages.InstanceAggregatedList.ItemsValue(
additionalProperties=[
_KV(key='zones/zone-1',
value=self.messages.InstancesScopedList(
instances=page_1, )),
])))
], []],
[[
encoding.MessageToJson(
self.messages.InstanceAggregatedList(
items=self.messages.InstanceAggregatedList.ItemsValue(
additionalProperties=[
_KV(key='zones/zone-1',
value=self.messages.InstancesScopedList(
instances=page_2, )),
])))
], []],
]
requests = [
(self.compute_v1.instances, 'AggregatedList',
self.messages.ComputeInstancesAggregatedListRequest(
project='my-project')),
]
errors = []
res = request_helper.ListJson(
requests=requests,
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute',
errors=errors)
self.assertEqual(list(res),
resource_projector.MakeSerializable(page_1 + page_2))
self.assertFalse(errors)
self.assertEqual(self.batch_helper.call_args_list, [
mock.call(
requests=[(self.compute_v1.instances, 'AggregatedList',
self.messages.ComputeInstancesAggregatedListRequest(
project='my-project'))],
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute'),
mock.call(
requests=[(self.compute_v1.instances, 'AggregatedList',
self.messages.ComputeInstancesAggregatedListRequest(
pageToken='page-2', project='my-project'))],
http=self.mock_http,
batch_url='https://www.googleapis.com/batch/compute'),
])
def GetResources(self, args, errors):
"""Yields zonal, regional, and/or global resources."""
# This is True if the user provided no flags indicating scope.
no_scope_flags = self.NoArguments(args)
requests = []
filter_expr = self.GetFilterExpr(args)
max_results = constants.MAX_RESULTS_PER_PAGE
project = self.project
# TODO(b/36050874): Start using aggregatedList for zones and regions when
# the operations list API supports them.
if no_scope_flags:
requests.append(
(self.global_service,
'AggregatedList',
self.global_service.GetRequestType('AggregatedList')(
filter=filter_expr,
maxResults=max_results,
project=project)))
if not self._ga:
# Add a request to get all Compute Account operations.
requests.append(
(self.account_service,
'List',
self.account_service.GetRequestType('List')(
filter=filter_expr,
maxResults=max_results,
project=project)))
else:
if getattr(args, 'global'):
requests.append(
(self.global_service,
'List',
self.global_service.GetRequestType('List')(
filter=filter_expr,
maxResults=max_results,
project=project)))
if args.regions is not None:
args_region_names = [
self.resources.Parse(
region,
params={'project': properties.VALUES.core.project.GetOrFail},
collection='compute.regions').Name()
for region in args.regions or []]
# If no regions were provided by the user, fetch a list.
region_names = (
args_region_names or [res.name for res in self.FetchChoiceResources(
attribute='region',
service=self.compute.regions,
flag_names=['--regions'])])
for region_name in region_names:
requests.append(
(self.regional_service,
'List',
self.regional_service.GetRequestType('List')(
filter=filter_expr,
maxResults=constants.MAX_RESULTS_PER_PAGE,
region=region_name,
project=self.project)))
if args.zones is not None:
args_zone_names = [
self.resources.Parse(
zone,
params={
'project': properties.VALUES.core.project.GetOrFail,
},
collection='compute.zones').Name()
for zone in args.zones or []]
# If no zones were provided by the user, fetch a list.
zone_names = (
args_zone_names or [res.name for res in self.FetchChoiceResources(
attribute='zone',
service=self.compute.zones,
flag_names=['--zones'])])
for zone_name in zone_names:
requests.append(
(self.zonal_service,
'List',
self.zonal_service.GetRequestType('List')(
filter=filter_expr,
maxResults=constants.MAX_RESULTS_PER_PAGE,
zone=zone_name,
project=self.project)))
if not self._ga and args.accounts:
requests.append(
(self.account_service,
'List',
self.account_service.GetRequestType('List')(
filter=filter_expr,
maxResults=max_results,
project=project)))
return request_helper.ListJson(
requests=requests,
http=self.http,
batch_url=self.batch_url,
errors=errors)
def Run(self, args):
"""Yields zonal, regional, and/or global resources."""
compute_holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
compute_client = compute_holder.client
# This is True if the user provided no flags indicating scope.
no_scope_flags = self.NoArguments(args)
requests = []
request_data = lister.ParseNamesAndRegexpFlags(args,
compute_holder.resources)
# TODO(b/36050874): Start using aggregatedList for zones and regions when
# the operations list API supports them.
if no_scope_flags:
requests.append(
(compute_client.apitools_client.globalOperations, 'AggregatedList',
compute_client.apitools_client.globalOperations.GetRequestType(
'AggregatedList')(
filter=request_data.filter,
maxResults=request_data.max_results,
project=list(request_data.scope_set)[0].project)))
else:
if getattr(args, 'global'):
requests.append(
(compute_client.apitools_client.globalOperations, 'List',
compute_client.apitools_client.globalOperations.GetRequestType(
'List')(
filter=request_data.filter,
maxResults=request_data.max_results,
project=list(request_data.scope_set)[0].project)))
if args.regions is not None:
args_region_names = [
compute_holder.resources.Parse( # pylint:disable=g-complex-comprehension
region,
params={'project': properties.VALUES.core.project.GetOrFail},
collection='compute.regions').Name()
for region in args.regions or []]
# If no regions were provided by the user, fetch a list.
errors = []
region_names = (
args_region_names or [res.name for res in lister.GetGlobalResources( # pylint:disable=g-complex-comprehension
service=compute_client.apitools_client.regions,
project=properties.VALUES.core.project.GetOrFail(),
filter_expr=None,
http=compute_client.apitools_client.http,
batch_url=compute_client.batch_url,
errors=errors)])
if errors:
utils.RaiseToolException(
errors,
'Unable to fetch a list of regions. Specifying [--regions] may '
'fix this issue:')
for region_name in region_names:
requests.append(
(compute_client.apitools_client.regionOperations, 'List',
compute_client.apitools_client.regionOperations.GetRequestType(
'List')(
filter=request_data.filter,
maxResults=request_data.max_results,
region=region_name,
project=list(request_data.scope_set)[0].project)))
if args.zones is not None:
args_zone_names = [
compute_holder.resources.Parse( # pylint:disable=g-complex-comprehension
zone,
params={
'project': properties.VALUES.core.project.GetOrFail,
},
collection='compute.zones').Name()
for zone in args.zones or []]
# If no zones were provided by the user, fetch a list.
errors = []
zone_names = (
args_zone_names or [res.name for res in lister.GetGlobalResources( # pylint:disable=g-complex-comprehension
service=compute_client.apitools_client.zones,
project=properties.VALUES.core.project.GetOrFail(),
filter_expr=None,
http=compute_client.apitools_client.http,
batch_url=compute_client.batch_url,
errors=errors)])
if errors:
utils.RaiseToolException(
errors,
'Unable to fetch a list of zones. Specifying [--zones] may '
'fix this issue:')
for zone_name in zone_names:
requests.append(
(compute_client.apitools_client.zoneOperations, 'List',
compute_client.apitools_client.zoneOperations.GetRequestType(
'List')(
filter=request_data.filter,
maxResults=request_data.max_results,
zone=zone_name,
project=list(request_data.scope_set)[0].project)))
errors = []
results = list(
request_helper.ListJson(
requests=requests,
http=compute_client.apitools_client.http,
batch_url=compute_client.batch_url,
errors=errors))
if errors:
utils.RaiseToolException(errors)
return results
