def testListingWithSingleRequestAndNoPaging(self): items = [ self.messages.Operation(name='operation-1'), self.messages.Operation(name='operation-2'), self.messages.Operation(name='operation-3'), ] self.batch_helper.side_effect = [ [[ encoding.MessageToJson( self.messages.OperationList(items=items)) ], []], ] requests = [ (self.compute_v1.globalOperations, 'List', self.messages.ComputeGlobalOperationsListRequest( project='my-project')), ] errors = [] res = request_helper.ListJson( requests=requests, http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute', errors=errors) self.assertEqual(list(res), resource_projector.MakeSerializable(items)) self.assertFalse(errors) self.batch_helper.assert_called_once_with( requests=requests, http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute')
def __call__(self, frontend): scope_set = frontend.scope_set requests = [] if isinstance(scope_set, ZoneSet): for project, zones in six.iteritems( _GroupByProject(sorted(list(scope_set)))): for zone_ref in zones: requests.append((self.zonal_service, 'List', self.zonal_service.GetRequestType('List')( filter=frontend.filter, maxResults=frontend.max_results, project=project, zone=zone_ref.zone))) elif isinstance(scope_set, RegionSet): for project, regions in six.iteritems( _GroupByProject(sorted(list(scope_set)))): for region_ref in regions: requests.append((self.regional_service, 'List', self.regional_service.GetRequestType('List')( filter=frontend.filter, maxResults=frontend.max_results, project=project, region=region_ref.region))) elif isinstance(scope_set, GlobalScope): for project_ref in sorted(list(scope_set)): requests.append((self.global_service, 'List', self.global_service.GetRequestType('List')( filter=frontend.filter, maxResults=frontend.max_results, project=project_ref.project))) else: # scopeSet is AllScopes # generate AggregatedList for project_ref in sorted(list(scope_set.projects)): requests.append( (self.aggregation_service, 'AggregatedList', self.aggregation_service.GetRequestType('AggregatedList')( filter=frontend.filter, maxResults=frontend.max_results, project=project_ref.project))) errors = [] response_count = 0 for item in request_helper.ListJson( requests=requests, http=self.client.apitools_client.http, batch_url=self.client.batch_url, errors=errors): response_count += 1 yield item if errors: # If the command allows partial server errors, instead of raising an # exception to show something went wrong, we show a warning message that # contains the error messages instead. if self.allow_partial_server_failure and response_count > 0: utils.WarnIfPartialRequestFail(errors) else: utils.RaiseException(errors, ListException)
def GetResources(self, args, errors): """Yields images from (potentially) multiple projects.""" filter_expr = self.GetFilterExpr(args) image_projects = [self.project] if args.standard_images: image_projects.extend(constants.PUBLIC_IMAGE_PROJECTS) if args.preview_images: image_projects.extend(constants.PREVIEW_IMAGE_PROJECTS) requests = [] for project in image_projects: requests.append((self.service, 'List', self.messages.ComputeImagesListRequest( filter=filter_expr, maxResults=constants.MAX_RESULTS_PER_PAGE, project=project))) images = request_helper.ListJson(requests=requests, http=self.http, batch_url=self.batch_url, errors=errors) for image in images: if not image.get('deprecated', False) or args.show_deprecated: yield image
def testWithAggregatedListRequestAndNoPaging(self): items = [ self.messages.Instance(name='instance-1'), self.messages.Instance(name='instance-2'), self.messages.Instance(name='instance-3'), ] self.batch_helper.side_effect = [ [[ encoding.MessageToJson( self.messages.InstanceAggregatedList( items=self.messages.InstanceAggregatedList.ItemsValue( additionalProperties=[ _KV(key='zones/zone-1', value=self.messages.InstancesScopedList( instances=[ self.messages.Instance( name='instance-1'), self.messages.Instance( name='instance-2'), ])), _KV(key='zones/zone-2', value=self.messages.InstancesScopedList( instances=[ self.messages.Instance( name='instance-3'), ])), ]))) ], []], ] requests = [ (self.compute_v1.instances, 'AggregatedList', self.messages.ComputeInstancesAggregatedListRequest( project='my-project')), ] errors = [] res = request_helper.ListJson( requests=requests, http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute', errors=errors) self.assertEqual(set(instance['name'] for instance in res), set(instance.name for instance in items)) self.assertFalse(errors) self.batch_helper.assert_called_once_with( requests=[(self.compute_v1.instances, 'AggregatedList', self.messages.ComputeInstancesAggregatedListRequest( project='my-project'))], http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute')
def __call__(self, frontend): scope_set = frontend.scope_set requests = [] if isinstance(scope_set, ZoneSet): for project, zones in _GroupByProject(sorted( list(scope_set))).iteritems(): for zone_ref in zones: requests.append((self.zonal_service, 'List', self.zonal_service.GetRequestType('List')( filter=frontend.filter, maxResults=frontend.max_results, project=project, zone=zone_ref.zone))) elif isinstance(scope_set, RegionSet): for project, regions in _GroupByProject(sorted( list(scope_set))).iteritems(): for region_ref in regions: requests.append( (self.regional_service, 'List', self.regional_service.GetRequestType('List')( filter=frontend.filter, maxResults=frontend.max_results, project=project, region=region_ref.region))) elif isinstance(scope_set, GlobalScope): for project_ref in sorted(list(scope_set)): requests.append((self.global_service, 'List', self.global_service.GetRequestType('List')( filter=frontend.filter, maxResults=frontend.max_results, project=project_ref.project))) else: # scopeSet is AllScopes # generate AggregatedList for project_ref in sorted(list(scope_set.projects)): requests.append( (self.aggregation_service, 'AggregatedList', self.aggregation_service.GetRequestType('AggregatedList')( filter=frontend.filter, maxResults=frontend.max_results, project=project_ref.project))) errors = [] for item in request_helper.ListJson( requests=requests, http=self.client.apitools_client.http, batch_url=self.client.batch_url, errors=errors): yield item if errors: utils.RaiseException(errors, ListException)
def testWithAggregatedListRequestWithUnreachableScope(self): items = [ self.messages.Instance(name='instance-3'), ] warning = self.messages.InstancesScopedList.WarningValue( code=(self.messages.InstancesScopedList.WarningValue. CodeValueValuesEnum.UNREACHABLE), message='Scope [zones/zone-1] is unreachable.', ) self.batch_helper.side_effect = [ [[ encoding.MessageToJson( self.messages.InstanceAggregatedList( items=self.messages.InstanceAggregatedList.ItemsValue( additionalProperties=[ _KV(key='zones/zone-1', value=self.messages.InstancesScopedList( warning=warning)), _KV(key='zones/zone-2', value=self.messages.InstancesScopedList( instances=[ self.messages.Instance( name='instance-3'), ])), ]))) ], []], ] requests = [ (self.compute_v1.instances, 'AggregatedList', self.messages.ComputeInstancesAggregatedListRequest( project='my-project')), ] errors = [] res = request_helper.ListJson( requests=requests, http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute', errors=errors) self.assertEqual(list(res), resource_projector.MakeSerializable(items)) self.assertEqual(errors, [(None, 'Scope [zones/zone-1] is unreachable.')]) self.batch_helper.assert_called_once_with( requests=[(self.compute_v1.instances, 'AggregatedList', self.messages.ComputeInstancesAggregatedListRequest( project='my-project'))], http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute')
def GetResources(self, args, errors): """Returns zonal, regional and/or global resources. Args: args: argparse.Namespace, Parsed arguments errors: list, Errors will be returned in this list Returns: Zonal, regional and/or global resources. """ regions = getattr(args, 'regions', None) zones = getattr(args, 'zones', None) global_arg = getattr(args, 'global', None) # This is true if the user provided no flags indicating scope no_scope_flags = not regions and not zones and not global_arg requests = [] filter_expr = self.GetFilterExpr(args) if args.page_size is not None: max_results = min(args.page_size, constants.MAX_RESULTS_PER_PAGE) else: max_results = constants.MAX_RESULTS_PER_PAGE project = self.project # If --regions is present with no arguments OR no scope flags are present # then we have to do an aggregated list # pylint:disable=g-explicit-bool-comparison if no_scope_flags and self.aggregation_service: requests.append( (self.aggregation_service, 'AggregatedList', self.aggregation_service.GetRequestType('AggregatedList')( filter=filter_expr, maxResults=max_results, project=project))) # Else if some regions were provided then only list within them elif regions is not None: region_names = set( self.CreateGlobalReference(region, resource_type='regions').Name() for region in regions) for region_name in sorted(region_names): requests.append( (self.regional_service, 'List', self.regional_service.GetRequestType('List')( filter=filter_expr, maxResults=max_results, region=region_name, project=project))) # Else if some regions were provided then only list within them elif zones is not None: zone_names = set( self.CreateGlobalReference(zone, resource_type='zones').Name() for zone in zones) for zone_name in sorted(zone_names): requests.append( (self.zonal_service, 'List', self.zonal_service.GetRequestType('List')( filter=filter_expr, maxResults=max_results, zone=zone_name, project=project))) else: # Either --global was specified or we do not have aggregation service. # Note that --global, --region and --zone are mutually exclusive. requests.append( (self.global_service, 'List', self.global_service.GetRequestType('List')( filter=filter_expr, maxResults=max_results, project=project))) return request_helper.ListJson( requests=requests, http=self.http, batch_url=self.batch_url, errors=errors)
def testWithManyListRequestsAndPaging(self): zonal_page_1 = [ self.messages.Operation(name='operation-1'), self.messages.Operation(name='operation-2'), ] zonal_page_2 = [ self.messages.Operation(name='operation-3'), self.messages.Operation(name='operation-4'), self.messages.Operation(name='operation-5'), ] global_page_1 = [ self.messages.Operation(name='operation-6'), self.messages.Operation(name='operation-7'), ] global_page_2 = [ self.messages.Operation(name='operation-8'), self.messages.Operation(name='operation-9'), ] self.batch_helper.side_effect = [ [[ encoding.MessageToJson( self.messages.OperationList(items=zonal_page_1, nextPageToken='page-2-a')), encoding.MessageToJson( self.messages.OperationList(items=global_page_1, nextPageToken='page-2-b')) ], []], [[ encoding.MessageToJson( self.messages.OperationList(items=zonal_page_2)), encoding.MessageToJson( self.messages.OperationList(items=global_page_2)), ], []], ] requests = [ (self.compute_v1.zoneOperations, 'List', self.messages.ComputeZoneOperationsListRequest( zone='zone-1', project='my-project')), (self.compute_v1.globalOperations, 'List', self.messages.ComputeGlobalOperationsListRequest( project='my-project')), ] errors = [] res = request_helper.ListJson( requests=requests, http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute', errors=errors) # Protocol messages in protorpc are mutable, so they cannot be # hashed which means they cannot be stored in a set. To get around # this, we create sets using just the name attribute of the # protocol buffers before doing the equality comparison. self.assertEqual( set(operation['name'] for operation in res), set(operation.name for operation in zonal_page_1 + zonal_page_2 + global_page_1 + global_page_2)) self.assertFalse(errors) self.assertEqual(self.batch_helper.call_args_list, [ mock.call( requests=[(self.compute_v1.zoneOperations, 'List', self.messages.ComputeZoneOperationsListRequest( zone='zone-1', project='my-project')), (self.compute_v1.globalOperations, 'List', self.messages.ComputeGlobalOperationsListRequest( project='my-project'))], http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute'), mock.call(requests=[ (self.compute_v1.zoneOperations, 'List', self.messages.ComputeZoneOperationsListRequest( zone='zone-1', pageToken='page-2-a', project='my-project')), (self.compute_v1.globalOperations, 'List', self.messages.ComputeGlobalOperationsListRequest( pageToken='page-2-b', project='my-project')) ], http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute'), ])
def testListingWithSingleListRequestAndPaging(self): page_1 = [ self.messages.Operation(name='operation-1'), self.messages.Operation(name='operation-2'), self.messages.Operation(name='operation-3'), ] page_2 = [ self.messages.Operation(name='operation-4'), self.messages.Operation(name='operation-5'), self.messages.Operation(name='operation-6'), ] page_3 = [ self.messages.Operation(name='operation-7'), self.messages.Operation(name='operation-8'), self.messages.Operation(name='operation-9'), ] self.batch_helper.side_effect = [ [[ encoding.MessageToJson( self.messages.OperationList(items=page_1, nextPageToken='page-2')) ], []], [[ encoding.MessageToJson( self.messages.OperationList(items=page_2, nextPageToken='page-3')) ], []], [[ encoding.MessageToJson( self.messages.OperationList(items=page_3)) ], []], ] requests = [ (self.compute_v1.globalOperations, 'List', self.messages.ComputeGlobalOperationsListRequest( project='my-project')), ] errors = [] res = request_helper.ListJson( requests=requests, http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute', errors=errors) self.assertEqual( list(res), resource_projector.MakeSerializable(page_1 + page_2 + page_3)) self.assertFalse(errors) self.assertEqual(self.batch_helper.call_args_list, [ mock.call( requests=[(self.compute_v1.globalOperations, 'List', self.messages.ComputeGlobalOperationsListRequest( project='my-project'))], http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute'), mock.call( requests=[(self.compute_v1.globalOperations, 'List', self.messages.ComputeGlobalOperationsListRequest( pageToken='page-2', project='my-project'))], http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute'), mock.call( requests=[(self.compute_v1.globalOperations, 'List', self.messages.ComputeGlobalOperationsListRequest( pageToken='page-3', project='my-project'))], http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute'), ])
def testWithAggregatedListRequestAndPaging(self): page_1 = [ self.messages.Instance(name='instance-1'), self.messages.Instance(name='instance-2'), self.messages.Instance(name='instance-3'), ] page_2 = [ self.messages.Instance(name='instance-4'), self.messages.Instance(name='instance-5'), self.messages.Instance(name='instance-6'), ] self.batch_helper.side_effect = [ [[ encoding.MessageToJson( self.messages.InstanceAggregatedList( nextPageToken='page-2', items=self.messages.InstanceAggregatedList.ItemsValue( additionalProperties=[ _KV(key='zones/zone-1', value=self.messages.InstancesScopedList( instances=page_1, )), ]))) ], []], [[ encoding.MessageToJson( self.messages.InstanceAggregatedList( items=self.messages.InstanceAggregatedList.ItemsValue( additionalProperties=[ _KV(key='zones/zone-1', value=self.messages.InstancesScopedList( instances=page_2, )), ]))) ], []], ] requests = [ (self.compute_v1.instances, 'AggregatedList', self.messages.ComputeInstancesAggregatedListRequest( project='my-project')), ] errors = [] res = request_helper.ListJson( requests=requests, http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute', errors=errors) self.assertEqual(list(res), resource_projector.MakeSerializable(page_1 + page_2)) self.assertFalse(errors) self.assertEqual(self.batch_helper.call_args_list, [ mock.call( requests=[(self.compute_v1.instances, 'AggregatedList', self.messages.ComputeInstancesAggregatedListRequest( project='my-project'))], http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute'), mock.call( requests=[(self.compute_v1.instances, 'AggregatedList', self.messages.ComputeInstancesAggregatedListRequest( pageToken='page-2', project='my-project'))], http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute'), ])
def GetResources(self, args, errors): """Yields zonal, regional, and/or global resources.""" # This is True if the user provided no flags indicating scope. no_scope_flags = self.NoArguments(args) requests = [] filter_expr = self.GetFilterExpr(args) max_results = constants.MAX_RESULTS_PER_PAGE project = self.project # TODO(b/36050874): Start using aggregatedList for zones and regions when # the operations list API supports them. if no_scope_flags: requests.append( (self.global_service, 'AggregatedList', self.global_service.GetRequestType('AggregatedList')( filter=filter_expr, maxResults=max_results, project=project))) if not self._ga: # Add a request to get all Compute Account operations. requests.append( (self.account_service, 'List', self.account_service.GetRequestType('List')( filter=filter_expr, maxResults=max_results, project=project))) else: if getattr(args, 'global'): requests.append( (self.global_service, 'List', self.global_service.GetRequestType('List')( filter=filter_expr, maxResults=max_results, project=project))) if args.regions is not None: args_region_names = [ self.resources.Parse( region, params={'project': properties.VALUES.core.project.GetOrFail}, collection='compute.regions').Name() for region in args.regions or []] # If no regions were provided by the user, fetch a list. region_names = ( args_region_names or [res.name for res in self.FetchChoiceResources( attribute='region', service=self.compute.regions, flag_names=['--regions'])]) for region_name in region_names: requests.append( (self.regional_service, 'List', self.regional_service.GetRequestType('List')( filter=filter_expr, maxResults=constants.MAX_RESULTS_PER_PAGE, region=region_name, project=self.project))) if args.zones is not None: args_zone_names = [ self.resources.Parse( zone, params={ 'project': properties.VALUES.core.project.GetOrFail, }, collection='compute.zones').Name() for zone in args.zones or []] # If no zones were provided by the user, fetch a list. zone_names = ( args_zone_names or [res.name for res in self.FetchChoiceResources( attribute='zone', service=self.compute.zones, flag_names=['--zones'])]) for zone_name in zone_names: requests.append( (self.zonal_service, 'List', self.zonal_service.GetRequestType('List')( filter=filter_expr, maxResults=constants.MAX_RESULTS_PER_PAGE, zone=zone_name, project=self.project))) if not self._ga and args.accounts: requests.append( (self.account_service, 'List', self.account_service.GetRequestType('List')( filter=filter_expr, maxResults=max_results, project=project))) return request_helper.ListJson( requests=requests, http=self.http, batch_url=self.batch_url, errors=errors)
def Run(self, args): """Yields zonal, regional, and/or global resources.""" compute_holder = base_classes.ComputeApiHolder(self.ReleaseTrack()) compute_client = compute_holder.client # This is True if the user provided no flags indicating scope. no_scope_flags = self.NoArguments(args) requests = [] request_data = lister.ParseNamesAndRegexpFlags(args, compute_holder.resources) # TODO(b/36050874): Start using aggregatedList for zones and regions when # the operations list API supports them. if no_scope_flags: requests.append( (compute_client.apitools_client.globalOperations, 'AggregatedList', compute_client.apitools_client.globalOperations.GetRequestType( 'AggregatedList')( filter=request_data.filter, maxResults=request_data.max_results, project=list(request_data.scope_set)[0].project))) else: if getattr(args, 'global'): requests.append( (compute_client.apitools_client.globalOperations, 'List', compute_client.apitools_client.globalOperations.GetRequestType( 'List')( filter=request_data.filter, maxResults=request_data.max_results, project=list(request_data.scope_set)[0].project))) if args.regions is not None: args_region_names = [ compute_holder.resources.Parse( # pylint:disable=g-complex-comprehension region, params={'project': properties.VALUES.core.project.GetOrFail}, collection='compute.regions').Name() for region in args.regions or []] # If no regions were provided by the user, fetch a list. errors = [] region_names = ( args_region_names or [res.name for res in lister.GetGlobalResources( # pylint:disable=g-complex-comprehension service=compute_client.apitools_client.regions, project=properties.VALUES.core.project.GetOrFail(), filter_expr=None, http=compute_client.apitools_client.http, batch_url=compute_client.batch_url, errors=errors)]) if errors: utils.RaiseToolException( errors, 'Unable to fetch a list of regions. Specifying [--regions] may ' 'fix this issue:') for region_name in region_names: requests.append( (compute_client.apitools_client.regionOperations, 'List', compute_client.apitools_client.regionOperations.GetRequestType( 'List')( filter=request_data.filter, maxResults=request_data.max_results, region=region_name, project=list(request_data.scope_set)[0].project))) if args.zones is not None: args_zone_names = [ compute_holder.resources.Parse( # pylint:disable=g-complex-comprehension zone, params={ 'project': properties.VALUES.core.project.GetOrFail, }, collection='compute.zones').Name() for zone in args.zones or []] # If no zones were provided by the user, fetch a list. errors = [] zone_names = ( args_zone_names or [res.name for res in lister.GetGlobalResources( # pylint:disable=g-complex-comprehension service=compute_client.apitools_client.zones, project=properties.VALUES.core.project.GetOrFail(), filter_expr=None, http=compute_client.apitools_client.http, batch_url=compute_client.batch_url, errors=errors)]) if errors: utils.RaiseToolException( errors, 'Unable to fetch a list of zones. Specifying [--zones] may ' 'fix this issue:') for zone_name in zone_names: requests.append( (compute_client.apitools_client.zoneOperations, 'List', compute_client.apitools_client.zoneOperations.GetRequestType( 'List')( filter=request_data.filter, maxResults=request_data.max_results, zone=zone_name, project=list(request_data.scope_set)[0].project))) errors = [] results = list( request_helper.ListJson( requests=requests, http=compute_client.apitools_client.http, batch_url=compute_client.batch_url, errors=errors)) if errors: utils.RaiseToolException(errors) return results