def GetResources(self, args, errors): get_request = self.GetUrlMapGetRequest(args) new_errors = [] objects = list(request_helper.MakeRequests( requests=[get_request], http=self.http, batch_url=self.batch_url, errors=new_errors, custom_get_requests=None)) errors.extend(new_errors) if new_errors: utils.RaiseToolException( errors, error_message='Could not fetch resource:') urlmap_id = objects[0].id filter_expr = ('(operationType eq invalidateCache) (targetId eq ' '{urlmap_id})').format(urlmap_id=urlmap_id) max_results = args.limit or constants.MAX_RESULTS_PER_PAGE project = self.project requests = [ (self.global_service, 'AggregatedList', self.global_service.GetRequestType('AggregatedList')( filter=filter_expr, maxResults=max_results, orderBy='creationTimestamp desc', project=project)) ] return request_helper.MakeRequests(requests=requests, http=self.http, batch_url=self.batch_url, errors=errors, custom_get_requests=None)
def Run(self, args): self.ref = self.CreateReference(args) get_request = self.GetGetRequest(args) errors = [] objects = list( request_helper.MakeRequests(requests=[get_request], http=self.http, batch_url=self.batch_url, errors=errors)) if errors: utils.RaiseToolException( errors, error_message='There was a problem fetching the resource:') new_object = self.Modify(args, objects[0]) # If existing object is equal to the proposed object or if # Modify() returns None, then there is no work to be done, so we # print the resource and return. if not new_object or objects[0] == new_object: for resource in lister.ProcessResults( resources=[objects[0]], field_selector=property_selector.PropertySelector( properties=None, transformations=self.transformations)): log.status.Print( 'No change requested; skipping update for [{0}].'.format( resource[u'name'])) yield resource return resource_list = request_helper.MakeRequests( requests=[self.GetSetRequest(args, new_object, objects[0])], http=self.http, batch_url=self.batch_url, errors=errors) resource_list = lister.ProcessResults( resources=resource_list, field_selector=property_selector.PropertySelector( properties=None, transformations=self.transformations)) for resource in resource_list: yield resource if errors: utils.RaiseToolException( errors, error_message='There was a problem modifying the resource:')
def ListPerInstanceConfigs(client, igm_ref): """Lists per-instance-configs for a given IGM.""" if not hasattr(client.messages, 'ComputeInstanceGroupManagersListPerInstanceConfigsRequest'): return [] if igm_ref.Collection() == 'compute.instanceGroupManagers': service = client.apitools_client.instanceGroupManagers request = (client.messages .ComputeInstanceGroupManagersListPerInstanceConfigsRequest)( instanceGroupManager=igm_ref.Name(), project=igm_ref.project, zone=igm_ref.zone, ) elif igm_ref.Collection() == 'compute.regionInstanceGroupManagers': service = client.apitools_client.regionInstanceGroupManagers request = (client.messages. ComputeRegionInstanceGroupManagersListPerInstanceConfigsRequest)( instanceGroupManager=igm_ref.Name(), project=igm_ref.project, region=igm_ref.region, ) else: raise ValueError('Unknown reference type {0}'.format(igm_ref.Collection())) errors = [] results = list( request_helper.MakeRequests( requests=[(service, 'ListPerInstanceConfigs', request)], http=client.apitools_client.http, batch_url=client.batch_url, errors=errors)) if not results: return [] return results[0].items
def GetImage(self, image_ref): """Returns the image resource corresponding to the given reference.""" errors = [] requests = [] name = image_ref.Name() if name.startswith(FAMILY_PREFIX): requests.append((self._compute.images, 'GetFromFamily', self._messages.ComputeImagesGetFromFamilyRequest( family=name[len(FAMILY_PREFIX):], project=image_ref.project))) else: requests.append((self._compute.images, 'Get', self._messages.ComputeImagesGetRequest( image=name, project=image_ref.project))) res = list( request_helper.MakeRequests(requests=requests, http=self._http, batch_url=self._batch_url, errors=errors)) if errors: utils.RaiseException( errors, utils.ImageNotFoundError, error_message='Could not fetch image resource:') return res[0]
def GetResources(self, args, errors): """Yields images from (potentially) multiple projects.""" filter_expr = self.GetFilterExpr(args) image_projects = [self.project] if args.standard_images: image_projects.extend(constants.PUBLIC_IMAGE_PROJECTS) if args.preview_images: image_projects.extend(constants.PREVIEW_IMAGE_PROJECTS) requests = [] for project in image_projects: requests.append((self.service, 'List', self.messages.ComputeImagesListRequest( filter=filter_expr, maxResults=constants.MAX_RESULTS_PER_PAGE, project=project))) images = request_helper.MakeRequests(requests=requests, http=self.http, batch_url=self.batch_url, errors=errors, custom_get_requests=None) for image in images: if not image.deprecated or args.show_deprecated: yield image
def Run(self, args): """Retrieves response with nodes in the node group.""" holder = base_classes.ComputeApiHolder(self.ReleaseTrack()) client = holder.client group_ref = flags.MakeNodeGroupArg().ResolveAsResource( args, holder.resources, scope_lister=compute_flags.GetDefaultScopeLister(holder.client)) request = client.messages.ComputeNodeGroupsListNodesRequest( nodeGroup=group_ref.Name(), zone=group_ref.zone, project=group_ref.project) errors = [] results = list( request_helper.MakeRequests(requests=[ (client.apitools_client.nodeGroups, 'ListNodes', request) ], http=client.apitools_client.http, batch_url=client.batch_url, errors=errors)) if errors: utils.RaiseToolException(errors) return self.getItems(results)
def GetZones(self, resource_refs): """Fetches zone resources.""" errors = [] requests = [] zone_names = set() for resource_ref in resource_refs: if resource_ref.zone not in zone_names: zone_names.add(resource_ref.zone) requests.append(( self._compute.zones, 'Get', self._messages.ComputeZonesGetRequest( project=resource_ref.project, zone=resource_ref.zone))) res = list(request_helper.MakeRequests( requests=requests, http=self._http, batch_url=self._batch_url, errors=errors)) if errors: return None else: return res
def Run(self, args): holder = base_classes.ComputeApiHolder(self.ReleaseTrack()) resources = holder.resources commitment_ref = flags.MakeCommitmentArg(False).ResolveAsResource( args, resources, scope_lister=compute_flags.GetDefaultScopeLister(holder.client)) messages = holder.client.messages region = properties.VALUES.compute.region.Get() project = properties.VALUES.core.project.Get() create_request = self._MakeCreateRequest(args, messages, project, region, commitment_ref, holder) service = holder.client.apitools_client.regionCommitments batch_url = holder.client.batch_url http = holder.client.apitools_client.http errors = [] result = list( request_helper.MakeRequests(requests=[(service, 'Insert', create_request)], http=http, batch_url=batch_url, errors=errors)) for i, error in enumerate(errors): if re.match(_MISSING_COMMITMENTS_QUOTA_REGEX, error[1]): errors[i] = ( error[0], error[1] + (' You can request commitments quota on ' 'https://cloud.google.com/compute/docs/instances/' 'signing-up-committed-use-discounts#quota')) if errors: utils.RaiseToolException(errors) return result
def MakeRequests(self, requests, errors_to_collect=None, progress_tracker=None, no_followup=False, always_return_operation=False, followup_overrides=None, log_warnings=True, log_result=True, timeout=None): """Sends given request in batch mode.""" errors = errors_to_collect if errors_to_collect is not None else [] objects = list( request_helper.MakeRequests( requests=requests, http=self._client.http, batch_url=self._batch_url, errors=errors, progress_tracker=progress_tracker, no_followup=no_followup, always_return_operation=always_return_operation, followup_overrides=followup_overrides, log_warnings=log_warnings, log_result=log_result, timeout=timeout)) if errors_to_collect is None and errors: utils.RaiseToolException(errors, error_message='Could not fetch resource:') return objects
def Run(self, args): instance_ref = self.CreateZonalReference(args.name, args.zone) start = None while True: request = ( self.compute.instances, 'GetSerialPortOutput', self.messages.ComputeInstancesGetSerialPortOutputRequest( instance=instance_ref.Name(), project=self.project, port=args.port, start=start, zone=instance_ref.zone)) errors = [] objects = list( request_helper.MakeRequests(requests=[request], http=self.http, batch_url=self.batch_url, errors=errors, custom_get_requests=None)) if errors: raise TailSerialPortOutputException( 'Could not fetch serial port output: ' + ','.join([error[1] for error in errors])) result = objects[0] log.out.write(result.contents) start = result.next # If we didn't get any results, we sleep for a short time before the next # call. if not result.contents: time.sleep(self.POLL_SLEEP_SECS)
def CheckCustomCpuRamRatio(self, zone, machine_type_name): """Checks that the CPU and memory ratio is a supported custom instance type. Args: self: the CreateGA 'instances create' calling class zone: the zone of the instance(s) being created machine_type_name: The machine type of the instance being created. Returns: Nothing. Function acts as a bound checker, and will raise an exception from within the function if needed. Raises: utils.RaiseToolException if a custom machine type ratio is out of bounds. """ if 'custom' in machine_type_name: mt_get_pb = self.messages.ComputeMachineTypesGetRequest( machineType=machine_type_name, project=self.project, zone=zone) mt_get_reqs = [(self.compute.machineTypes, 'Get', mt_get_pb)] errors = [] # Makes a 'machine-types describe' request to check the bounds _ = list(request_helper.MakeRequests( requests=mt_get_reqs, http=self.http, batch_url=self.batch_url, errors=errors, custom_get_requests=None)) if errors: utils.RaiseToolException( errors, error_message='Could not fetch machine type:')
def _SetInstanceMetadata(self, instance, new_metadata): """Sets the project metadata to the new metadata.""" compute = self.compute errors = [] # API wants just the zone name, not the full URL zone = instance.zone.split('/')[-1] list(request_helper.MakeRequests( requests=[ (compute.instances, 'SetMetadata', self.messages.ComputeInstancesSetMetadataRequest( instance=instance.name, metadata=new_metadata, project=properties.VALUES.core.project.Get( required=True), zone=zone ))], http=self.http, batch_url=self.batch_url, errors=errors)) if errors: utils.RaiseToolException( errors, error_message='Could not add SSH key to instance metadata:')
def Run(self, args): """Returns a list of backendServiceGroupHealth objects.""" self.backend_service_ref = self.CreateGlobalReference( args.name, resource_type='backendServices') backend_service = self.GetBackendService(args) if not backend_service.backends: return # Call GetHealth for each group in the backend service requests = [] for backend in backend_service.backends: request_message = self.messages.ComputeBackendServicesGetHealthRequest( resourceGroupReference=self.messages.ResourceGroupReference( group=backend.group), project=self.project, backendService=self.backend_service_ref.Name()) requests.append((self.service, 'GetHealth', request_message)) # Instead of batching-up all requests and making a single # request_helper.MakeRequests call, go one backend at a time. # We do this because getHealth responses don't say what resource # they correspond to. It's not obvious how to reliably match up # responses and backends when there are errors. Addtionally the contract # for MakeRequests doesn't guarantee response order will match # request order. # # TODO(b/25015230) Simply make a batch request once the response # gives more information. errors = [] for request in requests: # The list() call below is itended to force the generator returned by # MakeRequests. If there are exceptions the command will abort, which is # expected. Having a list simplifies some of the checks that follow. resources = list( request_helper.MakeRequests(requests=[request], http=self.http, batch_url=self.batch_url, errors=errors, custom_get_requests=None)) if len(resources) is 0: # Request failed, error information will accumulate in errors continue try: [resource] = resources except ValueError: # Intended to throw iff resources contains more than one element. Just # want to avoid a user potentially seeing an index out of bounds # exception. raise exceptions.InternalError('Invariant failure') yield { 'backend': request[2].resourceGroupReference.group, 'status': resource } if errors: utils.RaiseToolException( errors, error_message='Could not get health for some groups:')
def GetResources(self, args): """Retrieves response with instance in the instance group.""" # Note: only zonal resources parsed here. group_ref = self.resources.Parse( args.name, params={ 'project': properties.VALUES.core.project.GetOrFail, 'zone': args.zone }, collection='compute.' + self.resource_type) if args.regexp: filter_expr = 'instance eq {0}'.format(args.regexp) else: filter_expr = None request = self.service.GetRequestType(self.method)( instanceGroup=group_ref.Name(), instanceGroupsListInstancesRequest=( self.messages.InstanceGroupsListInstancesRequest()), zone=group_ref.zone, filter=filter_expr, project=group_ref.project) errors = [] results = list( request_helper.MakeRequests(requests=[(self.service, self.method, request)], http=self.http, batch_url=self.batch_url, errors=errors)) return results, errors
def Run(self, args): """Issues requests necessary for describing users.""" compute_holder = base_classes.ComputeApiHolder(self.ReleaseTrack()) holder = base_classes.ComputeUserAccountsApiHolder(self.ReleaseTrack()) client = holder.client user = args.name if not user: user = gaia.GetDefaultAccountName( compute_holder.client.apitools_client.http) user_ref = holder.resources.Parse( user, params={'project': properties.VALUES.core.project.GetOrFail}, collection='clouduseraccounts.users') request = client.MESSAGES_MODULE.ClouduseraccountsUsersGetRequest( project=user_ref.project, user=user_ref.Name()) errors = [] responses = list( request_helper.MakeRequests( requests=[(client.users, 'Get', request)], http=client.http, batch_url='https://www.googleapis.com/batch/', errors=errors)) if errors: utils.RaiseToolException(errors, error_message='Could not fetch resource:') return responses
def testWithSynchronousRequestsOnly(self): expected_responses = [ self.messages.Instance(name='my-instance'), self.messages.Zone(name='my-zone'), ] self.batch_helper.side_effect = iter([ (expected_responses, []), ]) requests = [ (self.compute_v1.instances, 'Get', self.messages.ComputeInstancesGetRequest(instance='my-instance', project='my-project', zone='my-zone')), (self.compute_v1.zones, 'Get', self.messages.ComputeZonesGetRequest(zone='my-zone', project='my-project')), ] errors = [] responses = request_helper.MakeRequests( requests=requests, http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute', errors=errors) self.assertEqual(list(responses), expected_responses) self.assertFalse(errors) self.batch_helper.assert_called_once_with( requests=requests, http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute') self.assertFalse(self.wait_for_operations.called)
def GetRegions(self, resource_refs): """Fetches region resources.""" # TODO(b/34445512) errors = [] requests = [] region_names = set() for resource_ref in resource_refs: if (resource_ref.project, resource_ref.region) not in region_names: region_names.add((resource_ref.project, resource_ref.region)) requests.append(( self.compute.regions, 'Get', self.messages.ComputeRegionsGetRequest( project=resource_ref.project, region=resource_ref.region))) if requests: res = list(request_helper.MakeRequests( requests=requests, http=self.http, batch_url=self.batch_url, errors=errors)) else: return None if errors: return None else: return res
def UploadPublicKey(self, user, public_key): """Uploads a public key to a user.""" description = 'Added by gcloud compute from {0}'.format( self.GetHostName()) default_expiration = '1d' parser = arg_parsers.Duration() expiration_rfc3339_str = time_utils.CalculateExpiration( parser(default_expiration)) request = self.PublicKeyGenerateUploadRequest( user, public_key, expiration_time=expiration_rfc3339_str, description=description) requests = [request] errors = [] res = list( request_helper.MakeRequests(requests=requests, http=self.http, batch_url=self.batch_url, errors=errors, custom_get_requests=None)) if errors: utils.RaiseException(errors, UserException, error_message='Could not upload public key:') return res
def Run(self, args): request_protobufs = self.CreateRequests(args) requests = [] for request in request_protobufs: requests.append((self.service, self.method, request)) errors = [] if args. async: resources, new_errors = batch_helper.MakeRequests( requests=requests, http=self.http, batch_url=self.batch_url) if not new_errors: for invalidation_operation in resources: log.status.write('Invalidation pending for [{0}]\n'.format( invalidation_operation.targetLink)) log.status.write('Monitor its progress at [{0}]\n'.format( invalidation_operation.selfLink)) errors.extend(new_errors) else: # We want to run through the generator that MakeRequests returns in order # to actually make the requests. resources = list( request_helper.MakeRequests(requests=requests, http=self.http, batch_url=self.batch_url, errors=errors)) resources = lister.ProcessResults( resources=resources, field_selector=property_selector.PropertySelector( properties=None, transformations=self.transformations)) if errors: utils.RaiseToolException(errors) return resources
def CreateUser(self, user, owner_email, description=None): """Creates an account service user.""" user = self.clouduseraccounts.MESSAGES_MODULE.User( name=user, description=description, owner=owner_email, ) request = (self.clouduseraccounts.users, 'Insert', self.clouduseraccounts.MESSAGES_MODULE. ClouduseraccountsUsersInsertRequest(project=self.project, user=user)) errors = [] res = list( request_helper.MakeRequests(requests=[request], http=self.http, batch_url=self.batch_url, errors=errors, custom_get_requests=None)) if errors: utils.RaiseException(errors, UserException, error_message='Could not create user:') return res
def GetResources(self, args): """Retrieves response with instance in the instance group.""" group_ref = self.CreateZonalReference(args.name, args.zone) if args.regexp: filter_expr = 'instance eq {0}'.format(args.regexp) else: filter_expr = None request = self.service.GetRequestType(self.method)( instanceGroup=group_ref.Name(), instanceGroupsListInstancesRequest=( self.messages.InstanceGroupsListInstancesRequest()), zone=group_ref.zone, filter=filter_expr, project=self.context['project']) errors = [] results = list( request_helper.MakeRequests(requests=[(self.service, self.method, request)], http=self.http, batch_url=self.batch_url, errors=errors, custom_get_requests=None)) return results, errors
def Run(self, args): """Returns a list of TargetPoolInstanceHealth objects.""" self.target_pool_ref = self.CreateRegionalReference( args.name, args.region, resource_type='targetPools') target_pool = self.GetTargetPool() instances = target_pool.instances # If the target pool has no instances, we should return an empty # list. if not instances: return requests = [] for instance in instances: request_message = self.messages.ComputeTargetPoolsGetHealthRequest( instanceReference=self.messages.InstanceReference( instance=instance), project=self.project, region=self.target_pool_ref.region, targetPool=self.target_pool_ref.Name()) requests.append((self.service, 'GetHealth', request_message)) errors = [] resources = request_helper.MakeRequests(requests=requests, http=self.http, batch_url=self.batch_url, errors=errors, custom_get_requests=None) for resource in resources: yield resource if errors: utils.RaiseToolException( errors, error_message='Could not get health for some targets:')
def GetProject(self, project): """Returns the project object. Args: project: str, the project we are requesting or None for value from from properties Returns: The project object """ errors = [] objects = list( request_helper.MakeRequests(requests=[ (self.compute.projects, 'Get', self.messages.ComputeProjectsGetRequest( project=project or properties.VALUES.core.project.Get(required=True), )) ], http=self.http, batch_url=self.batch_url, errors=errors)) if errors: utils.RaiseToolException( errors, error_message='Could not fetch project resource:') return objects[0]
def Run(self, args): instance_ref = instance_flags.INSTANCE_ARG.ResolveAsResource( args, self.resources, scope_lister=flags.GetDefaultScopeLister(self.compute_client, self.project)) request = (self.compute.instances, 'GetSerialPortOutput', self.messages.ComputeInstancesGetSerialPortOutputRequest( instance=instance_ref.Name(), project=self.project, port=args.port, start=args.start, zone=instance_ref.zone)) errors = [] objects = list( request_helper.MakeRequests(requests=[request], http=self.http, batch_url=self.batch_url, errors=errors)) if errors: raise GetSerialPortOutputException( 'Could not fetch serial port output: ' + ','.join([error[1] for error in errors])) response = objects[0] self._start = args.start self._response = response return response
def Run(self, args): instance_ref = self.CreateZonalReference(args.name, args.zone) request = (self.compute.instances, 'GetSerialPortOutput', self.messages.ComputeInstancesGetSerialPortOutputRequest( instance=instance_ref.Name(), project=self.project, port=args.port, start=args.start, zone=instance_ref.zone)) errors = [] objects = list( request_helper.MakeRequests(requests=[request], http=self.http, batch_url=self.batch_url, errors=errors, custom_get_requests=None)) if errors: raise GetSerialPortOutputException( 'Could not fetch serial port output: ' + ','.join([error[1] for error in errors])) response = objects[0] self._start = args.start self._response = response return response
def Run(self, args): ref = self.CreateReference(args) request_class = self.service.GetRequestType(self.method) request = request_class(project=self.project) self.ScopeRequest(ref, request) self.SetResourceName(ref, request) get_policy_request = (self.service, self.method, request) errors = [] objects = request_helper.MakeRequests(requests=[get_policy_request], http=self.http, batch_url=self.batch_url, errors=errors, custom_get_requests=None) # Converting the objects genrator to a list triggers the # logic that actually populates the errors list. resources = list(objects) if errors: utils.RaiseToolException(errors, error_message='Could not fetch resource:') # TODO(user): determine how this output should look when empty. # GetIamPolicy always returns either an error or a valid policy. # If no policy has been set it returns a valid empty policy (just an etag.) # It is not possible to have multiple policies for one resource. return resources[0]
def Run(self, args): """Yields JSON-serializable dicts of resources.""" ref = self.CreateReference(args) get_request_class = self.service.GetRequestType(self.method) request = get_request_class(project=getattr(ref, 'project', self.project)) self.SetNameField(ref, request) self.ScopeRequest(ref, request) get_request = (self.service, self.method, request) errors = [] objects = request_helper.MakeRequests( requests=[get_request], http=self.http, batch_url=self.batch_url, errors=errors) resource_list = lister.ProcessResults(objects, field_selector=None) resource_list = list(self.ComputeDynamicProperties(args, resource_list)) if errors: utils.RaiseToolException( errors, error_message='Could not fetch resource:') return resource_list[0]
def testThatListRequestsCannotBeMixedWithNonListRequests(self): requests = [ (self.compute_v1.instances, 'Get', self.messages.ComputeInstancesGetRequest(instance='my-instance', project='my-project', zone='my-zone')), (self.compute_v1.instances, 'List', self.messages.ComputeInstancesListRequest(project='my-project', zone='my-zone')), (self.compute_v1.zones, 'Get', self.messages.ComputeZonesGetRequest(zone='my-zone', project='my-project')), ] errors = [] with self.assertRaises(ValueError): list( request_helper.MakeRequests( requests=requests, http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute', errors=errors)) self.assertFalse(errors) self.assertFalse(self.batch_helper.called) self.assertFalse(self.wait_for_operations.called)
def GetResources(self, args): """Retrieves response with instance in the instance group.""" group_ref = self.CreateGroupReference(args) if hasattr(group_ref, 'zone'): service = self.compute.instanceGroupManagers request = service.GetRequestType(self.method)( instanceGroupManager=group_ref.Name(), zone=group_ref.zone, project=group_ref.project) elif hasattr(group_ref, 'region'): service = self.compute.regionInstanceGroupManagers request = service.GetRequestType(self.method)( instanceGroupManager=group_ref.Name(), region=group_ref.region, project=group_ref.project) errors = [] results = list( request_helper.MakeRequests(requests=[(service, self.method, request)], http=self.http, batch_url=self.batch_url, errors=errors)) return results, errors
def testListingWithSingleRequestAndNoPaging(self): items = [ self.messages.Operation(name='operation-1'), self.messages.Operation(name='operation-2'), self.messages.Operation(name='operation-3'), ] self.batch_helper.side_effect = iter([ [[self.messages.OperationList(items=items)], []], ]) requests = [ (self.compute_v1.globalOperations, 'List', self.messages.ComputeGlobalOperationsListRequest( project='my-project')), ] errors = [] res = request_helper.MakeRequests( requests=requests, http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute', errors=errors) self.assertEqual(list(res), items) self.assertFalse(errors) self.batch_helper.assert_called_once_with( requests=requests, http=self.mock_http, batch_url='https://www.googleapis.com/batch/compute') self.assertFalse(self.wait_for_operations.called)