def _GetModuleFromPath(impl_file, path, construction_id): """Import the module and dig into it to return the namespace we are after. Import the module relative to the top level directory. Then return the actual module corresponding to the last bit of the path. Args: impl_file: str, The path to the file this was loaded from (for error reporting). path: [str], A list of group names that got us down to this command group with respect to the CLI itself. This path should be used for things like error reporting when a specific element in the tree needs to be referenced. construction_id: str, A unique identifier for the CLILoader that is being constructed. Returns: The imported module. """ # Make sure this module name never collides with any real module name. # Use the CLI naming path, so values are always unique. name_to_give = '__calliope__command__.{construction_id}.{name}'.format( construction_id=construction_id, name='.'.join(path).replace('-', '_')) try: return pkg_resources.GetModuleFromPath(name_to_give, impl_file) # pylint:disable=broad-except, We really do want to catch everything here, # because if any exceptions make it through for any single command or group # file, the whole CLI will not work. Instead, just log whatever it is. except Exception as e: exceptions.reraise(CommandLoadFailure('.'.join(path), e))
def upload_object(self, upload_stream, upload_resource, progress_callback=None, request_config=None): """See super class.""" # TODO(b/160998556): Implement resumable upload. del progress_callback kwargs = { 'Bucket': upload_resource.storage_url.bucket_name, 'Key': upload_resource.storage_url.object_name, 'Body': upload_stream.read() } if request_config and request_config.predefined_acl_string: kwargs['ACL'] = _translate_predefined_acl_string_to_s3( request_config.predefined_acl_string) try: response = self.client.put_object(**kwargs) return self._get_object_resource_from_s3_response( response, upload_resource.storage_url.bucket_name, upload_resource.storage_url.object_name) except botocore.exceptions.ClientError as error: core_exceptions.reraise(errors.S3ApiError(error))
def get_object_metadata(self, bucket_name, object_name, generation=None, fields_scope=None): """See super class.""" request = {'Bucket': bucket_name, 'Key': object_name} # The VersionId keyword argument to head_object is not nullable if it is # present, so only include it in the function call if it has a value. if generation is not None: request['VersionId'] = generation try: object_dict = self.client.head_object(**request) except botocore.exceptions.ClientError as error: core_exceptions.reraise(errors.S3ApiError(error)) # User requested ACL's with FieldsScope.FULL. if fields_scope is cloud_api.FieldsScope.FULL: try: acl_response = self.client.get_object_acl(**request) acl_response.pop('ResponseMetadata', None) object_dict['ACL'] = acl_response except botocore.exceptions.ClientError as error: object_dict['ACL'] = errors.S3ApiError(error) return self._get_object_resource_from_s3_response( object_dict, bucket_name, object_name)
def list_buckets(self, fields_scope=cloud_api.FieldsScope.NO_ACL): """See super class.""" projection = self._get_projection( fields_scope, self.messages.StorageBucketsListRequest) request = self.messages.StorageBucketsListRequest( project=properties.VALUES.core.project.GetOrFail(), projection=projection) global_params = None if fields_scope == cloud_api.FieldsScope.SHORT: global_params = self.messages.StandardQueryParameters() global_params.fields = 'items/name' # TODO(b/160238394) Decrypt metadata fields if necessary. bucket_iter = list_pager.YieldFromList( self.client.buckets, request, batch_size=cloud_api.NUM_ITEMS_PER_LIST_PAGE, global_params=global_params) try: for bucket in bucket_iter: yield gcs_metadata_util.get_bucket_resource_from_metadata( bucket) except apitools_exceptions.HttpError as e: core_exceptions.reraise( cloud_errors.translate_error(e, _ERROR_TRANSLATION))
def _SetDefaultVersion(new_version, api_client): """Sets the given version as the default. Args: new_version: Version, The version to promote. api_client: appengine_api_client.AppengineApiClient to use to make requests. """ metrics.CustomTimedEvent(metric_names.SET_DEFAULT_VERSION_API_START) # TODO(b/31824825): It sometimes takes a while for a new service to show up. # Retry it if we get a service not found error. def ShouldRetry(exc_type, unused_exc_value, unused_traceback, unused_state): return issubclass(exc_type, apitools_exceptions.HttpError) try: retryer = retry.Retryer(max_retrials=3, exponential_sleep_multiplier=2) retryer.RetryOnException( api_client.SetDefaultVersion, [new_version.service, new_version.id], should_retry_if=ShouldRetry, sleep_ms=1000) except retry.MaxRetrialsException as e: (unused_result, exc_info) = e.last_result if exc_info: exceptions.reraise(exc_info[1], tb=exc_info[2]) else: # This shouldn't happen, but if we don't have the exception info for some # reason, just convert the MaxRetrialsException. raise exceptions.InternalError() metrics.CustomTimedEvent(metric_names.SET_DEFAULT_VERSION_API)
def CopyObject(self, source_resource, destination_resource, progress_callback=None, request_config=None): """See super class.""" del progress_callback source_kwargs = {'Bucket': source_resource.storage_url.bucket_name, 'Key': source_resource.storage_url.object_name} if source_resource.storage_url.generation: source_kwargs['VersionId'] = source_resource.storage_url.generation kwargs = {'Bucket': destination_resource.storage_url.bucket_name, 'Key': destination_resource.storage_url.object_name, 'CopySource': source_kwargs} if request_config and request_config.predefined_acl_string: kwargs['ACL'] = _TranslatePredefinedAclStringToS3( request_config.predefined_acl_string) try: response = self.client.copy_object(**kwargs) return self._GetObjectResourceFromS3Response( response, kwargs['Bucket'], kwargs['Key']) except botocore.exceptions.ClientError as error: core_exceptions.reraise(errors.S3ApiError(error))
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace, All the arguments that were provided to this command invocation. Returns: an Operation message which tracks the asynchronous export """ apitools_client = genomics_util.GetGenomicsClient() genomics_messages = genomics_util.GetGenomicsMessages() try: return apitools_client.readgroupsets.Export( genomics_messages.GenomicsReadgroupsetsExportRequest( readGroupSetId=args.read_group_set_id, exportReadGroupSetRequest=genomics_messages. ExportReadGroupSetRequest( projectId=genomics_util.GetProjectId(), exportUri=args.export_uri, referenceNames=args.reference_names))) except apitools_exceptions.HttpError as error: # Map our error messages (JSON API camelCased) back into flag names. msg = ( exceptions.HttpException(error).payload.status_message.replace( 'exportUri', '--export-uri').replace('referenceNames', '--reference-names')) core_exceptions.reraise(exceptions.HttpException(msg))
def GetOrRaise(self): if self.value: return self.value[0] # self.value is a one-tuple elif self.error: raise self.error # pylint: disable=raising-bad-type else: exceptions.reraise(self.exc_info[1], tb=self.exc_info[2])
def _SetDefaultVersion(new_version, api_client): """Sets the given version as the default. Args: new_version: Version, The version to promote. api_client: appengine_api_client.AppengineApiClient to use to make requests. """ metrics.CustomTimedEvent(metric_names.SET_DEFAULT_VERSION_API_START) # TODO(b/31824825): It sometimes takes a while for a new service to show up. # Retry it if we get a service not found error. def ShouldRetry(exc_type, unused_exc_value, unused_traceback, unused_state): return issubclass(exc_type, apitools_exceptions.HttpError) try: retryer = retry.Retryer(max_retrials=3, exponential_sleep_multiplier=2) retryer.RetryOnException(api_client.SetDefaultVersion, [new_version.service, new_version.id], should_retry_if=ShouldRetry, sleep_ms=1000) except retry.MaxRetrialsException as e: (unused_result, exc_info) = e.last_result if exc_info: exceptions.reraise(exc_info[1], tb=exc_info[2]) else: # This shouldn't happen, but if we don't have the exception info for some # reason, just convert the MaxRetrialsException. raise exceptions.InternalError() metrics.CustomTimedEvent(metric_names.SET_DEFAULT_VERSION_API)
def _HandleRemoveError(func, failed_path, exc_info): """A function to pass as the onerror arg to rmdir for handling errors. Args: func: function, The function that failed. failed_path: str, The path of the file the error occurred on. exc_info: sys.exc_info(), The current exception state. """ logging.debug('Handling file system error: %s, %s, %s', func, failed_path, exc_info) # Access denied on Windows. This happens when trying to delete a readonly # file. Change the permissions and retry the delete. # # In python 3.3+, WindowsError is an alias of OSError and exc_info[0] can be # a subclass of OSError. if (WindowsError and issubclass(exc_info[0], WindowsError) and getattr(exc_info[1], 'winerror', None) == 5): os.chmod(failed_path, stat.S_IWUSR) # Don't remove the trailing comma in the passed arg tuple. It indicates that # it is a tuple of 1, rather than a tuple of characters that will get expanded # by *args. if not _RetryOperation(exc_info, func, (failed_path, ), _ShouldRetryOperation): # Always raise the original error. exceptions.reraise(exc_info[1], tb=exc_info[2])
def HandleError(exc, command_path, known_error_handler=None): """Handles an error that occurs during command execution. It calls ConvertKnownError to convert exceptions to known types before processing. If it is a known type, it is printed nicely as as error. If not, it is raised as a crash. Args: exc: Exception, The original exception that occurred. command_path: str, The name of the command that failed (for error reporting). known_error_handler: f(exc): A function to process known errors. """ known_exc, print_error = ConvertKnownError(exc) if known_exc: msg = '({0}) {1}'.format(console_attr.SafeText(command_path), console_attr.SafeText(known_exc)) log.debug(msg, exc_info=sys.exc_info()) if print_error: log.error(msg) # Uncaught errors will be handled in gcloud_main. if known_error_handler: known_error_handler(exc) if properties.VALUES.core.print_handled_tracebacks.GetBool(): core_exceptions.reraise(exc) _Exit(known_exc) else: # Make sure any uncaught exceptions still make it into the log file. log.debug(console_attr.SafeText(exc), exc_info=sys.exc_info()) core_exceptions.reraise(exc)
def ListRepos(self): """Returns list of repos.""" request = self.messages.SourceProjectsReposListRequest(projectId=self._id) try: return self._service.List(request).repos except exceptions.HttpError as error: core_exceptions.reraise( base_exceptions.HttpException(GetHttpErrorMessage(error)))
def wrapper(*args, **kwargs): try: return function(*args, **kwargs) # pylint:disable=broad-except except Exception as e: # pylint:enable=broad-except core_exceptions.reraise( translate_error(e, translation_list, format_str))
def ListRepos(self): """Returns list of repos.""" request = self.messages.SourceProjectsReposListRequest( projectId=self._id) try: return self._service.List(request).repos except exceptions.HttpError as error: core_exceptions.reraise( base_exceptions.HttpException(GetHttpErrorMessage(error)))
def Run(self, args): """Default Run method implementation.""" flags.CheckParentFlags(args, parent_required=False) project_id = args.id if not project_id and args.name: candidate = command_lib_util.IdFromName(args.name) if candidate and console_io.PromptContinue( 'No project id provided.', 'Use [{}] as project id'.format(candidate), throw_if_unattended=True): project_id = candidate if not project_id: raise exceptions.RequiredArgumentException( 'PROJECT_ID', 'an id must be provided for the new project') project_ref = command_lib_util.ParseProject(project_id) labels = labels_util.ParseCreateArgs( args, projects_util.GetMessages().Project.LabelsValue) try: create_op = projects_api.Create( project_ref, display_name=args.name, parent=projects_api.ParentNameToResourceId( flags.GetParentFromFlags(args)), labels=labels) except apitools_exceptions.HttpConflictError: msg = ( 'Project creation failed. The project ID you specified is ' 'already in use by another project. Please try an alternative ' 'ID.') core_exceptions.reraise(exceptions.HttpException(msg)) log.CreatedResource(project_ref, is_async=True) create_op = operations.WaitForOperation(create_op) # Enable cloudapis.googleapis.com if args.enable_cloud_apis: log.debug('Enabling cloudapis.googleapis.com') services_client = apis.GetClientInstance('servicemanagement', 'v1') enable_operation = services_enable_api.EnableServiceApiCall( project_ref.Name(), 'cloudapis.googleapis.com') enable_operation_ref = resources.REGISTRY.Parse( enable_operation.name, collection='servicemanagement.operations') services_util.WaitForOperation(enable_operation_ref, services_client) if args.set_as_default: project_property = properties.FromString('core/project') properties.PersistProperty(project_property, project_id) log.status.Print( 'Updated property [core/project] to [{0}].'.format(project_id)) return operations.ExtractOperationResponse( create_op, apis.GetMessagesModule('cloudresourcemanager', 'v1').Project)
def RetryOnException(self, func, args=None, kwargs=None, should_retry_if=None, sleep_ms=None): """Retries the function if an exception occurs. Args: func: The function to call and retry. args: a sequence of positional arguments to be passed to func. kwargs: a dictionary of positional arguments to be passed to func. should_retry_if: func(exc_type, exc_value, exc_traceback, state) that returns True or False. sleep_ms: int or iterable for how long to wait between trials. Returns: Whatever the function returns. Raises: RetryException, WaitException: if function is retries too many times, or time limit is reached. """ args = args if args is not None else () kwargs = kwargs if kwargs is not None else {} def TryFunc(): try: return func(*args, **kwargs), None except: # pylint: disable=bare-except return None, sys.exc_info() if should_retry_if is None: should_retry = lambda x, s: x[1] is not None else: def ShouldRetryFunc(try_func_result, state): exc_info = try_func_result[1] if exc_info is None: # No exception, no reason to retry. return False return should_retry_if(exc_info[0], exc_info[1], exc_info[2], state) should_retry = ShouldRetryFunc result, exc_info = self.RetryOnResult(TryFunc, should_retry_if=should_retry, sleep_ms=sleep_ms) if exc_info: # Exception that was not retried was raised. Re-raise. exceptions.reraise(exc_info[1], tb=exc_info[2]) return result
def Run(self, args): """Returns the results for one completion.""" presentation_kwargs = args.resource_presentation_kwargs or {} with cache_util.GetCache(args.cache, create=True) as cache: log.info('cache name {}'.format(cache.name)) if not args.kwargs: args.kwargs = {} # Create the ResourceInfo object that is used to hook up the parameter # info to the argparse namespace for resource argument completers. if args.resource_spec_path: spec = _GetPresentationSpec(args.resource_spec_path, **presentation_kwargs) spec.required = False resource_info = concept_parsers.ConceptParser([spec]).GetInfo( spec.name) # Since the argument being completed doesn't have the correct # dest, make sure the handler always gives the same ResourceInfo # object. def ResourceInfoMonkeyPatch(*args, **kwargs): del args, kwargs return resource_info args.CONCEPTS.ArgNameToConceptInfo = ResourceInfoMonkeyPatch completer = _GetCompleter(args.module_path, cache=cache, qualify=args.qualify, resource_spec=args.resource_spec_path, presentation_kwargs=presentation_kwargs, attribute=args.attribute, **args.kwargs) parameter_info = completer.ParameterInfo( args, args.GetPositionalArgument('resource_to_complete')) if args.resource_to_complete is not None: matches = completer.Complete(args.resource_to_complete, parameter_info) return [matches] while True: name = console_io.PromptResponse('COMPLETE> ') if name is None: break try: completions = completer.Complete(name, parameter_info) except (Exception, SystemExit) as e: # pylint: disable=broad-except if args.stack_trace: exceptions.reraise(Exception(e)) else: log.error(six.text_type(e)) continue if completions: print('\n'.join(completions)) sys.stderr.write('\n') return None
def ListBuckets(self, fields_scope=None): """See super class.""" try: response = self.client.list_buckets() for bucket in response['Buckets']: yield resource_reference.BucketResource( storage_url.CloudUrl(storage_url.ProviderPrefix.S3, bucket['Name']), metadata={'Bucket': bucket, 'Owner': response['Owner']}) except botocore.exceptions.ClientError as error: core_exceptions.reraise(errors.S3ApiError(error))
def Wait(self): self.thread.join(self.timeout) if self.exc_info: exceptions.reraise(self.exc_info[1], tb=self.exc_info[2]) if self.thread.isAlive(): execution_utils.KillSubprocess(self.p) # Give the thread a chance to clean up if it can, now that we killed the # subprocess. self.thread.join(_ProcessRunner.fallback_timeout) timeout_message = 'The process timed out: {0}'.format( ' '.join(self.args)) raise TimeoutError(result=self.result, msg=timeout_message)
def Run(self, args): client = util.GetApiClientInstance() list_generator = list_pager.YieldFromList( service=client.projects_locations, request=self._BuildRequest(), field='locations', batch_size_attribute='pageSize') try: for item in list_generator: yield item except api_exceptions.HttpError as error: msg = util.GetHttpErrorMessage(error) exceptions.reraise(base_exceptions.HttpException(msg))
def Send(self, *args, **kwargs): try: response = self._server.Send(*args, **kwargs) log.debug('Got response: %s', response) return response except urllib.error.HTTPError as e: # This is the message body, if included in e if hasattr(e, 'read'): body = e.read() else: body = '' exceptions.reraise(RPCError(e, body=body))
def Send(self, *args, **kwargs): try: response = self._server.Send(*args, **kwargs) log.debug('Got response: %s', response) return response except urllib2.HTTPError as e: # This is the message body, if included in e if hasattr(e, 'read'): body = e.read() else: body = '' exceptions.reraise(RPCError(e, body=body))
def __exit__(self, prev_exc_type, prev_exc_val, prev_exc_trace): try: self.Close() except: # pylint: disable=bare-except if not prev_exc_type: raise message = ('Got exception {0}' 'while another exception was active {1} [{2}]'.format( encoding.Decode(traceback.format_exc()), prev_exc_type, encoding.Decode(prev_exc_val))) exceptions.reraise(prev_exc_type(message), tb=prev_exc_trace) # always return False so any exceptions will be re-raised return False
def list_objects(self, bucket_name, prefix=None, delimiter=None, all_versions=None, fields_scope=cloud_api.FieldsScope.NO_ACL): """See super class.""" projection = self._get_projection( fields_scope, self.messages.StorageObjectsListRequest) global_params = None if fields_scope == cloud_api.FieldsScope.SHORT: global_params = self.messages.StandardQueryParameters() global_params.fields = ( 'prefixes,items/name,items/size,items/generation,nextPageToken' ) object_list = None while True: apitools_request = self.messages.StorageObjectsListRequest( bucket=bucket_name, prefix=prefix, delimiter=delimiter, versions=all_versions, projection=projection, pageToken=object_list.nextPageToken if object_list else None, maxResults=cloud_api.NUM_ITEMS_PER_LIST_PAGE) try: object_list = self.client.objects.List( apitools_request, global_params=global_params) except apitools_exceptions.HttpError as e: core_exceptions.reraise( cloud_errors.translate_error(e, _ERROR_TRANSLATION)) # Yield objects. # TODO(b/160238394) Decrypt metadata fields if necessary. for object_metadata in object_list.items: object_metadata.bucket = bucket_name yield gcs_metadata_util.get_object_resource_from_metadata( object_metadata) # Yield prefixes. for prefix_string in object_list.prefixes: yield resource_reference.PrefixResource(storage_url.CloudUrl( scheme=storage_url.ProviderPrefix.GCS, bucket_name=bucket_name, object_name=prefix_string), prefix=prefix_string) if not object_list.nextPageToken: break
def list_buckets(self, fields_scope=cloud_api.FieldsScope.NO_ACL): """See super class.""" try: response = self.client.list_buckets() for bucket in response['Buckets']: if fields_scope == cloud_api.FieldsScope.FULL: yield self.get_bucket(bucket['Name'], fields_scope) else: yield s3_resource_reference.S3BucketResource( storage_url.CloudUrl( storage_url.ProviderPrefix.S3, bucket['Name']), metadata={'Bucket': bucket, 'Owner': response['Owner']}) except botocore.exceptions.ClientError as error: core_exceptions.reraise(errors.S3ApiError(error))
def DecoratedFunction(*args, **kwargs): retryer = Retryer( max_retrials=max_retrials, max_wait_ms=max_wait_ms, exponential_sleep_multiplier=exponential_sleep_multiplier, jitter_ms=jitter_ms, status_update_func=status_update_func) try: return retryer.RetryOnException(f, args=args, kwargs=kwargs, should_retry_if=should_retry_if, sleep_ms=sleep_ms) except MaxRetrialsException as mre: to_reraise = mre.last_result[1] exceptions.reraise(to_reraise[1], tb=to_reraise[2])
def Run(self, args): """Returns the parsed parameters for one resource.""" if args.api_version: api_name = args.collection.split('.')[0] resources.REGISTRY.RegisterApiByName(api_name, api_version=args.api_version) if args.resources_to_parse: parsed_resources = [] for uri in args.resources_to_parse: try: resource = resources.REGISTRY.Parse( uri, collection=args.collection) except (Exception, SystemExit) as e: # pylint: disable=broad-except if args.stack_trace: exceptions.reraise(e) log.error(six.text_type(e)) parsed_resources.append({ 'error': six.text_type(e), 'uri': uri, }) continue collection_info = resource.GetCollectionInfo() parsed_resources.append({ 'api_name': collection_info.api_name, 'api_version': collection_info.api_version, 'collection': collection_info.full_name, 'params': resource.AsDict(), 'uri': resource.SelfLink(), }) return parsed_resources while True: uri = console_io.PromptResponse('PARSE> ') if uri is None: break if not uri: continue try: params = resources.REGISTRY.Parse( uri, collection=args.collection).AsDict() except (Exception, SystemExit) as e: # pylint: disable=broad-except if args.stack_trace: exceptions.reraise(e) log.error(six.text_type(e)) continue resource_printer.Print(params, 'json') sys.stderr.write('\n') return None
def _YieldFromLocation(self, location_ref, limit, messages, client): list_generator = list_pager.YieldFromList( service=client.projects_locations_functions, request=self.BuildRequest(location_ref, messages), limit=limit, field='functions', batch_size_attribute='pageSize') # Decorators (e.g. util.CatchHTTPErrorRaiseHTTPException) don't work # for generators. We have to catch the exception above the iteration loop, # but inside the function. try: for item in list_generator: yield item except api_exceptions.HttpError as error: msg = util.GetHttpErrorMessage(error) exceptions.reraise(base_exceptions.HttpException(msg))
def Run(args): """Lists regions available with the given args.""" del args # unused by list command client = util.GetApiClientInstance() list_generator = list_pager.YieldFromList( service=client.projects_locations, request=_BuildRequest(), field='locations', batch_size_attribute='pageSize') try: for item in list_generator: yield item except api_exceptions.HttpError as error: msg = util.GetHttpErrorMessage(error) exceptions.reraise(base_exceptions.HttpException(msg))
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace, All the arguments that were provided to this command invocation. Returns: an Operation message which tracks the asynchronous import """ apitools_client = genomics_util.GetGenomicsClient() genomics_messages = genomics_util.GetGenomicsMessages() if not args.source_uris: raise GenomicsError( 'at least one value is required for --source-uris') partition_enum = (genomics_messages.ImportReadGroupSetsRequest. PartitionStrategyValueValuesEnum) partition_strat = None if args.partition_strategy: if args.partition_strategy not in partition_enum.to_dict(): raise GenomicsError( '--partition-strategy must be one of {0}; received: {1}'. format(sorted(partition_enum.names()), args.partition_strategy)) partition_strat = partition_enum.lookup_by_name( args.partition_strategy) try: return apitools_client.readgroupsets.Import( genomics_messages.ImportReadGroupSetsRequest( datasetId=args.dataset_id, sourceUris=args.source_uris, referenceSetId=args.reference_set_id, partitionStrategy=partition_strat, )) except apitools_exceptions.HttpError as error: # Map our error messages (JSON API camelCased) back into flag names. msg = ( exceptions.HttpException(error).payload.status_message.replace( 'datasetId', '--dataset-id').replace( 'partitionStrategy', '--partition-strategy').replace( 'sourceUris', '--source-uris').replace('referenceSetId', '--reference-set-id')) core_exceptions.reraise(exceptions.HttpException(msg))
def _IsStillRunning(pid): """Determines if the given pid is still running. Args: pid: The process id of the process to check. Returns: bool, True if it is still running. """ try: (actual_pid, code) = os.waitpid(pid, os.WNOHANG) if (actual_pid, code) == (0, 0): return True except OSError as error: if 'No child processes' not in error.strerror: exceptions.reraise(sys.exc_info()[1]) return False
def RetryOnException(self, func, args=None, kwargs=None, should_retry_if=None, sleep_ms=None): """Retries the function if an exception occurs. Args: func: The function to call and retry. args: a sequence of positional arguments to be passed to func. kwargs: a dictionary of positional arguments to be passed to func. should_retry_if: func(exc_type, exc_value, exc_traceback, state) that returns True or False. sleep_ms: int or iterable for how long to wait between trials. Returns: Whatever the function returns. Raises: RetryException, WaitException: if function is retries too many times, or time limit is reached. """ args = args if args is not None else () kwargs = kwargs if kwargs is not None else {} def TryFunc(): try: return func(*args, **kwargs), None except: # pylint: disable=bare-except return None, sys.exc_info() if should_retry_if is None: should_retry = lambda x, s: x[1] is not None else: def ShouldRetryFunc(try_func_result, state): exc_info = try_func_result[1] if exc_info is None: # No exception, no reason to retry. return False return should_retry_if(exc_info[0], exc_info[1], exc_info[2], state) should_retry = ShouldRetryFunc result, exc_info = self.RetryOnResult( TryFunc, should_retry_if=should_retry, sleep_ms=sleep_ms) if exc_info: # Exception that was not retried was raised. Re-raise. exceptions.reraise(exc_info[1], tb=exc_info[2]) return result
def ParseRequirementsFile(requirements_file_path): """Parses the given requirements file into a requirements dictionary. If the file path is GCS file path, use GCS file parser to parse requirements file. Otherwise, use local file parser. Args: requirements_file_path: Filepath to the requirements file. Returns: {string: string}, dict mapping from PyPI package name to extras and version specifier, if provided. Raises: Error: if requirements file cannot be read. """ try: is_gcs_file_path = requirements_file_path.startswith('gs://') if is_gcs_file_path: storage_client = storage_api.StorageClient() object_ref = storage_util.ObjectReference.FromUrl( requirements_file_path) file_content = storage_client.ReadObject(object_ref) else: file_content = files.FileReader(requirements_file_path) requirements = {} with file_content as requirements_file: for requirement_specifier in requirements_file: requirement_specifier = requirement_specifier.strip() if not requirement_specifier: continue package, version = SplitRequirementSpecifier( requirement_specifier) # Ensure package not already in entry list. if package in requirements: raise Error( 'Duplicate package in requirements file: {0}'.format( package)) requirements[package] = version return requirements except (files.Error, storage_api.Error, storage_util.Error): # Raise error when it fails to read requirements file. core_exceptions.reraise( Error('Unable to read requirements file {0}'.format( requirements_file_path)))
def _ApplySecretsArgsToFunction(function, args): """Populates cloud function message with secrets payload if applicable. It compares the CLI args with the existing secrets configuration to compute the effective secrets configuration. Args: function: Cloud function message to be checked and populated. args: All CLI arguments. Returns: updated_fields: update mask containing the list of fields to be updated. """ if not secrets_config.IsArgsSpecified(args): return [] old_secrets = secrets_util.GetSecretsAsDict( function.secretEnvironmentVariables, function.secretVolumes) new_secrets = {} try: new_secrets = secrets_config.ApplyFlags( old_secrets, args, _GetProject(), project_util.GetProjectNumber(_GetProject())) except ArgumentTypeError as error: exceptions.reraise(function_exceptions.FunctionsError(error)) if new_secrets: _LogSecretsPermissionMessage(_GetProject(), function.serviceAccountEmail) old_secret_env_vars, old_secret_volumes = secrets_config.SplitSecretsDict( old_secrets) new_secret_env_vars, new_secret_volumes = secrets_config.SplitSecretsDict( new_secrets) updated_fields = [] if old_secret_env_vars != new_secret_env_vars: function.secretEnvironmentVariables = secrets_util.SecretEnvVarsToMessages( new_secret_env_vars, api_util.GetApiMessagesModule()) updated_fields.append('secretEnvironmentVariables') if old_secret_volumes != new_secret_volumes: function.secretVolumes = secrets_util.SecretVolumesToMessages( new_secret_volumes, api_util.GetApiMessagesModule()) updated_fields.append('secretVolumes') return updated_fields
def _RawRequest(*args, **kwargs): def RetryIf(exc_type, exc_value, unused_traceback, unused_state): return exc_type == urllib.error.HTTPError and exc_value.code == 404 def StatusUpdate(unused_result, unused_state): log.debug('Retrying request...') retryer = retry.Retryer(max_retrials=3, exponential_sleep_multiplier=2, jitter_ms=100, status_update_func=StatusUpdate) try: return retryer.RetryOnException( url_opener.urlopen, args, kwargs, should_retry_if=RetryIf, sleep_ms=500) except retry.RetryException as e: # last_result is (return value, sys.exc_info) if e.last_result[1]: exceptions.reraise(e.last_result[1][1], tb=e.last_result[1][2]) raise
def _TryFunc(*args, **kwargs): try: return func(*args, **kwargs) # pytype: disable=missing-parameter except (OSError, IOError) as e: if e.errno == errno.EACCES: exceptions.reraise( PermissionsError(message=e.strerror, path=os.path.abspath(e.filename))) raise except shutil.Error as e: args = e.args[0][0] # unfortunately shutil.Error *only* has formatted strings to inspect. # Looking for this substring is looking for errno.EACCES, which has # a numeric value of 13. if args[2].startswith('[Errno 13]'): exceptions.reraise( PermissionsError(message=args[2], path=os.path.abspath(args[0]))) raise
def _KillPID(pid): """Kills the given process with SIGTERM, then with SIGKILL if it doesn't stop. Args: pid: The process id of the process to check. """ try: # Try sigterm first. os.kill(pid, signal.SIGTERM) # If still running, wait a few seconds to see if it dies. deadline = time.time() + 3 while time.time() < deadline: if not _IsStillRunning(pid): return time.sleep(0.1) # No luck, just force kill it. os.kill(pid, signal.SIGKILL) except OSError as error: if 'No such process' not in error.strerror: exceptions.reraise(sys.exc_info()[1])
def Wrapper(*args, **kwargs): try: return run_func(*args, **kwargs) except apitools_exceptions.HttpError as error: exc = HttpException(error, format_str) core_exceptions.reraise(exc)
def CatchHTTPErrorRaiseHTTPExceptionFn(*args, **kwargs): try: return func(*args, **kwargs) except apitools_exceptions.HttpError as error: core_exceptions.reraise( base_exceptions.HttpException(GetHttpErrorMessage(error)))
def TryFunc(*args, **kwargs): try: return func(*args, **kwargs) except error_types: core_exceptions.reraise(NewErrorFromCurrentException(error))
def _ReraiseMiscOperationError(err): """Transform and re-raise error helper.""" exceptions.reraise(MiscOperationError(str(err)))