def _GetModuleFromPath(impl_file, path, construction_id):
"""Import the module and dig into it to return the namespace we are after.
Import the module relative to the top level directory. Then return the
actual module corresponding to the last bit of the path.
Args:
impl_file: str, The path to the file this was loaded from (for error
reporting).
path: [str], A list of group names that got us down to this command group
with respect to the CLI itself. This path should be used for things
like error reporting when a specific element in the tree needs to be
referenced.
construction_id: str, A unique identifier for the CLILoader that is
being constructed.
Returns:
The imported module.
"""
# Make sure this module name never collides with any real module name.
# Use the CLI naming path, so values are always unique.
name_to_give = '__calliope__command__.{construction_id}.{name}'.format(
construction_id=construction_id,
name='.'.join(path).replace('-', '_'))
try:
return pkg_resources.GetModuleFromPath(name_to_give, impl_file)
# pylint:disable=broad-except, We really do want to catch everything here,
# because if any exceptions make it through for any single command or group
# file, the whole CLI will not work. Instead, just log whatever it is.
except Exception as e:
exceptions.reraise(CommandLoadFailure('.'.join(path), e))
def upload_object(self,
upload_stream,
upload_resource,
progress_callback=None,
request_config=None):
"""See super class."""
# TODO(b/160998556): Implement resumable upload.
del progress_callback
kwargs = {
'Bucket': upload_resource.storage_url.bucket_name,
'Key': upload_resource.storage_url.object_name,
'Body': upload_stream.read()
}
if request_config and request_config.predefined_acl_string:
kwargs['ACL'] = _translate_predefined_acl_string_to_s3(
request_config.predefined_acl_string)
try:
response = self.client.put_object(**kwargs)
return self._get_object_resource_from_s3_response(
response, upload_resource.storage_url.bucket_name,
upload_resource.storage_url.object_name)
except botocore.exceptions.ClientError as error:
core_exceptions.reraise(errors.S3ApiError(error))
def get_object_metadata(self,
bucket_name,
object_name,
generation=None,
fields_scope=None):
"""See super class."""
request = {'Bucket': bucket_name, 'Key': object_name}
# The VersionId keyword argument to head_object is not nullable if it is
# present, so only include it in the function call if it has a value.
if generation is not None:
request['VersionId'] = generation
try:
object_dict = self.client.head_object(**request)
except botocore.exceptions.ClientError as error:
core_exceptions.reraise(errors.S3ApiError(error))
# User requested ACL's with FieldsScope.FULL.
if fields_scope is cloud_api.FieldsScope.FULL:
try:
acl_response = self.client.get_object_acl(**request)
acl_response.pop('ResponseMetadata', None)
object_dict['ACL'] = acl_response
except botocore.exceptions.ClientError as error:
object_dict['ACL'] = errors.S3ApiError(error)
return self._get_object_resource_from_s3_response(
object_dict, bucket_name, object_name)
def list_buckets(self, fields_scope=cloud_api.FieldsScope.NO_ACL):
"""See super class."""
projection = self._get_projection(
fields_scope, self.messages.StorageBucketsListRequest)
request = self.messages.StorageBucketsListRequest(
project=properties.VALUES.core.project.GetOrFail(),
projection=projection)
global_params = None
if fields_scope == cloud_api.FieldsScope.SHORT:
global_params = self.messages.StandardQueryParameters()
global_params.fields = 'items/name'
# TODO(b/160238394) Decrypt metadata fields if necessary.
bucket_iter = list_pager.YieldFromList(
self.client.buckets,
request,
batch_size=cloud_api.NUM_ITEMS_PER_LIST_PAGE,
global_params=global_params)
try:
for bucket in bucket_iter:
yield gcs_metadata_util.get_bucket_resource_from_metadata(
bucket)
except apitools_exceptions.HttpError as e:
core_exceptions.reraise(
cloud_errors.translate_error(e, _ERROR_TRANSLATION))
def _SetDefaultVersion(new_version, api_client):
"""Sets the given version as the default.
Args:
new_version: Version, The version to promote.
api_client: appengine_api_client.AppengineApiClient to use to make requests.
"""
metrics.CustomTimedEvent(metric_names.SET_DEFAULT_VERSION_API_START)
# TODO(b/31824825): It sometimes takes a while for a new service to show up.
# Retry it if we get a service not found error.
def ShouldRetry(exc_type, unused_exc_value, unused_traceback, unused_state):
return issubclass(exc_type, apitools_exceptions.HttpError)
try:
retryer = retry.Retryer(max_retrials=3, exponential_sleep_multiplier=2)
retryer.RetryOnException(
api_client.SetDefaultVersion, [new_version.service, new_version.id],
should_retry_if=ShouldRetry, sleep_ms=1000)
except retry.MaxRetrialsException as e:
(unused_result, exc_info) = e.last_result
if exc_info:
exceptions.reraise(exc_info[1], tb=exc_info[2])
else:
# This shouldn't happen, but if we don't have the exception info for some
# reason, just convert the MaxRetrialsException.
raise exceptions.InternalError()
metrics.CustomTimedEvent(metric_names.SET_DEFAULT_VERSION_API)
def CopyObject(self,
source_resource,
destination_resource,
progress_callback=None,
request_config=None):
"""See super class."""
del progress_callback
source_kwargs = {'Bucket': source_resource.storage_url.bucket_name,
'Key': source_resource.storage_url.object_name}
if source_resource.storage_url.generation:
source_kwargs['VersionId'] = source_resource.storage_url.generation
kwargs = {'Bucket': destination_resource.storage_url.bucket_name,
'Key': destination_resource.storage_url.object_name,
'CopySource': source_kwargs}
if request_config and request_config.predefined_acl_string:
kwargs['ACL'] = _TranslatePredefinedAclStringToS3(
request_config.predefined_acl_string)
try:
response = self.client.copy_object(**kwargs)
return self._GetObjectResourceFromS3Response(
response, kwargs['Bucket'], kwargs['Key'])
except botocore.exceptions.ClientError as error:
core_exceptions.reraise(errors.S3ApiError(error))
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Returns:
an Operation message which tracks the asynchronous export
"""
apitools_client = genomics_util.GetGenomicsClient()
genomics_messages = genomics_util.GetGenomicsMessages()
try:
return apitools_client.readgroupsets.Export(
genomics_messages.GenomicsReadgroupsetsExportRequest(
readGroupSetId=args.read_group_set_id,
exportReadGroupSetRequest=genomics_messages.
ExportReadGroupSetRequest(
projectId=genomics_util.GetProjectId(),
exportUri=args.export_uri,
referenceNames=args.reference_names)))
except apitools_exceptions.HttpError as error:
# Map our error messages (JSON API camelCased) back into flag names.
msg = (
exceptions.HttpException(error).payload.status_message.replace(
'exportUri',
'--export-uri').replace('referenceNames',
'--reference-names'))
core_exceptions.reraise(exceptions.HttpException(msg))
def GetOrRaise(self):
if self.value:
return self.value[0] # self.value is a one-tuple
elif self.error:
raise self.error # pylint: disable=raising-bad-type
else:
exceptions.reraise(self.exc_info[1], tb=self.exc_info[2])
def GetOrRaise(self):
if self.value:
return self.value[0] # self.value is a one-tuple
elif self.error:
raise self.error # pylint: disable=raising-bad-type
else:
exceptions.reraise(self.exc_info[1], tb=self.exc_info[2])
def _SetDefaultVersion(new_version, api_client):
"""Sets the given version as the default.
Args:
new_version: Version, The version to promote.
api_client: appengine_api_client.AppengineApiClient to use to make requests.
"""
metrics.CustomTimedEvent(metric_names.SET_DEFAULT_VERSION_API_START)
# TODO(b/31824825): It sometimes takes a while for a new service to show up.
# Retry it if we get a service not found error.
def ShouldRetry(exc_type, unused_exc_value, unused_traceback,
unused_state):
return issubclass(exc_type, apitools_exceptions.HttpError)
try:
retryer = retry.Retryer(max_retrials=3, exponential_sleep_multiplier=2)
retryer.RetryOnException(api_client.SetDefaultVersion,
[new_version.service, new_version.id],
should_retry_if=ShouldRetry,
sleep_ms=1000)
except retry.MaxRetrialsException as e:
(unused_result, exc_info) = e.last_result
if exc_info:
exceptions.reraise(exc_info[1], tb=exc_info[2])
else:
# This shouldn't happen, but if we don't have the exception info for some
# reason, just convert the MaxRetrialsException.
raise exceptions.InternalError()
metrics.CustomTimedEvent(metric_names.SET_DEFAULT_VERSION_API)
def _GetModuleFromPath(impl_file, path, construction_id):
"""Import the module and dig into it to return the namespace we are after.
Import the module relative to the top level directory. Then return the
actual module corresponding to the last bit of the path.
Args:
impl_file: str, The path to the file this was loaded from (for error
reporting).
path: [str], A list of group names that got us down to this command group
with respect to the CLI itself. This path should be used for things
like error reporting when a specific element in the tree needs to be
referenced.
construction_id: str, A unique identifier for the CLILoader that is
being constructed.
Returns:
The imported module.
"""
# Make sure this module name never collides with any real module name.
# Use the CLI naming path, so values are always unique.
name_to_give = '__calliope__command__.{construction_id}.{name}'.format(
construction_id=construction_id, name='.'.join(path).replace('-', '_'))
try:
return pkg_resources.GetModuleFromPath(name_to_give, impl_file)
# pylint:disable=broad-except, We really do want to catch everything here,
# because if any exceptions make it through for any single command or group
# file, the whole CLI will not work. Instead, just log whatever it is.
except Exception as e:
exceptions.reraise(CommandLoadFailure('.'.join(path), e))
def _HandleRemoveError(func, failed_path, exc_info):
"""A function to pass as the onerror arg to rmdir for handling errors.
Args:
func: function, The function that failed.
failed_path: str, The path of the file the error occurred on.
exc_info: sys.exc_info(), The current exception state.
"""
logging.debug('Handling file system error: %s, %s, %s', func, failed_path,
exc_info)
# Access denied on Windows. This happens when trying to delete a readonly
# file. Change the permissions and retry the delete.
#
# In python 3.3+, WindowsError is an alias of OSError and exc_info[0] can be
# a subclass of OSError.
if (WindowsError and issubclass(exc_info[0], WindowsError)
and getattr(exc_info[1], 'winerror', None) == 5):
os.chmod(failed_path, stat.S_IWUSR)
# Don't remove the trailing comma in the passed arg tuple. It indicates that
# it is a tuple of 1, rather than a tuple of characters that will get expanded
# by *args.
if not _RetryOperation(exc_info, func,
(failed_path, ), _ShouldRetryOperation):
# Always raise the original error.
exceptions.reraise(exc_info[1], tb=exc_info[2])
def HandleError(exc, command_path, known_error_handler=None):
"""Handles an error that occurs during command execution.
It calls ConvertKnownError to convert exceptions to known types before
processing. If it is a known type, it is printed nicely as as error. If not,
it is raised as a crash.
Args:
exc: Exception, The original exception that occurred.
command_path: str, The name of the command that failed (for error
reporting).
known_error_handler: f(exc): A function to process known errors.
"""
known_exc, print_error = ConvertKnownError(exc)
if known_exc:
msg = '({0}) {1}'.format(console_attr.SafeText(command_path),
console_attr.SafeText(known_exc))
log.debug(msg, exc_info=sys.exc_info())
if print_error:
log.error(msg)
# Uncaught errors will be handled in gcloud_main.
if known_error_handler:
known_error_handler(exc)
if properties.VALUES.core.print_handled_tracebacks.GetBool():
core_exceptions.reraise(exc)
_Exit(known_exc)
else:
# Make sure any uncaught exceptions still make it into the log file.
log.debug(console_attr.SafeText(exc), exc_info=sys.exc_info())
core_exceptions.reraise(exc)
def ListRepos(self):
"""Returns list of repos."""
request = self.messages.SourceProjectsReposListRequest(projectId=self._id)
try:
return self._service.List(request).repos
except exceptions.HttpError as error:
core_exceptions.reraise(
base_exceptions.HttpException(GetHttpErrorMessage(error)))
def wrapper(*args, **kwargs):
try:
return function(*args, **kwargs)
# pylint:disable=broad-except
except Exception as e:
# pylint:enable=broad-except
core_exceptions.reraise(
translate_error(e, translation_list, format_str))
def ListRepos(self):
"""Returns list of repos."""
request = self.messages.SourceProjectsReposListRequest(
projectId=self._id)
try:
return self._service.List(request).repos
except exceptions.HttpError as error:
core_exceptions.reraise(
base_exceptions.HttpException(GetHttpErrorMessage(error)))
def Run(self, args):
"""Default Run method implementation."""
flags.CheckParentFlags(args, parent_required=False)
project_id = args.id
if not project_id and args.name:
candidate = command_lib_util.IdFromName(args.name)
if candidate and console_io.PromptContinue(
'No project id provided.',
'Use [{}] as project id'.format(candidate),
throw_if_unattended=True):
project_id = candidate
if not project_id:
raise exceptions.RequiredArgumentException(
'PROJECT_ID', 'an id must be provided for the new project')
project_ref = command_lib_util.ParseProject(project_id)
labels = labels_util.ParseCreateArgs(
args,
projects_util.GetMessages().Project.LabelsValue)
try:
create_op = projects_api.Create(
project_ref,
display_name=args.name,
parent=projects_api.ParentNameToResourceId(
flags.GetParentFromFlags(args)),
labels=labels)
except apitools_exceptions.HttpConflictError:
msg = (
'Project creation failed. The project ID you specified is '
'already in use by another project. Please try an alternative '
'ID.')
core_exceptions.reraise(exceptions.HttpException(msg))
log.CreatedResource(project_ref, is_async=True)
create_op = operations.WaitForOperation(create_op)
# Enable cloudapis.googleapis.com
if args.enable_cloud_apis:
log.debug('Enabling cloudapis.googleapis.com')
services_client = apis.GetClientInstance('servicemanagement', 'v1')
enable_operation = services_enable_api.EnableServiceApiCall(
project_ref.Name(), 'cloudapis.googleapis.com')
enable_operation_ref = resources.REGISTRY.Parse(
enable_operation.name,
collection='servicemanagement.operations')
services_util.WaitForOperation(enable_operation_ref,
services_client)
if args.set_as_default:
project_property = properties.FromString('core/project')
properties.PersistProperty(project_property, project_id)
log.status.Print(
'Updated property [core/project] to [{0}].'.format(project_id))
return operations.ExtractOperationResponse(
create_op,
apis.GetMessagesModule('cloudresourcemanager', 'v1').Project)
def RetryOnException(self,
func,
args=None,
kwargs=None,
should_retry_if=None,
sleep_ms=None):
"""Retries the function if an exception occurs.
Args:
func: The function to call and retry.
args: a sequence of positional arguments to be passed to func.
kwargs: a dictionary of positional arguments to be passed to func.
should_retry_if: func(exc_type, exc_value, exc_traceback, state) that
returns True or False.
sleep_ms: int or iterable for how long to wait between trials.
Returns:
Whatever the function returns.
Raises:
RetryException, WaitException: if function is retries too many times,
or time limit is reached.
"""
args = args if args is not None else ()
kwargs = kwargs if kwargs is not None else {}
def TryFunc():
try:
return func(*args, **kwargs), None
except: # pylint: disable=bare-except
return None, sys.exc_info()
if should_retry_if is None:
should_retry = lambda x, s: x[1] is not None
else:
def ShouldRetryFunc(try_func_result, state):
exc_info = try_func_result[1]
if exc_info is None:
# No exception, no reason to retry.
return False
return should_retry_if(exc_info[0], exc_info[1], exc_info[2],
state)
should_retry = ShouldRetryFunc
result, exc_info = self.RetryOnResult(TryFunc,
should_retry_if=should_retry,
sleep_ms=sleep_ms)
if exc_info:
# Exception that was not retried was raised. Re-raise.
exceptions.reraise(exc_info[1], tb=exc_info[2])
return result
def Run(self, args):
"""Returns the results for one completion."""
presentation_kwargs = args.resource_presentation_kwargs or {}
with cache_util.GetCache(args.cache, create=True) as cache:
log.info('cache name {}'.format(cache.name))
if not args.kwargs:
args.kwargs = {}
# Create the ResourceInfo object that is used to hook up the parameter
# info to the argparse namespace for resource argument completers.
if args.resource_spec_path:
spec = _GetPresentationSpec(args.resource_spec_path,
**presentation_kwargs)
spec.required = False
resource_info = concept_parsers.ConceptParser([spec]).GetInfo(
spec.name)
# Since the argument being completed doesn't have the correct
# dest, make sure the handler always gives the same ResourceInfo
# object.
def ResourceInfoMonkeyPatch(*args, **kwargs):
del args, kwargs
return resource_info
args.CONCEPTS.ArgNameToConceptInfo = ResourceInfoMonkeyPatch
completer = _GetCompleter(args.module_path,
cache=cache,
qualify=args.qualify,
resource_spec=args.resource_spec_path,
presentation_kwargs=presentation_kwargs,
attribute=args.attribute,
**args.kwargs)
parameter_info = completer.ParameterInfo(
args, args.GetPositionalArgument('resource_to_complete'))
if args.resource_to_complete is not None:
matches = completer.Complete(args.resource_to_complete,
parameter_info)
return [matches]
while True:
name = console_io.PromptResponse('COMPLETE> ')
if name is None:
break
try:
completions = completer.Complete(name, parameter_info)
except (Exception, SystemExit) as e: # pylint: disable=broad-except
if args.stack_trace:
exceptions.reraise(Exception(e))
else:
log.error(six.text_type(e))
continue
if completions:
print('\n'.join(completions))
sys.stderr.write('\n')
return None
def ListBuckets(self, fields_scope=None):
"""See super class."""
try:
response = self.client.list_buckets()
for bucket in response['Buckets']:
yield resource_reference.BucketResource(
storage_url.CloudUrl(storage_url.ProviderPrefix.S3,
bucket['Name']),
metadata={'Bucket': bucket, 'Owner': response['Owner']})
except botocore.exceptions.ClientError as error:
core_exceptions.reraise(errors.S3ApiError(error))
def Wait(self):
self.thread.join(self.timeout)
if self.exc_info:
exceptions.reraise(self.exc_info[1], tb=self.exc_info[2])
if self.thread.isAlive():
execution_utils.KillSubprocess(self.p)
# Give the thread a chance to clean up if it can, now that we killed the
# subprocess.
self.thread.join(_ProcessRunner.fallback_timeout)
timeout_message = 'The process timed out: {0}'.format(
' '.join(self.args))
raise TimeoutError(result=self.result, msg=timeout_message)
def Run(self, args):
client = util.GetApiClientInstance()
list_generator = list_pager.YieldFromList(
service=client.projects_locations,
request=self._BuildRequest(),
field='locations', batch_size_attribute='pageSize')
try:
for item in list_generator:
yield item
except api_exceptions.HttpError as error:
msg = util.GetHttpErrorMessage(error)
exceptions.reraise(base_exceptions.HttpException(msg))
def Send(self, *args, **kwargs):
try:
response = self._server.Send(*args, **kwargs)
log.debug('Got response: %s', response)
return response
except urllib.error.HTTPError as e:
# This is the message body, if included in e
if hasattr(e, 'read'):
body = e.read()
else:
body = ''
exceptions.reraise(RPCError(e, body=body))
def Send(self, *args, **kwargs):
try:
response = self._server.Send(*args, **kwargs)
log.debug('Got response: %s', response)
return response
except urllib2.HTTPError as e:
# This is the message body, if included in e
if hasattr(e, 'read'):
body = e.read()
else:
body = ''
exceptions.reraise(RPCError(e, body=body))
def __exit__(self, prev_exc_type, prev_exc_val, prev_exc_trace):
try:
self.Close()
except: # pylint: disable=bare-except
if not prev_exc_type:
raise
message = ('Got exception {0}'
'while another exception was active {1} [{2}]'.format(
encoding.Decode(traceback.format_exc()),
prev_exc_type, encoding.Decode(prev_exc_val)))
exceptions.reraise(prev_exc_type(message), tb=prev_exc_trace)
# always return False so any exceptions will be re-raised
return False
def list_objects(self,
bucket_name,
prefix=None,
delimiter=None,
all_versions=None,
fields_scope=cloud_api.FieldsScope.NO_ACL):
"""See super class."""
projection = self._get_projection(
fields_scope, self.messages.StorageObjectsListRequest)
global_params = None
if fields_scope == cloud_api.FieldsScope.SHORT:
global_params = self.messages.StandardQueryParameters()
global_params.fields = (
'prefixes,items/name,items/size,items/generation,nextPageToken'
)
object_list = None
while True:
apitools_request = self.messages.StorageObjectsListRequest(
bucket=bucket_name,
prefix=prefix,
delimiter=delimiter,
versions=all_versions,
projection=projection,
pageToken=object_list.nextPageToken if object_list else None,
maxResults=cloud_api.NUM_ITEMS_PER_LIST_PAGE)
try:
object_list = self.client.objects.List(
apitools_request, global_params=global_params)
except apitools_exceptions.HttpError as e:
core_exceptions.reraise(
cloud_errors.translate_error(e, _ERROR_TRANSLATION))
# Yield objects.
# TODO(b/160238394) Decrypt metadata fields if necessary.
for object_metadata in object_list.items:
object_metadata.bucket = bucket_name
yield gcs_metadata_util.get_object_resource_from_metadata(
object_metadata)
# Yield prefixes.
for prefix_string in object_list.prefixes:
yield resource_reference.PrefixResource(storage_url.CloudUrl(
scheme=storage_url.ProviderPrefix.GCS,
bucket_name=bucket_name,
object_name=prefix_string),
prefix=prefix_string)
if not object_list.nextPageToken:
break
def list_buckets(self, fields_scope=cloud_api.FieldsScope.NO_ACL):
"""See super class."""
try:
response = self.client.list_buckets()
for bucket in response['Buckets']:
if fields_scope == cloud_api.FieldsScope.FULL:
yield self.get_bucket(bucket['Name'], fields_scope)
else:
yield s3_resource_reference.S3BucketResource(
storage_url.CloudUrl(
storage_url.ProviderPrefix.S3, bucket['Name']),
metadata={'Bucket': bucket, 'Owner': response['Owner']})
except botocore.exceptions.ClientError as error:
core_exceptions.reraise(errors.S3ApiError(error))
def DecoratedFunction(*args, **kwargs):
retryer = Retryer(
max_retrials=max_retrials,
max_wait_ms=max_wait_ms,
exponential_sleep_multiplier=exponential_sleep_multiplier,
jitter_ms=jitter_ms,
status_update_func=status_update_func)
try:
return retryer.RetryOnException(f, args=args, kwargs=kwargs,
should_retry_if=should_retry_if,
sleep_ms=sleep_ms)
except MaxRetrialsException as mre:
to_reraise = mre.last_result[1]
exceptions.reraise(to_reraise[1], tb=to_reraise[2])
def DecoratedFunction(*args, **kwargs):
retryer = Retryer(
max_retrials=max_retrials,
max_wait_ms=max_wait_ms,
exponential_sleep_multiplier=exponential_sleep_multiplier,
jitter_ms=jitter_ms,
status_update_func=status_update_func)
try:
return retryer.RetryOnException(f, args=args, kwargs=kwargs,
should_retry_if=should_retry_if,
sleep_ms=sleep_ms)
except MaxRetrialsException as mre:
to_reraise = mre.last_result[1]
exceptions.reraise(to_reraise[1], tb=to_reraise[2])
def Run(self, args):
"""Returns the parsed parameters for one resource."""
if args.api_version:
api_name = args.collection.split('.')[0]
resources.REGISTRY.RegisterApiByName(api_name,
api_version=args.api_version)
if args.resources_to_parse:
parsed_resources = []
for uri in args.resources_to_parse:
try:
resource = resources.REGISTRY.Parse(
uri, collection=args.collection)
except (Exception, SystemExit) as e: # pylint: disable=broad-except
if args.stack_trace:
exceptions.reraise(e)
log.error(six.text_type(e))
parsed_resources.append({
'error': six.text_type(e),
'uri': uri,
})
continue
collection_info = resource.GetCollectionInfo()
parsed_resources.append({
'api_name': collection_info.api_name,
'api_version': collection_info.api_version,
'collection': collection_info.full_name,
'params': resource.AsDict(),
'uri': resource.SelfLink(),
})
return parsed_resources
while True:
uri = console_io.PromptResponse('PARSE> ')
if uri is None:
break
if not uri:
continue
try:
params = resources.REGISTRY.Parse(
uri, collection=args.collection).AsDict()
except (Exception, SystemExit) as e: # pylint: disable=broad-except
if args.stack_trace:
exceptions.reraise(e)
log.error(six.text_type(e))
continue
resource_printer.Print(params, 'json')
sys.stderr.write('\n')
return None
def _YieldFromLocation(self, location_ref, limit, messages, client):
list_generator = list_pager.YieldFromList(
service=client.projects_locations_functions,
request=self.BuildRequest(location_ref, messages),
limit=limit, field='functions',
batch_size_attribute='pageSize')
# Decorators (e.g. util.CatchHTTPErrorRaiseHTTPException) don't work
# for generators. We have to catch the exception above the iteration loop,
# but inside the function.
try:
for item in list_generator:
yield item
except api_exceptions.HttpError as error:
msg = util.GetHttpErrorMessage(error)
exceptions.reraise(base_exceptions.HttpException(msg))
def Run(args):
"""Lists regions available with the given args."""
del args # unused by list command
client = util.GetApiClientInstance()
list_generator = list_pager.YieldFromList(
service=client.projects_locations,
request=_BuildRequest(),
field='locations',
batch_size_attribute='pageSize')
try:
for item in list_generator:
yield item
except api_exceptions.HttpError as error:
msg = util.GetHttpErrorMessage(error)
exceptions.reraise(base_exceptions.HttpException(msg))
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Returns:
an Operation message which tracks the asynchronous import
"""
apitools_client = genomics_util.GetGenomicsClient()
genomics_messages = genomics_util.GetGenomicsMessages()
if not args.source_uris:
raise GenomicsError(
'at least one value is required for --source-uris')
partition_enum = (genomics_messages.ImportReadGroupSetsRequest.
PartitionStrategyValueValuesEnum)
partition_strat = None
if args.partition_strategy:
if args.partition_strategy not in partition_enum.to_dict():
raise GenomicsError(
'--partition-strategy must be one of {0}; received: {1}'.
format(sorted(partition_enum.names()),
args.partition_strategy))
partition_strat = partition_enum.lookup_by_name(
args.partition_strategy)
try:
return apitools_client.readgroupsets.Import(
genomics_messages.ImportReadGroupSetsRequest(
datasetId=args.dataset_id,
sourceUris=args.source_uris,
referenceSetId=args.reference_set_id,
partitionStrategy=partition_strat,
))
except apitools_exceptions.HttpError as error:
# Map our error messages (JSON API camelCased) back into flag names.
msg = (
exceptions.HttpException(error).payload.status_message.replace(
'datasetId', '--dataset-id').replace(
'partitionStrategy', '--partition-strategy').replace(
'sourceUris',
'--source-uris').replace('referenceSetId',
'--reference-set-id'))
core_exceptions.reraise(exceptions.HttpException(msg))
def _IsStillRunning(pid):
"""Determines if the given pid is still running.
Args:
pid: The process id of the process to check.
Returns:
bool, True if it is still running.
"""
try:
(actual_pid, code) = os.waitpid(pid, os.WNOHANG)
if (actual_pid, code) == (0, 0):
return True
except OSError as error:
if 'No child processes' not in error.strerror:
exceptions.reraise(sys.exc_info()[1])
return False
def RetryOnException(self, func, args=None, kwargs=None,
should_retry_if=None, sleep_ms=None):
"""Retries the function if an exception occurs.
Args:
func: The function to call and retry.
args: a sequence of positional arguments to be passed to func.
kwargs: a dictionary of positional arguments to be passed to func.
should_retry_if: func(exc_type, exc_value, exc_traceback, state) that
returns True or False.
sleep_ms: int or iterable for how long to wait between trials.
Returns:
Whatever the function returns.
Raises:
RetryException, WaitException: if function is retries too many times,
or time limit is reached.
"""
args = args if args is not None else ()
kwargs = kwargs if kwargs is not None else {}
def TryFunc():
try:
return func(*args, **kwargs), None
except: # pylint: disable=bare-except
return None, sys.exc_info()
if should_retry_if is None:
should_retry = lambda x, s: x[1] is not None
else:
def ShouldRetryFunc(try_func_result, state):
exc_info = try_func_result[1]
if exc_info is None:
# No exception, no reason to retry.
return False
return should_retry_if(exc_info[0], exc_info[1], exc_info[2], state)
should_retry = ShouldRetryFunc
result, exc_info = self.RetryOnResult(
TryFunc, should_retry_if=should_retry, sleep_ms=sleep_ms)
if exc_info:
# Exception that was not retried was raised. Re-raise.
exceptions.reraise(exc_info[1], tb=exc_info[2])
return result
def _IsStillRunning(pid):
"""Determines if the given pid is still running.
Args:
pid: The process id of the process to check.
Returns:
bool, True if it is still running.
"""
try:
(actual_pid, code) = os.waitpid(pid, os.WNOHANG)
if (actual_pid, code) == (0, 0):
return True
except OSError as error:
if 'No child processes' not in error.strerror:
exceptions.reraise(sys.exc_info()[1])
return False
def ParseRequirementsFile(requirements_file_path):
"""Parses the given requirements file into a requirements dictionary.
If the file path is GCS file path, use GCS file parser to parse requirements
file. Otherwise, use local file parser.
Args:
requirements_file_path: Filepath to the requirements file.
Returns:
{string: string}, dict mapping from PyPI package name to extras and version
specifier, if provided.
Raises:
Error: if requirements file cannot be read.
"""
try:
is_gcs_file_path = requirements_file_path.startswith('gs://')
if is_gcs_file_path:
storage_client = storage_api.StorageClient()
object_ref = storage_util.ObjectReference.FromUrl(
requirements_file_path)
file_content = storage_client.ReadObject(object_ref)
else:
file_content = files.FileReader(requirements_file_path)
requirements = {}
with file_content as requirements_file:
for requirement_specifier in requirements_file:
requirement_specifier = requirement_specifier.strip()
if not requirement_specifier:
continue
package, version = SplitRequirementSpecifier(
requirement_specifier)
# Ensure package not already in entry list.
if package in requirements:
raise Error(
'Duplicate package in requirements file: {0}'.format(
package))
requirements[package] = version
return requirements
except (files.Error, storage_api.Error, storage_util.Error):
# Raise error when it fails to read requirements file.
core_exceptions.reraise(
Error('Unable to read requirements file {0}'.format(
requirements_file_path)))
def _ApplySecretsArgsToFunction(function, args):
"""Populates cloud function message with secrets payload if applicable.
It compares the CLI args with the existing secrets configuration to compute
the effective secrets configuration.
Args:
function: Cloud function message to be checked and populated.
args: All CLI arguments.
Returns:
updated_fields: update mask containing the list of fields to be updated.
"""
if not secrets_config.IsArgsSpecified(args):
return []
old_secrets = secrets_util.GetSecretsAsDict(
function.secretEnvironmentVariables, function.secretVolumes)
new_secrets = {}
try:
new_secrets = secrets_config.ApplyFlags(
old_secrets, args, _GetProject(),
project_util.GetProjectNumber(_GetProject()))
except ArgumentTypeError as error:
exceptions.reraise(function_exceptions.FunctionsError(error))
if new_secrets:
_LogSecretsPermissionMessage(_GetProject(),
function.serviceAccountEmail)
old_secret_env_vars, old_secret_volumes = secrets_config.SplitSecretsDict(
old_secrets)
new_secret_env_vars, new_secret_volumes = secrets_config.SplitSecretsDict(
new_secrets)
updated_fields = []
if old_secret_env_vars != new_secret_env_vars:
function.secretEnvironmentVariables = secrets_util.SecretEnvVarsToMessages(
new_secret_env_vars, api_util.GetApiMessagesModule())
updated_fields.append('secretEnvironmentVariables')
if old_secret_volumes != new_secret_volumes:
function.secretVolumes = secrets_util.SecretVolumesToMessages(
new_secret_volumes, api_util.GetApiMessagesModule())
updated_fields.append('secretVolumes')
return updated_fields
def _RawRequest(*args, **kwargs):
def RetryIf(exc_type, exc_value, unused_traceback, unused_state):
return exc_type == urllib.error.HTTPError and exc_value.code == 404
def StatusUpdate(unused_result, unused_state):
log.debug('Retrying request...')
retryer = retry.Retryer(max_retrials=3, exponential_sleep_multiplier=2,
jitter_ms=100, status_update_func=StatusUpdate)
try:
return retryer.RetryOnException(
url_opener.urlopen, args, kwargs,
should_retry_if=RetryIf, sleep_ms=500)
except retry.RetryException as e:
# last_result is (return value, sys.exc_info)
if e.last_result[1]:
exceptions.reraise(e.last_result[1][1], tb=e.last_result[1][2])
raise
def _TryFunc(*args, **kwargs):
try:
return func(*args, **kwargs) # pytype: disable=missing-parameter
except (OSError, IOError) as e:
if e.errno == errno.EACCES:
exceptions.reraise(
PermissionsError(message=e.strerror,
path=os.path.abspath(e.filename)))
raise
except shutil.Error as e:
args = e.args[0][0]
# unfortunately shutil.Error *only* has formatted strings to inspect.
# Looking for this substring is looking for errno.EACCES, which has
# a numeric value of 13.
if args[2].startswith('[Errno 13]'):
exceptions.reraise(
PermissionsError(message=args[2],
path=os.path.abspath(args[0])))
raise
def _KillPID(pid):
"""Kills the given process with SIGTERM, then with SIGKILL if it doesn't stop.
Args:
pid: The process id of the process to check.
"""
try:
# Try sigterm first.
os.kill(pid, signal.SIGTERM)
# If still running, wait a few seconds to see if it dies.
deadline = time.time() + 3
while time.time() < deadline:
if not _IsStillRunning(pid):
return
time.sleep(0.1)
# No luck, just force kill it.
os.kill(pid, signal.SIGKILL)
except OSError as error:
if 'No such process' not in error.strerror:
exceptions.reraise(sys.exc_info()[1])
def Wrapper(*args, **kwargs):
try:
return run_func(*args, **kwargs)
except apitools_exceptions.HttpError as error:
exc = HttpException(error, format_str)
core_exceptions.reraise(exc)
def CatchHTTPErrorRaiseHTTPExceptionFn(*args, **kwargs):
try:
return func(*args, **kwargs)
except apitools_exceptions.HttpError as error:
core_exceptions.reraise(
base_exceptions.HttpException(GetHttpErrorMessage(error)))
def TryFunc(*args, **kwargs):
try:
return func(*args, **kwargs)
except error_types:
core_exceptions.reraise(NewErrorFromCurrentException(error))
def _ReraiseMiscOperationError(err): """Transform and re-raise error helper.""" exceptions.reraise(MiscOperationError(str(err)))
