def _ServiceFlagCompletionCallback(list_type, project):
"""Callback function for service tab-completion.
Args:
list_type: str, should be one of 'produced', 'enabled', or 'available'
project: str, the name of the project for which to retrieve candidates
Returns:
The list of arguments that the gcloud infrastructure will use to retrieve
candidate services.
"""
# Sanity check the type of list_type argument
if not isinstance(list_type, basestring):
raise exceptions.InternalError(
'Could not read list type in service flag completion callback.')
# Sanity check the list_type contents
if list_type not in ['produced', 'enabled']:
raise exceptions.InternalError(
'Invalid list type in service flag completion callback.')
result = [
'service-management', 'list',
'--%s' % list_type, '--format=value(serviceConfig.name)'
]
if project:
result.extend(['--project', project])
return result
def _SetDefaultVersion(new_version, api_client):
"""Sets the given version as the default.
Args:
new_version: Version, The version to promote.
api_client: appengine_api_client.AppengineApiClient to use to make requests.
"""
metrics.CustomTimedEvent(metric_names.SET_DEFAULT_VERSION_API_START)
# TODO(b/31824825): It sometimes takes a while for a new service to show up.
# Retry it if we get a service not found error.
def ShouldRetry(exc_type, unused_exc_value, unused_traceback,
unused_state):
return issubclass(exc_type, core_api_exceptions.HttpException)
try:
retryer = retry.Retryer(max_retrials=3, exponential_sleep_multiplier=2)
retryer.RetryOnException(api_client.SetDefaultVersion,
[new_version.service, new_version.id],
should_retry_if=ShouldRetry,
sleep_ms=1000)
except retry.MaxRetrialsException as e:
(unused_result, exc_info) = e.last_result
if exc_info:
# This is the 3 tuple of the last exception the function threw.
raise exc_info[0], exc_info[1], exc_info[2]
else:
# This shouldn't happen, but if we don't have the exception info for some
# reason, just convert the MaxRetrialsException.
raise exceptions.InternalError()
metrics.CustomTimedEvent(metric_names.SET_DEFAULT_VERSION_API)
def Run(self, args):
if not grpc_available:
raise core_exceptions.InternalError('gRPC is not available')
channel = grpc_util.MakeSecureChannel(
'bigtableadmin.googleapis.com:443')
instances = args.instances
results = []
for instance in instances:
instance_ref = resources.REGISTRY.Parse(
instance,
params={
'projectsId': properties.VALUES.core.project.GetOrFail
},
collection='bigtableadmin.projects.instances')
stub = bigtable_table_admin_pb2_grpc.BigtableTableAdminStub(
channel)
request = bigtable_table_admin_pb2.ListTablesRequest(
parent=instance_ref.RelativeName(), )
for table in grpc_util.YieldFromList(stub.ListTables,
request,
items_field='tables'):
results.append(table)
return results
def Run(self, args):
"""Returns a list of backendServiceGroupHealth objects."""
self.backend_service_ref = self.CreateGlobalReference(
args.name, resource_type='backendServices')
backend_service = self.GetBackendService(args)
if not backend_service.backends:
return
# Call GetHealth for each group in the backend service
requests = []
for backend in backend_service.backends:
request_message = self.messages.ComputeBackendServicesGetHealthRequest(
resourceGroupReference=self.messages.ResourceGroupReference(
group=backend.group),
project=self.project,
backendService=self.backend_service_ref.Name())
requests.append((self.service, 'GetHealth', request_message))
# Instead of batching-up all requests and making a single
# request_helper.MakeRequests call, go one backend at a time.
# We do this because getHealth responses don't say what resource
# they correspond to. It's not obvious how to reliably match up
# responses and backends when there are errors. Addtionally the contract
# for MakeRequests doesn't guarantee response order will match
# request order.
#
# TODO(b/25015230) Simply make a batch request once the response
# gives more information.
errors = []
for request in requests:
# The list() call below is itended to force the generator returned by
# MakeRequests. If there are exceptions the command will abort, which is
# expected. Having a list simplifies some of the checks that follow.
resources = list(
request_helper.MakeRequests(requests=[request],
http=self.http,
batch_url=self.batch_url,
errors=errors,
custom_get_requests=None))
if len(resources) is 0:
# Request failed, error information will accumulate in errors
continue
try:
[resource] = resources
except ValueError:
# Intended to throw iff resources contains more than one element. Just
# want to avoid a user potentially seeing an index out of bounds
# exception.
raise exceptions.InternalError('Invariant failure')
yield {
'backend': request[2].resourceGroupReference.group,
'status': resource
}
if errors:
utils.RaiseToolException(
errors, error_message='Could not get health for some groups:')
def ActivateNamedConfig(name):
"""Activates an existing named configuration."""
_ValidateConfigNameOrRaise(name)
configs = tuple(c.name for c in ListNamedConfigs())
if name not in configs + _RESERVED_NAMED_CONFIG_NAMES:
raise NamedConfigWriteError(
'Activating named configuration failed because configuration '
'[{0}] cannot be found.'.format(name))
config_path = GetPathForConfigName(name)
if not IsPathReadable(config_path):
raise NamedConfigWriteError(
'Activating named configuration failed because configuration '
'file [{0}] is missing or cannot be read.'.format(config_path))
_EnsureDir(config.Paths().global_config_dir)
activator_path = config.Paths().named_config_activator_path
try:
with open(activator_path, 'w') as f:
f.write(name)
except IOError as exp:
raise NamedConfigWriteError(
'Activating named configuration failed when writing '
'file [{0}] because [{1}]'.format(activator_path,
_OSErrorReason(exp)))
if ReadActivatorFile(silent=True) != name:
# Fail rather than erronously report success. Should be dead code.
raise exceptions.InternalError('Configuration creation or activation '
'failed for an unknown reason.')
def _GetEventTriggerEventParams(trigger_event, trigger_resource):
"""Get the args for creating an event trigger.
Args:
trigger_event: The trigger event
trigger_resource: The trigger resource
Returns:
A dictionary containing trigger_provider, trigger_event, and
trigger_resource.
"""
trigger_provider = triggers.INPUT_TRIGGER_PROVIDER_REGISTRY.ProviderForEvent(
trigger_event).label
resource_type = triggers.INPUT_TRIGGER_PROVIDER_REGISTRY.Event(
trigger_provider, trigger_event).resource_type
if resource_type == triggers.Resources.TOPIC:
trigger_resource = api_util.ValidatePubsubTopicNameOrRaise(
trigger_resource)
elif resource_type == triggers.Resources.BUCKET:
trigger_resource = storage_util.BucketReference.FromBucketUrl(
trigger_resource).bucket
elif resource_type == triggers.Resources.PROJECT:
if trigger_resource:
properties.VALUES.core.project.Validate(trigger_resource)
else:
# Check if programmer allowed other methods in
# api_util.PROVIDER_EVENT_RESOURCE but forgot to update code here
raise core_exceptions.InternalError()
# checked if provided resource and path have correct format
return {
'trigger_provider': trigger_provider,
'trigger_event': trigger_event,
'trigger_resource': trigger_resource,
}
def _CheckTriggerProviderArgs(args):
"""Check --trigger-provider dependent arguments and deduce if possible.
0. Check if --trigger-provider is correct.
1. Check if --trigger-event is present, assign default if not.
2. Check if --trigger-event is correct WRT to --trigger-provider.
3. Check if --trigger-resource is present if necessary.
4. Check if --trigger-resource is correct WRT to *-provider and *-event.
5. Check if --trigger-path is present if necessary.
6. Check if --trigger-path is not present if forbidden.
7. Check if --trigger-path is correct if present.
Args:
args: The argument namespace.
Returns:
None, when using HTTPS trigger. Otherwise a dictionary containing
trigger_provider, trigger_event, and trigger_resource.
"""
if args.trigger_http:
return None
if args.trigger_bucket:
return _BucketTrigger(args.trigger_bucket)
if args.trigger_topic:
return _TopicTrigger(args.trigger_topic)
if args.trigger_provider is None:
return None
trigger_provider = args.trigger_provider
trigger_event = args.trigger_event
trigger_resource = args.trigger_resource
# check and infer correct usage of flags accompanying --trigger-provider
if trigger_event is None:
trigger_event = util.input_trigger_provider_registry.Provider(
trigger_provider).default_event.label
resource_type = util.input_trigger_provider_registry.Event(
trigger_provider, trigger_event).resource_type
if resource_type == util.Resources.TOPIC:
trigger_resource = util.ValidatePubsubTopicNameOrRaise(
trigger_resource)
elif resource_type == util.Resources.BUCKET:
trigger_resource = storage_util.BucketReference.FromBucketUrl(
trigger_resource).bucket
elif resource_type == util.Resources.PROJECT:
if trigger_resource:
properties.VALUES.core.project.Validate(trigger_resource)
else:
# Check if programmer allowed other methods in
# util.PROVIDER_EVENT_RESOURCE but forgot to update code here
raise core_exceptions.InternalError()
# checked if provided resource and path have correct format
return {
'trigger_provider': trigger_provider,
'trigger_event': trigger_event,
'trigger_resource': trigger_resource,
}
def _RaiseIfFailed(self):
if self._pull_future.done():
e = self._pull_future.exception()
if e:
raise SubscribeOperationException(
'Subscribe operation failed with error: {error}'.format(
error=e))
log.debug(
'The streaming pull future completed unexpectedly without '
'raising an exception.')
raise exceptions.InternalError(
'The subscribe stream terminated unexpectedly.')
def _CheckTriggerEventArgs(args):
"""Check --trigger-* arguments and deduce if possible.
0. if --trigger-http is return None.
1. if --trigger-bucket return bucket trigger args (_BucketTrigger)
2. if --trigger-topic return pub-sub trigger args (_TopicTrigger)
3. if --trigger-event, deduce provider and resource from registry and return
Args:
args: The argument namespace.
Returns:
None, when using HTTPS trigger. Otherwise a dictionary containing
trigger_provider, trigger_event, and trigger_resource.
"""
if args.trigger_http:
return None
if args.trigger_bucket:
return _BucketTrigger(args.trigger_bucket)
if args.trigger_topic:
return _TopicTrigger(args.trigger_topic)
if not args.trigger_event:
return None
trigger_event = args.trigger_event
trigger_provider = util.input_trigger_provider_registry.ProviderForEvent(
trigger_event).label
trigger_resource = args.trigger_resource
resource_type = util.input_trigger_provider_registry.Event(
trigger_provider, trigger_event).resource_type
if resource_type == util.Resources.TOPIC:
trigger_resource = util.ValidatePubsubTopicNameOrRaise(
trigger_resource)
elif resource_type == util.Resources.BUCKET:
trigger_resource = storage_util.BucketReference.FromBucketUrl(
trigger_resource).bucket
elif resource_type == util.Resources.PROJECT:
if trigger_resource:
properties.VALUES.core.project.Validate(trigger_resource)
else:
# Check if programmer allowed other methods in
# util.PROVIDER_EVENT_RESOURCE but forgot to update code here
raise core_exceptions.InternalError()
# checked if provided resource and path have correct format
return {
'trigger_provider': trigger_provider,
'trigger_event': trigger_event,
'trigger_resource': trigger_resource,
}
def UpdateCache(self, operation, uris):
"""Updates the cache using operation on uris.
Args:
operation: AddToCacheOp, DeleteFromCacheOp, or ReplaceCacheOp.
uris: The list of uris for the operation.
Raises:
InternalError: if operation is invalid.
"""
if operation not in (AddToCacheOp, DeleteFromCacheOp, ReplaceCacheOp):
raise exceptions.InternalError(
'RemoteCompletion.UpdateCache operation [{0}] must be an '
'_UpdateCacheOp.'.format(operation))
operation().UpdateCache(self, uris)
def _CheckTriggerProviderArgs(args):
"""Check --trigger-provider dependent arguments and deduce if possible.
0. Check if --trigger-provider is correct.
1. Check if --trigger-event is present, assign default if not.
2. Check if --trigger-event is correct WRT to --trigger-provider.
3. Check if --trigger-resource is present if necessary.
4. Check if --trigger-resource is correct WRT to *-provider and *-event.
5. Check if --trigger-path is present if necessary.
6. Check if --trigger-path is not present if forbidden.
7. Check if --trigger-path is correct if present.
Args:
args: The argument namespace.
Returns:
args with all implicit information turned into explicit form.
"""
# Create a copy of namespace (copy.copy doesn't work here)
result = argparse.Namespace(**vars(args))
# check and infer correct usage of flags accompanying --trigger-provider
if result.trigger_event is None:
result.trigger_event = util.trigger_provider_registry.Provider(
result.trigger_provider).default_event.label
elif result.trigger_event not in util.trigger_provider_registry.EventsLabels(
result.trigger_provider):
raise exceptions.FunctionsError('You can use only one of [' + ','.join(
util.trigger_provider_registry.EventsLabels(
result.trigger_provider)) + '] with --trigger-provider=' +
result.trigger_provider)
# checked if Event Type is correct
if result.trigger_resource is None and util.trigger_provider_registry.Event(
result.trigger_provider,
result.trigger_event).resource_type != util.Resources.PROJECT:
raise exceptions.FunctionsError(
'You must provide --trigger-resource when using '
'--trigger-provider={0} and --trigger-event={1}'.format(
result.trigger_provider, result.trigger_event))
path_allowance = util.trigger_provider_registry.Event(
result.trigger_provider, result.trigger_event).path_obligatoriness
if result.trigger_path is None and (path_allowance
== util.Obligatoriness.REQUIRED):
raise exceptions.FunctionsError(
'You must provide --trigger-path when using '
'--trigger-provider={0} and --trigger-event={1}'.format(
result.trigger_provider, result.trigger_event))
if result.trigger_path is not None and (path_allowance
== util.Obligatoriness.FORBIDDEN):
raise exceptions.FunctionsError(
'You must not provide --trigger-path when using '
'--trigger-provider={0} and --trigger-event={1}'.format(
result.trigger_provider, result.trigger_event))
# checked if Resource Type and Path were provided or not as required
resource_type = util.trigger_provider_registry.Event(
result.trigger_provider, result.trigger_event).resource_type
if resource_type == util.Resources.TOPIC:
result.trigger_resource = util.ValidatePubsubTopicNameOrRaise(
result.trigger_resource)
elif resource_type == util.Resources.BUCKET:
result.trigger_resource = storage_util.BucketReference.FromBucketUrl(
result.trigger_resource).bucket
elif resource_type == util.Resources.PROJECT:
if result.trigger_resource:
properties.VALUES.core.project.Validate(result.trigger_resource)
else:
# Check if programmer allowed other methods in
# util.PROVIDER_EVENT_RESOURCE but forgot to update code here
raise core_exceptions.InternalError()
if result.trigger_path is not None:
util.ValidatePathOrRaise(result.trigger_path)
# checked if provided resource and path have correct format
return result
def _CheckTriggerProviderArgs(args):
"""Check --trigger-provider dependent arguments and deduce if possible.
0. Check if --trigger-provider is correct.
1. Check if --trigger-event is present, assign default if not.
2. Check if --trigger-event is correct WRT to --trigger-provider.
3. Check if --trigger-resource is present if necessary.
4. Check if --trigger-resource is correct WRT to *-provider and *-event.
5. Check if --trigger-path is present if necessary.
6. Check if --trigger-path is not present if forbidden.
7. Check if --trigger-path is correct if present.
Args:
args: The argument namespace.
Returns:
None, when using HTTPS trigger. Otherwise a dictionary containing
trigger_provider, trigger_event, and trigger_resource.
"""
if args.trigger_http:
return None
if args.trigger_bucket:
return _BucketTrigger(args.trigger_bucket)
if args.trigger_topic:
return _TopicTrigger(args.trigger_topic)
# TODO(b/36020181): move validation to a separate function.
trigger_provider = args.trigger_provider
trigger_event = args.trigger_event
trigger_resource = args.trigger_resource
# check and infer correct usage of flags accompanying --trigger-provider
if trigger_event is None:
trigger_event = util.input_trigger_provider_registry.Provider(
trigger_provider).default_event.label
elif trigger_event not in util.input_trigger_provider_registry.EventsLabels(
trigger_provider):
raise exceptions.FunctionsError('You can use only one of [' + ','.join(
util.input_trigger_provider_registry.EventsLabels(
trigger_provider)) + '] with --trigger-provider=' +
trigger_provider)
# checked if Event Type is correct
if trigger_resource is None and util.input_trigger_provider_registry.Event(
trigger_provider,
trigger_event).resource_type != util.Resources.PROJECT:
raise exceptions.FunctionsError(
'You must provide --trigger-resource when using '
'--trigger-provider={0} and --trigger-event={1}'.format(
trigger_provider, trigger_event))
# checked if Resource Type and Path were provided or not as required
resource_type = util.input_trigger_provider_registry.Event(
trigger_provider, trigger_event).resource_type
if resource_type == util.Resources.TOPIC:
trigger_resource = util.ValidatePubsubTopicNameOrRaise(
trigger_resource)
elif resource_type == util.Resources.BUCKET:
trigger_resource = storage_util.BucketReference.FromBucketUrl(
trigger_resource).bucket
elif resource_type == util.Resources.PROJECT:
if trigger_resource:
properties.VALUES.core.project.Validate(trigger_resource)
else:
# Check if programmer allowed other methods in
# util.PROVIDER_EVENT_RESOURCE but forgot to update code here
raise core_exceptions.InternalError()
# checked if provided resource and path have correct format
return {
'trigger_provider': trigger_provider,
'trigger_event': trigger_event,
'trigger_resource': trigger_resource,
}
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Returns:
None
Raises:
bigqueryError.BigqueryError: If the source and destination files are not
both specified.
calliope_exceptions.ToolException: If user cancels this operation.
Exception: If an unexpected value for the --if-exists flag passed gcloud
validation (which should never happen)
"""
apitools_client = self.context[commands.APITOOLS_CLIENT_KEY]
bigquery_messages = self.context[commands.BIGQUERY_MESSAGES_MODULE_KEY]
resource_parser = self.context[commands.BIGQUERY_REGISTRY_KEY]
project_id = properties.VALUES.core.project.Get(required=True)
source_reference = resource_parser.Parse(args.source,
collection='bigquery.tables')
source_reference_message = message_conversions.TableResourceToReference(
bigquery_messages, source_reference)
destination_resource = resource_parser.Parse(
args.destination, collection='bigquery.tables')
destination_reference = message_conversions.TableResourceToReference(
bigquery_messages, destination_resource)
if args.if_exists == 'append':
write_disposition = 'WRITE_APPEND'
ignore_already_exists = True
elif args.if_exists == 'fail':
write_disposition = 'WRITE_EMPTY'
ignore_already_exists = False
elif args.if_exists == 'prompt':
write_disposition = 'WRITE_TRUNCATE'
ignore_already_exists = False
if bigquery_client_helper.TableExists(apitools_client,
bigquery_messages,
destination_reference):
if not console_io.PromptContinue(prompt_string='Replace {0}'.
format(destination_resource)):
raise calliope_exceptions.ToolException('canceled by user')
elif args.if_exists == 'replace':
write_disposition = 'WRITE_TRUNCATE'
ignore_already_exists = False
elif args.if_exists == 'skip':
if bigquery_client_helper.TableExists(apitools_client,
bigquery_messages,
destination_reference):
return
else:
# This should be unreachable.
raise core_exceptions.InternalError(
'Unexpected value "{0}" for --if-exists flag.'.format(
args.if_exists))
copy_config = bigquery_messages.JobConfigurationTableCopy(
sourceTable=source_reference_message,
destinationTable=destination_reference,
writeDisposition=write_disposition)
job_id = job_ids.JobIdProvider().GetJobId(args.job_id,
args.fingerprint_job_id)
try:
job = job_control.ExecuteJob(
apitools_client,
bigquery_messages,
args,
configuration=bigquery_messages.JobConfiguration(
copy=copy_config),
project_id=project_id,
job_id=job_id)
except bigquery.DuplicateError as e:
if ignore_already_exists:
job = None
else:
raise e
if job is None:
log.status.Print('Table "{0}" already exists, skipping'.format(
destination_resource))
elif args. async:
registry = self.context[commands.BIGQUERY_REGISTRY_KEY]
job_resource = registry.Create(
'bigquery.jobs',
projectId=job.jobReference.projectId,
jobId=job.jobReference.jobId)
log.CreatedResource(job_resource)
else:
log.status.Print('Table [{0}] successfully copied to [{1}]'.format(
source_reference, destination_resource))
def PersistProperty(prop, value, scope=None, properties_file=None):
"""Sets the given property in the properties file.
This function should not generally be used as part of normal program
execution. The property files are user editable config files that they should
control. This is mostly for initial setup of properties that get set during
SDK installation.
Args:
prop: properties.Property, The property to set.
value: str, The value to set for the property. If None, the property is
removed.
scope: Scope, The config location to set the property in. If given, only
this location will be udpated and it is an error if that location does
not exist. If not given, it will attempt to update the property in the
first of the following places that exists:
- the workspace config
- the active named config
- user level config
It will never fall back to installation properties; you must
use that scope explicitly to set that value.
properties_file: str, Path to an explicit properties file to use (instead of
one of the known locations). It is an error to specify a scope and an
explicit file.
Raises:
ValueError: If you give both a scope and a properties file.
MissingConfigLocationError: If there is not file for the given scope.
ReadOnlyNamedConfigNotSettableError: If the user is attempting to set
a property in a read-only configuration.
InternalError: If there's a programming error.
"""
prop.Validate(value)
if scope and properties_file:
raise ValueError(
'You cannot provide both a scope and a specific properties'
' file.')
if not properties_file:
if scope:
if scope == Scope.INSTALLATION:
config.EnsureSDKWriteAccess()
properties_file = scope.get_file()
if not properties_file:
raise MissingConfigLocationError(scope)
else:
properties_file = Scope.WORKSPACE.get_file()
if not properties_file:
properties_file = named_configs.GetEffectiveNamedConfigFile()
if properties_file is None:
# Should be dead code.
raise exceptions.InternalError(
'Unexpected None properties file.')
if properties_file == os.path.devnull:
# Refuse to write and fail with an informative error
# TODO(b/22817095) Simplify control flow and update
# messaging when moving to automatic upgrade scenario
# on all release tracks.
if (named_configs.GetNameOfActiveNamedConfig() ==
named_configs.RESERVED_NAMED_CONFIG_NAME_NONE):
raise ReadOnlyNamedConfigNotSettableError(
named_configs.RESERVED_NAMED_CONFIG_NAME_NONE)
if not Scope.USER.get_file():
raise MissingConfigLocationError(Scope.USER)
parsed_config = ConfigParser.ConfigParser()
parsed_config.read(properties_file)
if not parsed_config.has_section(prop.section):
if value is None:
return
parsed_config.add_section(prop.section)
if value is None:
parsed_config.remove_option(prop.section, prop.name)
else:
parsed_config.set(prop.section, prop.name, str(value))
properties_dir, unused_name = os.path.split(properties_file)
files.MakeDir(properties_dir)
with open(properties_file, 'w') as fp:
parsed_config.write(fp)
PropertiesFile.Invalidate()