def Run(self, args):
self.ref = self.CreateReference(args)
get_request = self.GetGetRequest(args)
errors = []
objects = list(request_helper.MakeRequests(
requests=[get_request],
http=self.http,
batch_url=self.batch_url,
errors=errors,
custom_get_requests=None))
if errors:
utils.RaiseToolException(
errors,
error_message='Could not fetch resource:')
self.original_object = objects[0]
self.original_record = encoding.MessageToDict(self.original_object)
# Selects only the fields that can be modified.
field_selector = property_selector.PropertySelector(
properties=self._resource_spec.editables)
self.modifiable_record = field_selector.Apply(self.original_record)
buf = cStringIO.StringIO()
for line in _HELP.splitlines():
buf.write('#')
if line:
buf.write(' ')
buf.write(line)
buf.write('\n')
buf.write('\n')
buf.write(_SerializeDict(self.modifiable_record,
args.format or BaseEdit.DEFAULT_FORMAT))
buf.write('\n')
example = _SerializeDict(
encoding.MessageToDict(self.example_resource),
args.format or BaseEdit.DEFAULT_FORMAT)
_WriteResourceInCommentBlock(example, 'Example resource:', buf)
buf.write('#\n')
original = _SerializeDict(self.original_record,
args.format or BaseEdit.DEFAULT_FORMAT)
_WriteResourceInCommentBlock(original, 'Original resource:', buf)
file_contents = buf.getvalue()
while True:
file_contents = edit.OnlineEdit(file_contents)
try:
resources = self.ProcessEditedResource(file_contents, args)
break
except (ValueError, yaml.error.YAMLError,
protorpc.messages.ValidationError,
calliope_exceptions.ToolException) as e:
if isinstance(e, ValueError):
message = e.message
else:
message = str(e)
if isinstance(e, calliope_exceptions.ToolException):
problem_type = 'applying'
else:
problem_type = 'parsing'
message = ('There was a problem {0} your changes: {1}'
.format(problem_type, message))
if not console_io.PromptContinue(
message=message,
prompt_string='Would you like to edit the resource again?'):
raise calliope_exceptions.ToolException('Edit aborted by user.')
resources = lister.ProcessResults(
resources=resources,
field_selector=property_selector.PropertySelector(
properties=None,
transformations=self.transformations))
for resource in resources:
yield resource
def ConstructInstanceFromArgs(cls,
sql_messages,
args,
original=None,
instance_ref=None):
"""Construct a Cloud SQL instance from command line args.
Args:
sql_messages: module, The messages module that should be used.
args: argparse.Namespace, The CLI arg namespace.
original: sql_messages.DatabaseInstance, The original instance, if some of
it might be used to fill fields in the new one.
instance_ref: reference to DatabaseInstance object, used to fill project
and instance information.
Returns:
sql_messages.DatabaseInstance, The constructed (and possibly partial)
database instance.
Raises:
ToolException: An error other than http error occured while executing the
command.
"""
settings = cls._ConstructSettingsFromArgs(sql_messages, args, original)
cls._SetBackupConfiguration(sql_messages, settings, args, original)
cls._SetDatabaseFlags(sql_messages, settings, args)
cls._SetMaintenanceWindow(sql_messages, settings, args, original)
on_premises_host_port = getattr(args, 'on_premises_host_port', None)
if on_premises_host_port:
if args.require_ssl:
raise exceptions.ToolException(
'Argument --on-premises-host-port not '
'allowed with --require_ssl')
settings.onPremisesConfiguration = sql_messages.OnPremisesConfiguration(
hostPort=on_premises_host_port)
storage_size = getattr(args, 'storage_size', None)
if storage_size:
settings.dataDiskSizeGb = int(storage_size / (1 << 30))
# these flags are only present for the create command
region = getattr(args, 'region', None)
database_version = getattr(args, 'database_version', None)
instance_resource = sql_messages.DatabaseInstance(
region=region,
databaseVersion=database_version,
masterInstanceName=getattr(args, 'master_instance_name', None),
settings=settings)
if hasattr(args, 'master_instance_name'):
if args.master_instance_name:
replication = 'ASYNCHRONOUS'
if hasattr(args,
'replica_type') and args.replica_type == 'FAILOVER':
instance_resource.replicaConfiguration = (
sql_messages.ReplicaConfiguration(failoverTarget=True))
else:
replication = 'SYNCHRONOUS'
if not args.replication:
instance_resource.settings.replicationType = replication
if instance_ref:
cls.SetProjectAndInstanceFromRef(instance_resource, instance_ref)
if hasattr(args, 'storage_type') and args.storage_type:
instance_resource.settings.dataDiskType = 'PD_' + args.storage_type
if hasattr(args,
'failover_replica_name') and args.failover_replica_name:
instance_resource.failoverReplica = (
sql_messages.DatabaseInstance.FailoverReplicaValue(
name=args.failover_replica_name))
if (hasattr(args, 'storage_auto_increase')
and args.storage_auto_increase is not None):
instance_resource.settings.storageAutoResize = args.storage_auto_increase
if (hasattr(args, 'storage_auto_increase_limit')
and args.IsSpecified('storage_auto_increase_limit')):
# Resize limit should be settable if the original instance has resize
# turned on, or if the instance to be created has resize flag.
if (original and original.settings.storageAutoResize) or (
args.storage_auto_increase):
# If the limit is set to None, we want it to be set to 0. This is a
# backend requirement.
instance_resource.settings.storageAutoResizeLimit = (
args.storage_auto_increase_limit) or 0
else:
raise exceptions.RequiredArgumentException(
'--storage-auto-increase',
'To set the storage capacity limit '
'using [--storage-auto-increase-limit], [--storage-auto-increase] '
'must be enabled.')
return instance_resource
def PromptForDeletionHelper(resource_name, prompt_list, prompt_title=None):
prompt_title = (prompt_title or
'The following {0} will be deleted:'.format(resource_name))
prompt_message = ConstructList(prompt_title, prompt_list)
if not console_io.PromptContinue(message=prompt_message):
raise calliope_exceptions.ToolException('Deletion aborted by user.')
def Run(self, args):
holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
client = holder.client
if (args.check_interval is not None
and (args.check_interval < CHECK_INTERVAL_LOWER_BOUND_SEC
or args.check_interval > CHECK_INTERVAL_UPPER_BOUND_SEC)):
raise exceptions.ToolException(
'[--check-interval] must not be less than {0} second or greater '
'than {1} seconds; received [{2}] seconds.'.format(
CHECK_INTERVAL_LOWER_BOUND_SEC,
CHECK_INTERVAL_UPPER_BOUND_SEC, args.check_interval))
if (args.timeout is not None
and (args.timeout < TIMEOUT_LOWER_BOUND_SEC
or args.timeout > TIMEOUT_UPPER_BOUND_SEC)):
raise exceptions.ToolException(
'[--timeout] must not be less than {0} second or greater than {1} '
'seconds; received: [{2}] seconds.'.format(
TIMEOUT_LOWER_BOUND_SEC, TIMEOUT_UPPER_BOUND_SEC,
args.timeout))
if (args.healthy_threshold is not None
and (args.healthy_threshold < THRESHOLD_LOWER_BOUND
or args.healthy_threshold > THRESHOLD_UPPER_BOUND)):
raise exceptions.ToolException(
'[--healthy-threshold] must be an integer between {0} and {1}, '
'inclusive; received: [{2}].'.format(THRESHOLD_LOWER_BOUND,
THRESHOLD_UPPER_BOUND,
args.healthy_threshold))
if (args.unhealthy_threshold is not None
and (args.unhealthy_threshold < THRESHOLD_LOWER_BOUND
or args.unhealthy_threshold > THRESHOLD_UPPER_BOUND)):
raise exceptions.ToolException(
'[--unhealthy-threshold] must be an integer between {0} and {1}, '
'inclusive; received [{2}].'.format(THRESHOLD_LOWER_BOUND,
THRESHOLD_UPPER_BOUND,
args.unhealthy_threshold))
args_unset = not (args.port or args.request_path or args.check_interval
or args.timeout or args.healthy_threshold
or args.unhealthy_threshold)
if args.description is None and args.host is None and args_unset:
raise exceptions.ToolException(
'At least one property must be modified.')
http_health_check_ref = self.CreateReference(holder.resources, args)
get_request = self.GetGetRequest(client, http_health_check_ref)
objects = client.MakeRequests([get_request])
new_object = self.Modify(client, args, objects[0])
# If existing object is equal to the proposed object or if
# Modify() returns None, then there is no work to be done, so we
# print the resource and return.
if objects[0] == new_object:
log.status.Print(
'No change requested; skipping update for [{0}].'.format(
objects[0].name))
return objects
return client.MakeRequests(
[self.GetSetRequest(client, http_health_check_ref, new_object)])
def Run(self, args):
if six.PY3:
raise exceptions.ToolException(
'This command does not support python3.')
_Run(args)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
"""
adapter = self.context['api_adapter']
cluster_refs = []
for name in args.names:
cluster_refs.append(adapter.ParseCluster(name))
if not console_io.PromptContinue(message=util.ConstructList(
'The following clusters will be deleted.', [
'[{name}] in [{zone}]'.format(name=ref.clusterId,
zone=adapter.Zone(ref))
for ref in cluster_refs
]),
throw_if_unattended=True):
raise exceptions.ToolException('Deletion aborted by user.')
operations = []
errors = []
# Issue all deletes first
for cluster_ref in cluster_refs:
try:
# Make sure it exists (will raise appropriate error if not)
adapter.GetCluster(cluster_ref)
op_ref = adapter.DeleteCluster(cluster_ref)
operations.append((op_ref, cluster_ref))
except apitools_base.HttpError as error:
errors.append(util.GetError(error))
except util.Error as error:
errors.append(error)
if args.wait:
# Poll each operation for completion
for operation_ref, cluster_ref in operations:
try:
adapter.WaitForOperation(operation_ref,
'Deleting cluster {0}'.format(
cluster_ref.clusterId),
timeout_s=args.timeout)
# Purge cached config files
util.ClusterConfig.Purge(cluster_ref.clusterId,
adapter.Zone(cluster_ref),
cluster_ref.projectId)
if properties.VALUES.container.cluster.Get(
) == cluster_ref.clusterId:
properties.PersistProperty(
properties.VALUES.container.cluster, None)
log.DeletedResource(cluster_ref)
except apitools_base.HttpError as error:
errors.append(util.GetError(error))
except util.Error as error:
errors.append(error)
if errors:
raise exceptions.ToolException(
util.ConstructList('Some requests did not succeed:', errors))
def GetAddress(self, messages, args, address, address_ref,
resource_parser):
network_tier = self.ConstructNetworkTier(messages, args)
if args.ip_version or (address is None and address_ref.Collection()
== 'compute.globalAddresses'):
ip_version = messages.Address.IpVersionValueValuesEnum(
args.ip_version or 'IPV4')
else:
# IP version is only specified in global requests if an address is not
# specified to determine whether an ipv4 or ipv6 address should be
# allocated.
ip_version = None
if args.subnet and args.network:
raise exceptions.ConflictingArgumentsException(
'--network', '--subnet')
purpose = None
if args.purpose and not args.network and not args.subnet:
raise exceptions.MinimumArgumentException(
['--network', '--subnet'], ' if --purpose is specified')
# TODO(b/36862747): get rid of args.subnet check
if args.subnet:
if address_ref.Collection() == 'compute.globalAddresses':
raise exceptions.ToolException(
'[--subnet] may not be specified for global addresses.')
if not args.subnet_region:
args.subnet_region = address_ref.region
subnetwork_url = flags.SubnetworkArgument().ResolveAsResource(
args, resource_parser).SelfLink()
purpose = messages.Address.PurposeValueValuesEnum(
args.purpose or 'GCE_ENDPOINT')
if purpose != messages.Address.PurposeValueValuesEnum.GCE_ENDPOINT:
raise exceptions.InvalidArgumentException(
'--purpose',
'must be GCE_ENDPOINT for regional internal addresses.')
else:
subnetwork_url = None
network_url = None
if args.network:
if address_ref.Collection() == 'compute.addresses':
raise exceptions.InvalidArgumentException(
'--network',
'network may not be specified for regional addresses.')
network_url = flags.NetworkArgument().ResolveAsResource(
args, resource_parser).SelfLink()
purpose = messages.Address.PurposeValueValuesEnum(args.purpose
or 'VPC_PEERING')
if purpose != messages.Address.PurposeValueValuesEnum.VPC_PEERING:
raise exceptions.InvalidArgumentException(
'--purpose',
'must be VPC_PEERING for global internal addresses.')
if not args.prefix_length:
raise exceptions.RequiredArgumentException(
'--prefix-length',
'prefix length is needed for reserving IP ranges.')
if args.prefix_length:
if purpose != messages.Address.PurposeValueValuesEnum.VPC_PEERING:
raise exceptions.InvalidArgumentException(
'--prefix-length',
'can only be used with [--purpose VPC_PEERING].')
return messages.Address(
address=address,
prefixLength=args.prefix_length,
description=args.description,
networkTier=network_tier,
ipVersion=ip_version,
name=address_ref.Name(),
addressType=(messages.Address.AddressTypeValueValuesEnum.INTERNAL
if subnetwork_url or network_url else None),
purpose=purpose,
subnetwork=subnetwork_url,
network=network_url)
def Filter(self, tool_context, args):
if not args.instance:
raise exceptions.ToolException('argument --instance/-i is required')
def Run(self, args):
super(ConfigSSH, self).Run(args)
ssh_config_file = os.path.expanduser(
args.ssh_config_file or constants.PER_USER_SSH_CONFIG_FILE)
instances = None
if args.remove:
compute_section = ''
else:
self.EnsureSSHKeyIsInProject(getpass.getuser())
instances = list(self.GetInstances())
if instances:
compute_section = _BuildComputeSection(instances,
self.ssh_key_file)
else:
compute_section = ''
existing_content = _ReadFile(ssh_config_file)
if existing_content:
section_re = re.compile(_COMPUTE_SECTION_RE,
flags=re.MULTILINE | re.DOTALL)
matches = len(section_re.findall(existing_content))
if matches == 0:
# There are no existing Compute Engine sections. If there is
# at least one instance in the project (signified by
# compute_section not being None), we append it to the end of
# the configs. Otherwise, we set content to None which will
# cause nothing to be written to the SSH config file.
if compute_section:
# Ensures that there is a blank line between the existing
# configs and the Compute section.
if existing_content[-1] != '\n':
existing_content += '\n'
if existing_content[-2:] != '\n\n':
existing_content += '\n'
new_content = existing_content + compute_section
else:
new_content = existing_content
elif matches == 1:
new_content = re.sub(section_re, compute_section,
existing_content)
else:
raise exceptions.ToolException(
'Found more than one Google Compute Engine section in [{0}]. '
'You can either delete [{0}] and let this command recreate it for '
'you or you can manually delete all sections marked with '
'[{1}] and [{2}].'.format(ssh_config_file, _BEGIN_MARKER,
_END_MARKER))
else:
new_content = compute_section
if args.dry_run:
log.out.write(new_content or '')
return
if new_content != existing_content:
with open(ssh_config_file, 'w') as f:
f.write(new_content)
if compute_section:
log.out.write(
textwrap.dedent("""\
You should now be able to use ssh/scp with your instances.
For example, try running:
ssh {alias}
""".format(alias=_CreateAlias(instances[0]))))
elif not instances and not args.remove:
log.warn(
'No host aliases were added to your SSH configs because you do not '
'have any instances. Try running this command again after creating '
'some instances.')
def Run(self, args):
if not os.path.exists(args.records_file):
raise exceptions.ToolException('no such file [{0}]'.format(
args.records_file))
if os.path.isdir(args.records_file):
raise exceptions.ToolException('[{0}] is a directory'.format(
args.records_file))
dns = self.context['dns_client']
messages = self.context['dns_messages']
resources = self.context['dns_resources']
project_id = properties.VALUES.core.project.Get(required=True)
# Get the managed-zone.
zone_ref = resources.Parse(args.zone, collection='dns.managedZones')
try:
zone = dns.managedZones.Get(zone_ref.Request())
except apitools_exceptions.HttpError as error:
raise exceptions.HttpException(util.GetErrorMessage(error))
# Get the current record-sets.
current = {}
for record in list_pager.YieldFromList(
dns.resourceRecordSets,
messages.DnsResourceRecordSetsListRequest(
project=project_id, managedZone=zone_ref.Name()),
field='rrsets'):
current[(record.name, record.type)] = record
# Get the imported record-sets.
try:
with files.Context(open(args.records_file)) as import_file:
if args.zone_file_format:
imported = import_util.RecordSetsFromZoneFile(
import_file, zone.dnsName)
else:
imported = import_util.RecordSetsFromYamlFile(import_file)
except Exception as exp:
msg = (
u'unable to read record-sets from specified records-file [{0}] '
u'because [{1}]')
msg = msg.format(args.records_file, exp.message)
raise exceptions.ToolException(msg)
# Get the change resulting from the imported record-sets.
change = import_util.ComputeChange(current, imported,
args.delete_all_existing,
zone.dnsName,
args.replace_origin_ns)
if not change:
msg = u'Nothing to do, all the records in [{0}] already exist.'.format(
args.records_file)
log.status.Print(msg)
return None
# Send the change to the service.
result = dns.changes.Create(
messages.DnsChangesCreateRequest(change=change,
managedZone=zone.name,
project=project_id))
change_ref = resources.Create(collection='dns.changes',
project=project_id,
managedZone=zone.name,
changeId=result.id)
msg = u'Imported record-sets from [{0}] into managed-zone [{1}].'.format(
args.records_file, zone_ref.Name())
log.status.Print(msg)
log.CreatedResource(change_ref)
return result
def Filter(self, context, args):
"""Modify the context that will be given to this group's commands when run.
Args:
context: {str:object}, The current context, which is a set of key-value
pairs that can be used for common initialization among commands.
args: argparse.Namespace: The same Namespace given to the corresponding
.Run() invocation.
Returns:
The refined command context.
"""
# Get service endpoints and ensure they are compatible with each other
testing_url = properties.VALUES.api_endpoint_overrides.testing.Get()
toolresults_url = properties.VALUES.api_endpoint_overrides.toolresults.Get(
)
log.info('Test Service endpoint: [{0}]'.format(testing_url))
log.info('Tool Results endpoint: [{0}]'.format(toolresults_url))
if ((toolresults_url is None
or 'apis.com/toolresults' in toolresults_url) !=
(testing_url is None or 'testing.googleapis' in testing_url)):
raise exceptions.ToolException(
'Service endpoints [{0}] and [{1}] are not compatible.'.format(
testing_url, toolresults_url))
http = self.Http()
# Create the Testing service client
resources.SetParamDefault(api='test',
collection=None,
param='project',
resolver=resolvers.FromProperty(
properties.VALUES.core.project))
# TODO(user) Support multiple versions when they exist
testing_client_v1 = testing_v1.TestingV1(get_credentials=False,
url=testing_url,
http=http)
testing_registry = resources.REGISTRY.CloneAndSwitchAPIs(
testing_client_v1)
context['testing_client'] = testing_client_v1
context['testing_messages'] = testing_v1
context['testing_registry'] = testing_registry
# Create the Tool Results service client.
resources.SetParamDefault(api='toolresults',
collection=None,
param='project',
resolver=resolvers.FromProperty(
properties.VALUES.core.project))
toolresults_client_v1 = toolresults_v1beta3.ToolresultsV1beta3(
get_credentials=False, url=toolresults_url, http=http)
tr_registry = resources.REGISTRY.CloneAndSwitchAPIs(
toolresults_client_v1)
context['toolresults_client'] = toolresults_client_v1
context['toolresults_messages'] = toolresults_v1beta3
context['toolresults_registry'] = tr_registry
# TODO(user): remove this message for general release.
log.status.Print(
'\nHave questions, feedback, or issues? Please let us know by using '
'this Google Group:\n https://groups.google.com/forum/#!forum'
'/google-cloud-test-lab-external\n')
return context
def Run(self, args):
holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
client = holder.client
backend_service_ref = (
flags.GLOBAL_REGIONAL_BACKEND_SERVICE_ARG.ResolveAsResource(
args,
holder.resources,
scope_lister=compute_flags.GetDefaultScopeLister(client)))
data = console_io.ReadFromFileOrStdin(args.source or '-', binary=False)
try:
backend_service = export_util.Import(
message_type=client.messages.BackendService,
stream=data,
schema_path=self.GetSchemaPath())
except yaml_validator.ValidationError as e:
raise exceptions.ToolException(str(e))
# Get existing backend service.
try:
backend_service_old = backend_services_utils.SendGetRequest(
client, backend_service_ref)
except apitools_exceptions.HttpError as error:
if error.status_code != 404:
raise error
# Backend service does not exist, create a new one.
return self.SendInsertRequest(client, backend_service_ref,
backend_service)
# No change, do not send requests to server.
if backend_service_old == backend_service:
return
console_io.PromptContinue(
message=('Backend Service [{0}] will be overwritten.').format(
backend_service_ref.Name()),
cancel_on_no=True)
# populate id and fingerprint fields. These two fields are manually
# removed from the schema files.
backend_service.id = backend_service_old.id
backend_service.fingerprint = backend_service_old.fingerprint
# Unspecified fields are assumed to be cleared.
cleared_fields = []
if hasattr(backend_service, 'securitySettings') is None:
cleared_fields.append('securitySettings')
if hasattr(backend_service, 'localityLbPolicy') is None:
cleared_fields.append('localityLbPolicy')
if hasattr(backend_service, 'circuitBreakers') is None:
cleared_fields.append('circuitBreakers')
if hasattr(backend_service, 'consistentHash') is None:
cleared_fields.append('consistentHash')
if hasattr(backend_service, 'outlierDetection') is None:
cleared_fields.append('outlierDetection')
if not backend_service.customRequestHeaders:
cleared_fields.append('customRequestHeaders')
if self._support_flexible_cache_step_one:
if not backend_service.customResponseHeaders:
cleared_fields.append('customResponseHeaders')
if backend_service.cdnPolicy:
cdn_policy = backend_service.cdnPolicy
if cdn_policy.defaultTtl is None:
cleared_fields.append('cdnPolicy.defaultTtl')
if cdn_policy.clientTtl is None:
cleared_fields.append('cdnPolicy.clientTtl')
if cdn_policy.maxTtl is None:
cleared_fields.append('cdnPolicy.maxTtl')
if not cdn_policy.negativeCachingPolicy:
cleared_fields.append('cdnPolicy.negativeCachingPolicy')
else:
cleared_fields.append('cdnPolicy')
with client.apitools_client.IncludeFields(cleared_fields):
return self.SendPatchRequest(client, backend_service_ref,
backend_service)
def Run(self, args):
"""Run the helper command."""
if args.method not in GitHelper.METHODS:
if args.ignore_unknown:
return
raise c_exc.ToolException(
'Unexpected method [{meth}]. One of [{methods}] expected.'.
format(meth=args.method, methods=', '.join(GitHelper.METHODS)))
info = self._ParseInput()
credentialed_domains = ['source.developers.google.com']
extra = properties.VALUES.core.credentialed_hosted_repo_domains.Get()
if extra:
credentialed_domains.extend(extra.split(','))
if info.get('host') not in credentialed_domains:
if args.ignore_unknown:
return
raise c_exc.ToolException(
'Unknown host [{host}].'.format(host=info.get('host')))
if args.method == GitHelper.GET:
account = properties.VALUES.core.account.Get()
try:
cred = c_store.Load(account)
c_store.Refresh(cred)
except c_store.Error as e:
sys.stderr.write(
textwrap.dedent("""\
ERROR: {error}
Run 'gcloud auth login' to log in.
""".format(error=str(e))))
return
self._CheckNetrc()
sys.stdout.write(
textwrap.dedent("""\
username={username}
password={password}
""").format(username=account, password=cred.access_token))
elif args.method == GitHelper.STORE:
# On OSX, there is an additional credential helper that gets called before
# ours does. When we return a token, it gets cached there. Git continues
# to get it from there first until it expires. That command then fails,
# and the token is deleted, but it does not retry the operation. The next
# command gets a new token from us and it starts working again, for an
# hour. This erases our credential from the other cache whenever 'store'
# is called on us. Because they are called first, the token will already
# be stored there, and so we can successfully erase it to prevent caching.
if (platforms.OperatingSystem.Current() ==
platforms.OperatingSystem.MACOSX):
log.debug('Clearing OSX credential cache.')
try:
input_string = 'protocol={protocol}\nhost={host}\n\n'.format(
protocol=info.get('protocol'), host=info.get('host'))
log.debug('Calling erase with input:\n%s', input_string)
p = subprocess.Popen(
['git-credential-osxkeychain', 'erase'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(out, err) = p.communicate(input_string)
if p.returncode:
log.debug(
'Failed to clear OSX keychain:\nstdout: {%s}\nstderr: {%s}',
out, err)
# pylint:disable=broad-except, This can fail and should only be done as
# best effort.
except Exception as e:
log.debug('Failed to clear OSX keychain', exc_info=True)
def _Run(self, args):
"""Returns a list of requests necessary for adding images."""
holder = self._GetApiHolder()
client = holder.client
messages = client.messages
resource_parser = holder.resources
image_ref = Create.DISK_IMAGE_ARG.ResolveAsResource(
args, holder.resources)
image = messages.Image(
name=image_ref.image,
description=args.description,
sourceType=messages.Image.SourceTypeValueValuesEnum.RAW,
family=args.family)
csek_keys = csek_utils.CsekKeyStore.FromArgs(
args, self._ALLOW_RSA_ENCRYPTED_CSEK_KEYS)
if csek_keys:
image.imageEncryptionKey = csek_utils.MaybeToMessage(
csek_keys.LookupKey(
image_ref, raise_if_missing=args.require_csek_key_create),
client.apitools_client)
image.imageEncryptionKey = kms_utils.MaybeGetKmsKey(
args, messages, image.imageEncryptionKey)
# Validate parameters.
if args.source_disk_zone and not args.source_disk:
raise exceptions.ToolException(
'You cannot specify [--source-disk-zone] unless you are specifying '
'[--source-disk].')
source_image_project = args.source_image_project
source_image = args.source_image
source_image_family = args.source_image_family
if source_image_project and not (source_image or source_image_family):
raise exceptions.ToolException(
'You cannot specify [--source-image-project] unless you are '
'specifying [--source-image] or [--source-image-family].')
if source_image or source_image_family:
image_expander = image_utils.ImageExpander(client, resource_parser)
_, source_image_ref = image_expander.ExpandImageFlag(
user_project=image_ref.project,
image=source_image,
image_family=source_image_family,
image_project=source_image_project,
return_image_resource=True)
image.sourceImage = source_image_ref.selfLink
image.sourceImageEncryptionKey = csek_utils.MaybeLookupKeyMessage(
csek_keys, source_image_ref, client.apitools_client)
if args.source_uri:
source_uri = six.text_type(
resources.REGISTRY.Parse(args.source_uri))
image.rawDisk = messages.Image.RawDiskValue(source=source_uri)
elif args.source_disk:
source_disk_ref = flags.SOURCE_DISK_ARG.ResolveAsResource(
args,
holder.resources,
scope_lister=compute_flags.GetDefaultScopeLister(client))
image.sourceDisk = source_disk_ref.SelfLink()
image.sourceDiskEncryptionKey = csek_utils.MaybeLookupKeyMessage(
csek_keys, source_disk_ref, client.apitools_client)
elif hasattr(args, 'source_snapshot') and args.source_snapshot:
source_snapshot_ref = flags.SOURCE_SNAPSHOT_ARG.ResolveAsResource(
args,
holder.resources,
scope_lister=compute_flags.GetDefaultScopeLister(client))
image.sourceSnapshot = source_snapshot_ref.SelfLink()
image.sourceSnapshotEncryptionKey = csek_utils.MaybeLookupKeyMessage(
csek_keys, source_snapshot_ref, client.apitools_client)
if args.licenses:
image.licenses = args.licenses
guest_os_features = getattr(args, 'guest_os_features', [])
if guest_os_features:
guest_os_feature_messages = []
for feature in guest_os_features:
gf_type = messages.GuestOsFeature.TypeValueValuesEnum(feature)
guest_os_feature = messages.GuestOsFeature()
guest_os_feature.type = gf_type
guest_os_feature_messages.append(guest_os_feature)
image.guestOsFeatures = guest_os_feature_messages
initial_state, has_set =\
image_utils.CreateInitialStateConfig(args, messages)
if has_set:
image.shieldedInstanceInitialState = initial_state
if args.IsSpecified('storage_location'):
image.storageLocations = [args.storage_location]
request = messages.ComputeImagesInsertRequest(
image=image, project=image_ref.project)
args_labels = getattr(args, 'labels', None)
if args_labels:
labels = messages.Image.LabelsValue(additionalProperties=[
messages.Image.LabelsValue.AdditionalProperty(key=key,
value=value)
for key, value in sorted(six.iteritems(args_labels))
])
request.image.labels = labels
# --force is in GA, --force-create is in beta and deprecated.
if args.force or getattr(args, 'force_create', None):
request.forceCreate = True
return client.MakeRequests(
[(client.apitools_client.images, 'Insert', request)],
timeout=POLL_TIMEOUT)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Returns:
None
Raises:
bigqueryError.BigqueryError: If the source and destination files are not
both specified.
calliope_exceptions.ToolException: If user cancels this operation.
Exception: If an unexpected value for the --if-exists flag passed gcloud
validation (which should never happen)
"""
apitools_client = self.context[commands.APITOOLS_CLIENT_KEY]
bigquery_messages = self.context[commands.BIGQUERY_MESSAGES_MODULE_KEY]
resource_parser = self.context[commands.BIGQUERY_REGISTRY_KEY]
project_id = properties.VALUES.core.project.Get(required=True)
source_reference = resource_parser.Parse(args.source,
collection='bigquery.tables')
source_reference_message = message_conversions.TableResourceToReference(
bigquery_messages, source_reference)
destination_resource = resource_parser.Parse(
args.destination, collection='bigquery.tables')
destination_reference = message_conversions.TableResourceToReference(
bigquery_messages, destination_resource)
if args.if_exists == 'append':
write_disposition = 'WRITE_APPEND'
ignore_already_exists = True
elif args.if_exists == 'fail':
write_disposition = 'WRITE_EMPTY'
ignore_already_exists = False
elif args.if_exists == 'prompt':
write_disposition = 'WRITE_TRUNCATE'
ignore_already_exists = False
if bigquery_client_helper.TableExists(apitools_client,
bigquery_messages,
destination_reference):
if not console_io.PromptContinue(prompt_string='Replace {0}'.
format(destination_resource)):
raise calliope_exceptions.ToolException('canceled by user')
elif args.if_exists == 'replace':
write_disposition = 'WRITE_TRUNCATE'
ignore_already_exists = False
elif args.if_exists == 'skip':
if bigquery_client_helper.TableExists(apitools_client,
bigquery_messages,
destination_reference):
return
else:
# This should be unreachable.
raise core_exceptions.InternalError(
'Unexpected value "{0}" for --if-exists flag.'.format(
args.if_exists))
copy_config = bigquery_messages.JobConfigurationTableCopy(
sourceTable=source_reference_message,
destinationTable=destination_reference,
writeDisposition=write_disposition)
job_id = job_ids.JobIdProvider().GetJobId(args.job_id,
args.fingerprint_job_id)
try:
job = job_control.ExecuteJob(
apitools_client,
bigquery_messages,
args,
configuration=bigquery_messages.JobConfiguration(
copy=copy_config),
project_id=project_id,
job_id=job_id)
except bigquery.DuplicateError as e:
if ignore_already_exists:
job = None
else:
raise e
if job is None:
log.status.Print('Table "{0}" already exists, skipping'.format(
destination_resource))
elif args. async:
registry = self.context[commands.BIGQUERY_REGISTRY_KEY]
job_resource = registry.Create(
'bigquery.jobs',
projectId=job.jobReference.projectId,
jobId=job.jobReference.jobId)
log.CreatedResource(job_resource)
else:
log.status.Print('Table [{0}] successfully copied to [{1}]'.format(
source_reference, destination_resource))
def Run(self, args):
"""Issue API requests for route creation, callable from multiple tracks."""
holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
client = holder.client
network_uri = self.NETWORK_ARG.ResolveAsResource(
args, holder.resources).SelfLink()
route_ref = self.ROUTE_ARG.ResolveAsResource(args, holder.resources)
if args.next_hop_instance:
next_hop_instance_uri = self.INSTANCE_ARG.ResolveAsResource(
args,
holder.resources,
scope_lister=instance_flags.GetInstanceZoneScopeLister(
client)).SelfLink()
else:
if args.next_hop_instance_zone:
raise exceptions.ToolException(
'[--next-hop-instance-zone] can only be specified in conjunction '
'with [--next-hop-instance].')
next_hop_instance_uri = None
if args.next_hop_gateway:
next_hop_gateway_ref = flags.NEXT_HOP_GATEWAY_ARG.ResolveAsResource(
args, holder.resources)
next_hop_gateway_uri = next_hop_gateway_ref.SelfLink()
else:
next_hop_gateway_uri = None
next_hop_vpn_tunnel_uri = None
if args.next_hop_vpn_tunnel:
next_hop_vpn_tunnel_uri = self.VPN_TUNNEL_ARG.ResolveAsResource(
args,
holder.resources,
scope_lister=compute_flags.GetDefaultScopeLister(client)).SelfLink()
elif args.next_hop_vpn_tunnel_region:
raise exceptions.ToolException(
'[--next-hop-vpn-tunnel-region] can only be specified in '
'conjunction with [--next-hop-vpn-tunnel].')
next_hop_ilb_uri = None
if args.next_hop_ilb:
next_hop_ilb_uri = self.ILB_ARG.ResolveAsResource(
args,
holder.resources,
scope_lister=compute_flags.GetDefaultScopeLister(client)).SelfLink()
elif args.next_hop_ilb_region:
raise exceptions.ToolException(
'[--next-hop-ilb-region] can only be specified in '
'conjunction with [--next-hop-ilb].')
request = client.messages.ComputeRoutesInsertRequest(
project=route_ref.project,
route=client.messages.Route(
description=args.description,
destRange=args.destination_range,
name=route_ref.Name(),
network=network_uri,
nextHopInstance=next_hop_instance_uri,
nextHopIp=args.next_hop_address,
nextHopGateway=next_hop_gateway_uri,
nextHopVpnTunnel=next_hop_vpn_tunnel_uri,
priority=args.priority,
tags=args.tags,
))
request.route.nextHopIlb = next_hop_ilb_uri
return client.MakeRequests([(client.apitools_client.routes, 'Insert',
request)])
def ConvertToJsonObject(json_string):
"""Tries to convert the JSON string into JsonObject."""
try:
return extra_types.JsonProtoDecoder(json_string)
except Exception as e:
raise exceptions.ToolException('Invalid JSON value: %s' % e.message)
def ComputeChange(current,
to_be_imported,
replace_all=False,
origin=None,
replace_origin_ns=False):
"""Returns a change for importing the given record-sets.
Args:
current: dict, (name, type) keyed dict of current record-sets.
to_be_imported: dict, (name, type) keyed dict of record-sets to be imported.
replace_all: bool, Whether the record-sets to be imported should replace the
current record-sets.
origin: string, the name of the apex zone ex. "foo.com"
replace_origin_ns: bool, Whether origin NS records should be imported.
Raises:
ToolException: If conflicting CNAME records are found.
Returns:
A Change that describes the actions required to import the given
record-sets.
"""
messages = core_apis.GetMessagesModule('dns', 'v1')
change = messages.Change()
change.additions = []
change.deletions = []
current_keys = set(current.keys())
keys_to_be_imported = set(to_be_imported.keys())
intersecting_keys = current_keys.intersection(keys_to_be_imported)
if not replace_all and intersecting_keys:
raise exceptions.ToolException(
'Conflicting records for the following (name type): {0}'.format([
_NameAndType(current[key]) for key in sorted(intersecting_keys)
]))
for key in intersecting_keys:
current_record = current[key]
record_to_be_imported = to_be_imported[key]
rdtype = rdatatype.from_text(key[1])
if not _FilterOutRecord(current_record.name, rdtype, origin,
replace_origin_ns):
replacement = _RDATA_REPLACEMENTS[rdtype](current_record,
record_to_be_imported)
if replacement:
change.deletions.append(current_record)
change.additions.append(replacement)
for key in keys_to_be_imported.difference(current_keys):
change.additions.append(to_be_imported[key])
for key in current_keys.difference(keys_to_be_imported):
current_record = current[key]
rdtype = rdatatype.from_text(key[1])
if rdtype is rdatatype.SOA:
change.deletions.append(current_record)
change.additions.append(NextSOARecordSet(current_record))
elif replace_all and not _FilterOutRecord(current_record.name, rdtype,
origin, replace_origin_ns):
change.deletions.append(current_record)
# If the only change is an SOA increment, there is nothing to be done.
if IsOnlySOAIncrement(change):
return None
change.additions.sort(key=_NameAndType)
change.deletions.sort(key=_NameAndType)
return change
def Run(self, args):
"""Connects to a Cloud SQL instance.
Args:
args: argparse.Namespace, The arguments that this command was invoked
with.
Returns:
If no exception is raised this method does not return. A new process is
started and the original one is killed.
Raises:
HttpException: An http error response was received while executing api
request.
ToolException: An error other than http error occured while executing the
command.
"""
# TODO(b/62055495): Replace ToolExceptions with specific exceptions.
client = api_util.SqlClient(api_util.API_VERSION_DEFAULT)
sql_client = client.sql_client
sql_messages = client.sql_messages
validate.ValidateInstanceName(args.instance)
instance_ref = client.resource_parser.Parse(
args.instance,
params={'project': properties.VALUES.core.project.GetOrFail},
collection='sql.instances')
acl_name = _WhitelistClientIP(instance_ref, sql_client, sql_messages,
client.resource_parser)
# Get the client IP that the server sees. Sadly we can only do this by
# checking the name of the authorized network rule.
retryer = retry.Retryer(max_retrials=2, exponential_sleep_multiplier=2)
try:
instance_info, client_ip = retryer.RetryOnResult(
_GetClientIP,
[instance_ref, sql_client, acl_name],
should_retry_if=lambda x, s: x[1] is None, # client_ip is None
sleep_ms=500)
except retry.RetryException:
raise exceptions.ToolException(
'Could not whitelist client IP. Server '
'did not reply with the whitelisted IP.')
# Check for the mysql or psql executable based on the db version.
db_type = instance_info.databaseVersion.split('_')[0]
exe_name = constants.DB_EXE.get(db_type, 'mysql')
exe = files.FindExecutableOnPath(exe_name)
if not exe:
raise exceptions.ToolException(
'{0} client not found. Please install a {1} client and make sure '
'it is in PATH to be able to connect to the database instance.'
.format(exe_name.title(), exe_name))
# Check the version of IP and decide if we need to add ipv4 support.
ip_type = network.GetIpVersion(client_ip)
if ip_type == network.IP_VERSION_4:
if instance_info.settings.ipConfiguration.ipv4Enabled:
ip_address = instance_info.ipAddresses[0].ipAddress
else:
# TODO(b/36049930): ask user if we should enable ipv4 addressing
message = (
'It seems your client does not have ipv6 connectivity and '
'the database instance does not have an ipv4 address. '
'Please request an ipv4 address for this database instance.'
)
raise exceptions.ToolException(message)
elif ip_type == network.IP_VERSION_6:
ip_address = instance_info.ipv6Address
else:
raise exceptions.ToolException('Could not connect to SQL server.')
# We have everything we need, time to party!
flags = constants.EXE_FLAGS[exe_name]
sql_args = [exe_name, flags['hostname'], ip_address]
if args.user:
sql_args.extend([flags['user'], args.user])
sql_args.append(flags['password'])
execution_utils.Exec(sql_args)
def RaiseOnPromptFailure():
"""Call this to raise an exn when prompt cannot read from input stream."""
phrases = ('one of ', 'flags') if len(flag_names) > 1 else ('', 'flag')
raise calliope_exceptions.ToolException(
'Unable to prompt. Specify {0}the [{1}] {2}.'.format(
phrases[0], ', '.join(flag_names), phrases[1]))
def _CreateRequests(cmd, args):
"""Make API requests for route creation, callable from multiple tracks."""
network_uri = cmd.CreateGlobalReference(
args.network, resource_type='networks').SelfLink()
if args.next_hop_instance:
next_hop_instance_uri = cmd.CreateZonalReference(
args.next_hop_instance,
args.next_hop_instance_zone,
flag_names=['--next-hop-instance-zone'],
resource_type='instances').SelfLink()
else:
if args.next_hop_instance_zone:
raise exceptions.ToolException(
'[--next-hop-instance-zone] can only be specified in conjunction '
'with [--next-hop-instance].')
next_hop_instance_uri = None
if args.next_hop_gateway:
# TODO(b/18201355): This is hack.
#
# There is currently no "gateways" resource type in the Compute
# API, however, the API does accept a "gateways" URI. We need to
# extend the resources module to allow for arbitrary URI
# patterns to be registered. With the logic below, a URI value
# for --next-hop-gateway will not work.
next_hop_gateway_uri = (cmd.compute.url + 'projects/' + cmd.project +
'/global/gateways/' + args.next_hop_gateway)
else:
next_hop_gateway_uri = None
route_ref = cmd.CreateGlobalReference(args.name)
next_hop_vpn_tunnel_uri = None
if args.next_hop_vpn_tunnel:
next_hop_vpn_tunnel_uri = cmd.CreateRegionalReference(
args.next_hop_vpn_tunnel,
args.next_hop_vpn_tunnel_region,
flag_names=['--next-hop-vpn-tunnel'],
resource_type='vpnTunnels').SelfLink()
elif args.next_hop_vpn_tunnel_region:
raise exceptions.ToolException(
'[--next-hop-vpn-tunnel-region] can only be specified in '
'conjunction with [--next-hop-vpn-tunnel].')
request = cmd.messages.ComputeRoutesInsertRequest(
project=cmd.project,
route=cmd.messages.Route(
description=args.description,
destRange=args.destination_range,
name=route_ref.Name(),
network=network_uri,
nextHopInstance=next_hop_instance_uri,
nextHopIp=args.next_hop_address,
nextHopGateway=next_hop_gateway_uri,
nextHopVpnTunnel=next_hop_vpn_tunnel_uri,
priority=args.priority,
tags=args.tags,
))
return [request]
def Run(self, args):
"""Returns a list of requests necessary for adding images."""
holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
client = holder.client
messages = client.messages
resource_parser = holder.resources
image_ref = Create.DISK_IMAGE_ARG.ResolveAsResource(
args, holder.resources)
image = messages.Image(
name=image_ref.image,
description=args.description,
sourceType=messages.Image.SourceTypeValueValuesEnum.RAW,
family=args.family)
csek_keys = csek_utils.CsekKeyStore.FromArgs(
args, self._ALLOW_RSA_ENCRYPTED_CSEK_KEYS)
if csek_keys:
image.imageEncryptionKey = csek_utils.MaybeToMessage(
csek_keys.LookupKey(
image_ref, raise_if_missing=args.require_csek_key_create),
client.apitools_client)
image.imageEncryptionKey = kms_utils.MaybeGetKmsKey(
args, image_ref.project, client.apitools_client,
image.imageEncryptionKey)
# Validate parameters.
if args.source_disk_zone and not args.source_disk:
raise exceptions.ToolException(
'You cannot specify [--source-disk-zone] unless you are specifying '
'[--source-disk].')
source_image_project = getattr(args, 'source_image_project', None)
source_image = getattr(args, 'source_image', None)
source_image_family = getattr(args, 'source_image_family', None)
if source_image_project and not (source_image or source_image_family):
raise exceptions.ToolException(
'You cannot specify [--source-image-project] unless you are '
'specifying [--source-image] or [--source-image-family].')
if source_image or source_image_family:
image_expander = image_utils.ImageExpander(client, resource_parser)
_, source_image_ref = image_expander.ExpandImageFlag(
user_project=image_ref.project,
image=source_image,
image_family=source_image_family,
image_project=source_image_project,
return_image_resource=True)
image.sourceImage = source_image_ref.selfLink
image.sourceImageEncryptionKey = csek_utils.MaybeLookupKeyMessage(
csek_keys, source_image_ref, client.apitools_client)
# TODO(b/30086260): use resources.REGISTRY.Parse() for GCS URIs.
if args.source_uri:
source_uri = utils.NormalizeGoogleStorageUri(args.source_uri)
image.rawDisk = messages.Image.RawDiskValue(source=source_uri)
elif args.source_disk:
source_disk_ref = flags.SOURCE_DISK_ARG.ResolveAsResource(
args,
holder.resources,
scope_lister=compute_flags.GetDefaultScopeLister(client))
image.sourceDisk = source_disk_ref.SelfLink()
image.sourceDiskEncryptionKey = csek_utils.MaybeLookupKeyMessage(
csek_keys, source_disk_ref, client.apitools_client)
if args.licenses:
image.licenses = args.licenses
guest_os_features = getattr(args, 'guest_os_features', [])
if guest_os_features:
guest_os_feature_messages = []
for feature in guest_os_features:
gf_type = messages.GuestOsFeature.TypeValueValuesEnum(feature)
guest_os_feature = messages.GuestOsFeature()
guest_os_feature.type = gf_type
guest_os_feature_messages.append(guest_os_feature)
image.guestOsFeatures = guest_os_feature_messages
request = messages.ComputeImagesInsertRequest(
image=image, project=image_ref.project)
args_labels = getattr(args, 'labels', None)
if args_labels:
labels = messages.Image.LabelsValue(additionalProperties=[
messages.Image.LabelsValue.AdditionalProperty(key=key,
value=value)
for key, value in sorted(args_labels.iteritems())
])
request.image.labels = labels
# --force is in GA, --force-create is in beta and deprecated.
if args.force or getattr(args, 'force_create', None):
request.forceCreate = True
return client.MakeRequests([(client.apitools_client.images, 'Insert',
request)])
def MonitorTestExecutionProgress(self, test_id):
"""Monitor and report the progress of a single running test.
This method prints more detailed test progress messages for the case where
the matrix has exactly one supported test configuration.
Args:
test_id: str, the unique id of the single supported test in the matrix.
Raises:
ToolException if the Test service reports a backend error.
"""
states = self._messages.TestExecution.StateValueValuesEnum
last_state = ''
error = ''
progress = []
last_progress_len = 0
while True:
status = self._GetTestExecutionStatus(test_id)
timestamp = self._clock().strftime(_TIMESTAMP_FORMAT)
# Check for optional error and progress details
details = status.testDetails
if details:
error = details.errorMessage or ''
progress = details.progressMessages or []
# Progress is cumulative, so only print what's new since the last update.
for msg in progress[last_progress_len:]:
log.status.Print('{0} {1}'.format(timestamp, msg.rstrip()))
last_progress_len = len(progress)
if status.state == states.ERROR:
raise exceptions.ToolException(
'Firebase Test Lab infrastructure failure: {e}'.format(
e=error),
exit_code=exit_code.INFRASTRUCTURE_ERR)
if status.state == states.UNSUPPORTED_ENVIRONMENT:
msg = (
'Device dimensions are not compatible: {d}. '
'Please use "gcloud beta test android devices list" to '
'determine which device dimensions are compatible.'.format(
d=_FormatInvalidDimension(status.environment)))
raise exceptions.ToolException(
msg, exit_code=exit_code.UNSUPPORTED_ENV)
# Inform user of test progress, typically PENDING -> RUNNING -> FINISHED
if status.state != last_state:
last_state = status.state
log.status.Print('{0} Test is {1}'.format(
timestamp, self._state_names[last_state]))
if status.state in self._completed_execution_states:
break
self._SleepForStatusInterval()
# Even if the single TestExecution is done, we also need to wait for the
# matrix to reach a finalized state before monitoring is done.
matrix = self.GetTestMatrixStatus()
while matrix.state not in self.completed_matrix_states:
log.debug('Matrix not yet complete, still in state: %s',
matrix.state)
self._SleepForStatusInterval()
matrix = self.GetTestMatrixStatus()
self._LogTestComplete(matrix.state)
return
def Run(self, args):
instances_flags.ValidateDiskFlags(
args,
enable_kms=self._support_kms,
enable_snapshots=True,
enable_source_snapshot_csek=self._support_source_snapshot_csek,
enable_image_csek=self._support_image_csek)
instances_flags.ValidateImageFlags(args)
instances_flags.ValidateLocalSsdFlags(args)
instances_flags.ValidateNicFlags(args)
instances_flags.ValidateServiceAccountAndScopeArgs(args)
instances_flags.ValidateAcceleratorArgs(args)
instances_flags.ValidateNetworkTierArgs(args)
if self._support_reservation:
instances_flags.ValidateReservationAffinityGroup(args)
holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
compute_client = holder.client
resource_parser = holder.resources
instance_refs = instance_utils.GetInstanceRefs(args, compute_client,
holder)
requests = self._CreateRequests(args, instance_refs, compute_client,
resource_parser, holder)
if not args. async:
# TODO(b/63664449): Replace this with poller + progress tracker.
try:
# Using legacy MakeRequests (which also does polling) here until
# replaced by api_lib.utils.waiter.
return compute_client.MakeRequests(requests)
except exceptions.ToolException as e:
invalid_machine_type_message_regex = (
r'Invalid value for field \'resource.machineType\': .+. '
r'Machine type with name \'.+\' does not exist in zone \'.+\'\.'
)
if re.search(invalid_machine_type_message_regex,
six.text_type(e)):
raise exceptions.ToolException(
six.text_type(e) +
'\nUse `gcloud compute machine-types list --zones` to see the '
'available machine types.')
raise
errors_to_collect = []
responses = compute_client.BatchRequests(requests, errors_to_collect)
for r in responses:
err = getattr(r, 'error', None)
if err:
errors_to_collect.append(poller.OperationErrors(err.errors))
if errors_to_collect:
raise core_exceptions.MultiError(errors_to_collect)
operation_refs = [
holder.resources.Parse(r.selfLink) for r in responses
]
log.status.Print(
'NOTE: The users will be charged for public IPs when VMs '
'are created.')
for instance_ref, operation_ref in zip(instance_refs, operation_refs):
log.status.Print(
'Instance creation in progress for [{}]: {}'.format(
instance_ref.instance, operation_ref.SelfLink()))
log.status.Print(
'Use [gcloud compute operations describe URI] command '
'to check the status of the operation(s).')
if not args.IsSpecified('format'):
# For async output we need a separate format. Since we already printed in
# the status messages information about operations there is nothing else
# needs to be printed.
args.format = 'disable'
return responses
def Run(self, args):
super(Scp, self).Run(args)
file_specs = []
# Parses the positional arguments.
for arg in args.sources + [args.destination]:
if ssh_utils.IsScpLocalPath(arg):
file_specs.append(LocalFile(arg))
else:
user_host, file_path = arg.split(':', 1)
user_host_parts = user_host.split('@', 1)
if len(user_host_parts) == 1:
user = ssh_utils.GetDefaultSshUsername(
warn_on_account_user=True)
instance = user_host_parts[0]
else:
user, instance = user_host_parts
file_specs.append(RemoteFile(user, instance, file_path))
log.debug('Normalized arguments: %s', file_specs)
# Validates the positional arguments.
# TODO(b/21515495): Look into relaxing these conditions.
sources = file_specs[:-1]
destination = file_specs[-1]
if isinstance(destination, LocalFile):
for source in sources:
if isinstance(source, LocalFile):
raise exceptions.ToolException(
'All sources must be remote files when the destination '
'is local.')
else: # RemoteFile
for source in sources:
if isinstance(source, RemoteFile):
raise exceptions.ToolException(
'All sources must be local files when the destination '
'is remote.')
instances = set()
for file_spec in file_specs:
if isinstance(file_spec, RemoteFile):
instances.add(file_spec.instance_name)
if len(instances) > 1:
raise exceptions.ToolException(
'Copies must involve exactly one virtual machine instance; '
'your invocation refers to [{0}] instances: [{1}].'.format(
len(instances), ', '.join(sorted(instances))))
instance_ref = self.CreateZonalReference(instances.pop(), args.zone)
instance = self.GetInstance(instance_ref)
external_ip_address = ssh_utils.GetExternalIPAddress(instance)
# Builds the scp command.
scp_args = [self.scp_executable]
if not args.plain:
scp_args.extend(self.GetDefaultFlags())
# apply args
if args.quiet:
scp_args.append('-q')
if args.port:
scp_args.extend(['-P', args.port])
if args.recurse:
scp_args.append('-r')
if args.compress:
scp_args.append('-C')
if args.scp_flag:
scp_args.extend(args.scp_flag)
for file_spec in file_specs:
if isinstance(file_spec, LocalFile):
scp_args.append(file_spec.file_path)
else:
scp_args.append('{0}:{1}'.format(
ssh_utils.UserHost(file_spec.user, external_ip_address),
file_spec.file_path))
self.ActuallyRun(args, scp_args, user, instance)
def Run(self, args):
super(CopyFiles, self).Run(args)
file_specs = []
# Parses the positional arguments.
for arg in args.sources + [args.destination]:
if ssh.IsScpLocalPath(arg):
file_specs.append(LocalFile(arg))
else:
user_host, file_path = arg.split(':', 1)
user_host_parts = user_host.split('@', 1)
if len(user_host_parts) == 1:
user = ssh.GetDefaultSshUsername(warn_on_account_user=True)
source_instance = user_host_parts[0]
else:
user, source_instance = user_host_parts
file_specs.append(RemoteFile(user, source_instance, file_path))
log.debug('Normalized arguments: %s', file_specs)
# Validates the positional arguments.
# TODO(user): Look into relaxing these conditions.
sources = file_specs[:-1]
destination = file_specs[-1]
if isinstance(destination, LocalFile):
for source in sources:
if isinstance(source, LocalFile):
raise exceptions.ToolException(
'All sources must be remote files when the destination '
'is local.')
else: # RemoteFile
for source in sources:
if isinstance(source, RemoteFile):
raise exceptions.ToolException(
'All sources must be local files when the destination '
'is remote.')
destination_instances = set()
for file_spec in file_specs:
if isinstance(file_spec, RemoteFile):
destination_instances.add(file_spec.instance_name)
if len(destination_instances) > 1:
raise exceptions.ToolException(
'Copies must involve exactly one virtual machine instance; '
'your invocation refers to [{0}] instances: [{1}].'.format(
len(destination_instances),
', '.join(sorted(destination_instances))))
source_instance_ref = instance_flags.SSH_INSTANCE_RESOLVER.ResolveResources(
[source_instance],
compute_scope.ScopeEnum.ZONE,
args.zone,
self.resources,
scope_lister=flags.GetDefaultScopeLister(self.compute_client,
self.project))[0]
source_instance = self.GetInstance(source_instance_ref)
external_ip_address = ssh_utils.GetExternalIPAddress(source_instance)
# Builds the scp command.
scp_args = [self.env.scp]
if not args.plain:
scp_args.extend(ssh.GetDefaultFlags(self.keys.key_file))
host_key_alias = self.HostKeyAlias(source_instance)
scp_args.extend(
ssh.GetHostKeyArgs(host_key_alias, args.plain,
args.strict_host_key_checking))
scp_args.append('-r')
for file_spec in file_specs:
if isinstance(file_spec, LocalFile):
scp_args.append(file_spec.file_path)
else:
scp_args.append('{0}:{1}'.format(
ssh.UserHost(file_spec.user, external_ip_address),
file_spec.file_path))
self.ActuallyRun(args, scp_args, user, source_instance,
source_instance_ref.project)
def BackupConfiguration(sql_messages,
instance=None,
backup_enabled=None,
backup_location=None,
backup_start_time=None,
enable_bin_log=None,
enable_point_in_time_recovery=None):
"""Generates the backup configuration for the instance.
Args:
sql_messages: module, The messages module that should be used.
instance: sql_messages.DatabaseInstance, the original instance, if the
previous state is needed.
backup_enabled: boolean, True if backup should be enabled.
backup_location: string, location where to store backups by default.
backup_start_time: string, start time of backup specified in 24-hour format.
enable_bin_log: boolean, True if binary logging should be enabled.
enable_point_in_time_recovery: boolean, True if point-in-time recovery
(using write-ahead log archiving) should be enabled.
Returns:
sql_messages.BackupConfiguration object, or None
Raises:
ToolException: Bad combination of arguments.
"""
should_generate_config = any([
backup_location is not None,
backup_start_time,
enable_bin_log is not None,
enable_point_in_time_recovery is not None,
not backup_enabled,
])
if not should_generate_config:
return None
if not instance or not instance.settings.backupConfiguration:
backup_config = sql_messages.BackupConfiguration(
kind='sql#backupConfiguration',
startTime='00:00',
enabled=backup_enabled)
else:
backup_config = instance.settings.backupConfiguration
if backup_location is not None:
backup_config.location = backup_location
backup_config.enabled = True
if backup_start_time:
backup_config.startTime = backup_start_time
backup_config.enabled = True
if not backup_enabled:
if backup_location is not None or backup_start_time:
raise exceptions.ToolException(
'Argument --no-backup not allowed with --backup_location or '
'--backup-start-time')
backup_config.enabled = False
if enable_bin_log is not None:
backup_config.binaryLogEnabled = enable_bin_log
if enable_point_in_time_recovery is not None:
backup_config.pointInTimeRecoveryEnabled = enable_point_in_time_recovery
return backup_config
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Raises:
ToolException: if no query was provided.
Returns:
If the --dry_run or --async flag was specified, None; otherwise, a
SchemaAndRows object.
"""
apitools_client = self.context[commands.APITOOLS_CLIENT_KEY]
bigquery_messages = self.context[commands.BIGQUERY_MESSAGES_MODULE_KEY]
resource_parser = self.context[commands.BIGQUERY_REGISTRY_KEY]
project_id = properties.VALUES.core.project.Get(required=True)
if not args.sql_query:
raise exceptions.ToolException('No query string provided')
destination_table = args.append_to or args.write_to
if destination_table:
output_resource = resource_parser.Parse(
destination_table, collection='bigquery.tables')
output_reference = message_conversions.TableResourceToReference(
bigquery_messages, output_resource)
else:
output_reference = None
query_configuration = bigquery_messages.JobConfigurationQuery(
allowLargeResults=args.allow_large_results,
createDisposition='CREATE_NEVER' if args.require_cache else None,
# Set defaultDataset here if we choose to support a
# --default-dataset-in-query flag.
destinationTable=output_reference,
flattenResults=not args.structured,
preserveNulls=None,
priority='BATCH' if args.batch else None,
query=args.sql_query,
useQueryCache=args.use_cache,
writeDisposition=((args.append_to and 'WRITE_APPEND')
or (args.write_to and 'WRITE_TRUNCATE')))
job = job_control.ExecuteJob(
apitools_client,
bigquery_messages,
args,
configuration=bigquery_messages.JobConfiguration(
query=query_configuration, dryRun=args.dry_run),
async=args. async,
project_id=project_id,
job_id=job_ids.JobIdProvider().GetJobId(args.job_id,
args.fingerprint_job_id))
if args.dry_run:
log.Print(
'Query successfully validated. Assuming the tables are not '
'modified, running this query will process {0} bytes of data.'.
format(job.statistics.query.totalBytesProcessed))
elif args. async:
job_resource = resource_parser.Create(
'bigquery.jobs',
projectId=job.jobReference.projectId,
jobId=job.jobReference.jobId)
log.CreatedResource(job_resource)
else:
return schema_and_rows.GetJobSchemaAndRows(apitools_client,
bigquery_messages,
job.jobReference,
args.start_row,
args.limit)
def Run(self, args):
super(CopyFiles, self).Run(args)
file_specs = []
# Parses the positional arguments.
for arg in args.sources + [args.destination]:
# If the argument begins with "./" or "/", then we are dealing
# with a local file that can potentially contain colons, so we
# avoid splitting on colons. The case of remote files containing
# colons is handled below by splitting only on the first colon.
if arg.startswith('./') or arg.startswith('/'):
file_specs.append(LocalFile(arg))
continue
host_file_parts = arg.split(':', 1)
if len(host_file_parts) == 1:
file_specs.append(LocalFile(host_file_parts[0]))
else:
user_host, file_path = host_file_parts
user_host_parts = user_host.split('@', 1)
if len(user_host_parts) == 1:
user = getpass.getuser()
instance = user_host_parts[0]
else:
user, instance = user_host_parts
file_specs.append(RemoteFile(user, instance, file_path))
logging.debug('Normalized arguments: %s', file_specs)
# Validates the positional arguments.
# TODO(user): Look into relaxing these conditions.
sources = file_specs[:-1]
destination = file_specs[-1]
if isinstance(destination, LocalFile):
for source in sources:
if isinstance(source, LocalFile):
raise exceptions.ToolException(
'All sources must be remote files when the destination '
'is local.')
else: # RemoteFile
for source in sources:
if isinstance(source, RemoteFile):
raise exceptions.ToolException(
'All sources must be local files when the destination '
'is remote.')
instances = set()
for file_spec in file_specs:
if isinstance(file_spec, RemoteFile):
instances.add(file_spec.instance_name)
if len(instances) > 1:
raise exceptions.ToolException(
'Copies must involve exactly one virtual machine instance; '
'your invocation refers to [{0}] instances: [{1}].'.format(
len(instances), ', '.join(sorted(instances))))
instance_ref = self.CreateZonalReference(instances.pop(), args.zone)
external_ip_address = self.GetInstanceExternalIpAddress(instance_ref)
# Builds the scp command.
scp_args = [self.scp_executable]
if not args.plain:
scp_args.extend(self.GetDefaultFlags())
scp_args.append('-r')
for file_spec in file_specs:
if isinstance(file_spec, LocalFile):
scp_args.append(file_spec.file_path)
else:
scp_args.append('{0}:{1}'.format(
ssh_utils.UserHost(file_spec.user, external_ip_address),
file_spec.file_path))
self.ActuallyRun(args, scp_args, user, external_ip_address)
def Run(self, args):
if not args.all and not args.keys:
raise calliope_exceptions.ToolException(
'One of [--all] or [--keys] must be provided.')
return super(BaseMetadataRemover, self).Run(args)
