def Filter(self, context, args):
"""Setup the API client within the context for this group's commands.
Args:
context: {str:object}, A set of key-value pairs that can be used for
common initialization among commands.
args: argparse.Namespace: The same namespace given to the corresponding
.Run() invocation.
"""
cloud_resources.SetParamDefault(
api='dataflow', collection=None, param='projectId',
resolver=resolvers.FromProperty(properties.VALUES.core.project))
context[DATAFLOW_MESSAGES_MODULE_KEY] = dataflow_v1b3_messages
context[DATAFLOW_APITOOLS_CLIENT_KEY] = (
dataflow_v1b3_client.DataflowV1b3(
url=properties.VALUES.api_endpoint_overrides.dataflow.Get(),
get_credentials=False,
http=self.Http()))
context[DATAFLOW_REGISTRY_KEY] = cloud_resources.REGISTRY
if args.endpoint:
log.warn('The --endpoint flag is deprecated and will be removed. '
'Set the api_endpoint_overrides/dataflow property instead. '
'e.g. gcloud config set api_endpoint_overrides/dataflow '
'https://www.googleapis.com/dataflow/v1b3')
def _AddPositionalToSummary(command, summary, length_per_snippet,
location, terms):
"""Adds summary of arg, given location such as ['positionals']['myarg']."""
positionals = command.get(lookup.POSITIONALS)
lines = []
line = ''
if _FormatHeader(lookup.POSITIONALS) not in summary:
lines.append(_FormatHeader(lookup.POSITIONALS))
# Add specific positional if given in location.
if len(location) > 1:
lines.append(_FormatItem(location[1]))
positionals = [p for p in positionals if p[lookup.NAME] == location[1]]
if positionals:
positional = positionals[0]
line = positional.get(lookup.DESCRIPTION, '')
line = _Snip(line, length_per_snippet, terms)
else:
log.warn('Attempted to look up a location [{}] that was not '
'found.'.format(location[1]))
# If no specific positional given, just add list of all available.
else:
line = ', '.join(sorted([p[lookup.NAME] for p in positionals]))
if line:
lines.append(line)
summary += lines
def ListEmulators(self):
"""Returns the list of emulators, or None.
Returns:
A list of emulator objects (Json dicts), or None if the list operation
fails.
Raises:
BrokerNotRunningError: If the broker is not running.
"""
if not self.IsRunning():
raise BrokerNotRunningError('Failed to list emulators')
try:
response, data = self._SendJsonRequest('GET', _EmulatorPath())
if response.status != httplib.OK:
log.warn('Failed to list emulators: {0} ({1})'.format(response.reason,
response.status))
return
except IOError:
return
list_response = json.loads(data)
try:
return list_response['emulators']
except KeyError:
# The expected values were not present.
return
def Run(self, args):
log.warn('Please use instead [gcloud compute instance-groups '
'managed set-autoscaling].')
client = self.context['autoscaler-client']
messages = self.context['autoscaler_messages_module']
resources = self.context['autoscaler_resources']
try:
autoscaler_ref = resources.Parse(
args.name, collection='autoscaler.autoscalers')
request = messages.AutoscalerAutoscalersUpdateRequest()
request.project = autoscaler_ref.project
request.zone = autoscaler_ref.zone
request.autoscaler = autoscaler_ref.autoscaler
request.autoscalerResource = util.PrepareAutoscaler(
args, messages, resources)
request.autoscalerResource.name = autoscaler_ref.autoscaler
result = client.autoscalers.Update(request)
if result.error:
raise exceptions.Error(util.GetErrorMessage(result.error))
operation_ref = resources.Create(
'autoscaler.zoneOperations',
operation=result.name,
autoscaler=request.autoscaler,
)
util.WaitForOperation(client, operation_ref,
'Updating Autoscaler instance')
log.status.write('Updated [{0}].\n'.format(args.name))
except exceptions.HttpError as error:
raise calliope_exceptions.HttpException(util.GetErrorMessage(error))
except ValueError as error:
raise calliope_exceptions.HttpException(error)
def Commit(self):
"""Commits the transaction, making the new app version available.
All the files returned by Begin() must have been uploaded with UploadFile()
before Commit() can be called.
This tries the new 'deploy' method; if that fails it uses the old 'commit'.
Returns:
An appinfo.AppInfoSummary if one was returned from the Deploy, None
otherwise.
Raises:
Error: Some required files were not uploaded.
CannotStartServingError: Another operation is in progress on this version.
"""
assert self.in_transaction, 'Begin() must be called before Commit().'
if self.files:
raise Error('Not all required files have been uploaded.')
def PrintRetryMessage(_, delay):
log.debug('Will check again in %s seconds.' % delay)
app_summary = self.Deploy()
# These backoff numbers seem reasonable and allow up to 15 minutes.
success, unused_contents = util.RetryWithBackoff(
lambda: (self.IsReady(), None), PrintRetryMessage, 1, 2, 60, 20)
if not success:
# TODO(mblain): Nicer exception, and handle printing it below.
log.warn('Version still not ready to serve, aborting.')
raise Error('Version not ready.')
result = self.StartServing()
if not result:
# This is an old version of the admin console (i.e. 1.5.0 or
# earlier). The update is now complete.
self.in_transaction = False
else:
if result == '0':
raise CannotStartServingError(
'Another operation on this version is in progress.')
success, response = util.RetryNoBackoff(self.IsServing, PrintRetryMessage)
if not success:
# TODO(mblain): Nicer exception, and handle printing it below.
log.warn('Version still not serving, aborting.')
raise Error('Version not ready.')
# If the server indicates that the Google Cloud Endpoints configuration
# is going to be updated, wait until that configuration is updated.
check_config_updated = response.get('check_endpoints_config')
if check_config_updated:
unused_done, (last_state, user_error) = util.RetryWithBackoff(
self.IsEndpointsConfigUpdated,
PrintRetryMessage, 1, 2, 60, 20)
if last_state != EndpointsState.SERVING:
self.HandleEndpointsError(user_error)
self.in_transaction = False
return app_summary
def Shutdown(self, wait_secs=10):
"""Shuts down the broker server."""
if self._process:
try:
exec_utils.KillSubprocess(self._process)
self._process = None
if self._comm_thread:
self._comm_thread.join()
self._comm_thread = None
except RuntimeError as e:
log.warn('Failed to shutdown broker: %s' % e)
raise BrokerError('Broker failed to shutdown: %s' % e)
else:
# Invoke the /shutdown handler.
try:
self._SendJsonRequest('POST', '/shutdown')
except IOError:
raise BrokerError('Broker failed to shutdown: '
'failed to send shutdown request')
except httplib.HTTPException:
# We may get an exception reading the response to the shutdown request,
# because the shutdown may preempt the response.
pass
if not _Await(lambda: not self.IsRunning(), wait_secs):
log.warn('Failed to shutdown broker: still running after {0}s'.format(
wait_secs))
raise BrokerError('Broker failed to shutdown: timed-out')
log.info('Shutdown broker.')
def CreateEmulator(self,
emulator_id,
path,
args,
target_patterns,
resolved_host=None):
"""Creates a new emulator entry."""
if not self.IsRunning():
raise BrokerNotRunningError('Failed to create emulator')
emulator = {
'emulator_id': emulator_id,
'start_command': {
'path': path,
'args': args,
},
'rule': {
'rule_id': emulator_id,
'target_patterns': target_patterns,
}
}
if resolved_host:
emulator['rule']['resolved_host'] = resolved_host
url = _EmulatorPath()
body = json.dumps(emulator)
try:
response, data = self._SendJsonRequest('POST', url, body=body)
if response.status != httplib.OK:
log.warn('Failed to create emulator: {0} ({1})'.format(response.reason,
response.status))
raise BrokerError('Failed to create emulator: %s' % data)
except IOError:
raise BrokerError('Failed to create emulator: '
'failed to send create request')
def CheckKubectlInstalled():
if config.Paths().sdk_root is not None:
platform = platforms.Platform.Current()
manager = update_manager.UpdateManager(platform_filter=platform, warn=False)
installed_components = manager.GetCurrentVersionsInformation()
if 'kubectl' not in installed_components:
log.warn(MISSING_KUBECTL_MSG)
def _PrepareSSHKeysValue(ssh_keys):
"""Returns a string appropriate for the metadata.
Values from are taken from the tail until either all values are
taken or _MAX_METADATA_VALUE_SIZE_IN_BYTES is reached, whichever
comes first. The selected values are then reversed. Only values at
the head of the list will be subject to removal.
Args:
ssh_keys: A list of keys. Each entry should be one key.
Returns:
A new-line-joined string of SSH keys.
"""
keys = []
bytes_consumed = 0
for key in reversed(ssh_keys):
num_bytes = len(key + '\n')
if bytes_consumed + num_bytes > constants.MAX_METADATA_VALUE_SIZE_IN_BYTES:
log.warn('The following SSH key will be removed from your project '
'because your sshKeys metadata value has reached its '
'maximum allowed size of {0} bytes: {1}'
.format(constants.MAX_METADATA_VALUE_SIZE_IN_BYTES, key))
else:
keys.append(key)
bytes_consumed += num_bytes
keys.reverse()
return '\n'.join(keys)
def FromFile(file_path):
"""Parses the given service file.
Args:
file_path: str, The full path to the service file.
Raises:
YamlParseError: If the file is not a valid Yaml-file.
YamlValidationError: If validation of parsed info fails.
Returns:
A ServiceYamlInfo object for the parsed file.
"""
try:
parsed = _YamlInfo._ParseYaml(file_path, appinfo_includes.Parse)
except (yaml_errors.Error, appinfo_errors.Error) as e:
raise YamlParseError(file_path, e)
if parsed.runtime == 'vm':
vm_runtime = parsed.GetEffectiveRuntime()
else:
vm_runtime = None
if parsed.runtime == 'python':
raise YamlValidationError(
'Service [{service}] uses unsupported Python 2.5 runtime. '
'Please use [runtime: python27] instead.'.format(
service=parsed.module))
elif parsed.runtime == 'python-compat':
raise YamlValidationError(
'"python-compat" is not a supported runtime.')
elif parsed.runtime == 'custom' and not parsed.env:
raise YamlValidationError(
'runtime "custom" requires that env be explicitly specified.')
if util.IsFlex(parsed.env) and vm_runtime == 'python27':
raise YamlValidationError(
'The "python27" is not a valid runtime in env: 2. '
'Please use [python-compat] instead.')
if parsed.module:
log.warn('The "module" parameter in application .yaml files is '
'deprecated. Please use the "service" parameter instead.')
else:
parsed.module = parsed.service or ServiceYamlInfo.DEFAULT_SERVICE_NAME
_CheckIllegalAttribute(
name='application',
yaml_info=parsed,
extractor_func=lambda yaml: yaml.application,
file_path=file_path,
msg=HINT_PROJECT)
_CheckIllegalAttribute(
name='version',
yaml_info=parsed,
extractor_func=lambda yaml: yaml.version,
file_path=file_path,
msg=HINT_VERSION)
return ServiceYamlInfo(file_path, parsed)
def Run(self, args):
contexts = context_util.CalculateExtendedSourceContexts(
args.source_directory)
# First create the old-style source-context.json file
if args.output_file:
log.warn(
'The --output-file option is deprecated and will soon be removed.')
output_directory = os.path.dirname(args.output_file)
output_file = args.output_file
else:
output_directory = ''
output_file = OLD_SOURCE_CONTEXT_FILENAME
if not output_directory:
if args.output_directory:
output_directory = args.output_directory
output_file = os.path.join(output_directory, output_file)
else:
output_directory = '.'
best_context = context_util.BestSourceContext(contexts,
args.source_directory)
files.MakeDir(output_directory)
with open(output_file, 'w') as f:
json.dump(best_context, f, indent=2, sort_keys=True)
# Create the new source-contexts.json file.
if args.output_directory and args.output_directory != output_directory:
output_directory = args.output_directory
files.MakeDir(output_directory)
with open(
os.path.join(output_directory, SOURCE_CONTEXTS_FILENAME), 'w') as f:
json.dump(contexts, f, indent=2, sort_keys=True)
def StopEmulator(self, emulator_id):
"""Stops the specified emulator via the broker, which must be running.
Args:
emulator_id: (str) The id of the emulator to stop.
Returns:
True if the emulator is stopped or wasn't running to begin with. False
if the emulator could not be stopped or is unknown.
Raises:
BrokerNotRunningError: If the broker is not running.
BrokerError: If the emulator could not be stopped for another reason.
"""
if not self.IsRunning():
raise BrokerNotRunningError('Failed to stop emulator: %s' % emulator_id)
url = _EmulatorPath(emulator_id, verb='stop')
try:
response, data = self._SendJsonRequest('POST', url)
if response.status != httplib.OK:
log.warn('Failed to stop emulator {0}: {1} ({2})'.format(
emulator_id, response.reason, response.status))
raise BrokerError('Failed to stop emulator: %s' % data)
except IOError:
raise BrokerError('Failed to stop emulator: failed to send stop request')
def GetObjectRef(path):
"""Build an Object proto message from a GCS path.
Args:
path: The GCS path of the form "gs://<bucket>/<object>"
Returns:
A proto message of the parsed objects
Raises:
ToolException: If there is a parsing issue or the bucket is unspecified.
"""
# TODO(pclay): Let resources.Parse take GCS paths.
url = urlparse.urlparse(path)
if url.scheme != STORAGE_SCHEME:
log.warn(path)
log.warn(url)
raise exceptions.ToolException(
'Invalid scheme [{0}] for a GCS path [{1}].'.format(url.scheme, path))
gcs_bucket = url.netloc
gcs_object = url.path.lstrip('/')
if not gcs_bucket:
raise exceptions.ToolException(
'Missing bucket in GCS path [{0}].'.format(path))
if not gcs_object:
raise exceptions.ToolException(
'Missing object in GCS path [{0}].'.format(path))
return messages.Object(bucket=gcs_bucket, name=gcs_object)
def _KeyFilesAreValid(private=None, public=None):
"""Returns True if private and public pass minimum key file requirements.
Args:
private: The private key file path.
public: The public key file path.
Returns:
True if private and public meet minumum key file requirements.
"""
# The private key file must be readable and non-empty.
if not _WarnOrReadFirstKeyLine(private, 'private'):
return False
# The PuTTY PPK key file must be readable and non-empty.
if IsRunningOnWindows() and not _WarnOrReadFirstKeyLine(private + '.ppk',
'PuTTY PPK'):
return False
# The public key file must be readable and non-empty.
public_line = _WarnOrReadFirstKeyLine(public, 'public')
if not public_line:
return False
# The remaining checks are for the public key file.
# Must have at least 2 space separated fields.
fields = public_line.split(' ')
if len(fields) < 2 or _IsPublicKeyCorrupt95Through97(fields[1]):
log.warn('The public SSH key file for Google Compute Engine is corrupt.')
return False
# Looks OK.
return True
def Run(self, args):
if ':' in properties.VALUES.core.project.Get(required=True):
raise UnsupportedAppIdError(
'`browse` command is currently unsupported for app IDs with custom '
'domains.')
client = appengine_api_client.GetApiClient(self.Http(timeout=None))
services = client.ListServices()
versions = version_util.GetMatchingVersions(client.ListVersions(services),
args.versions, args.service,
client.project)
if not args.service and not any('/' in v for v in args.versions):
# If no resource paths were provided and the service was not specified,
# assume the default service.
versions = [v for v in versions if v.service == 'default']
if not versions:
log.warn('No matching versions found.')
for version in versions:
# Assume HTTPS. There's not enough information to determine based on the
# results of ListVersions, but HTTPS is always more secure (though HTTP
# will work in all cases, since it will redirect to HTTPS).
url = deploy_command_util.GetAppHostname(version.project, version.service,
version.id,
use_ssl=appinfo.SECURE_HTTPS)
log.status.Print(
'Opening [{0}] in a new tab in your default browser.'.format(url))
OpenInBrowser(url)
def _WarnOrReadFirstKeyLine(path, kind):
"""Returns the first line from the key file path.
A None return indicates an error and is always accompanied by a log.warn
message.
Args:
path: The path of the file to read from.
kind: The kind of key file, 'private' or 'public'.
Returns:
None (and prints a log.warn message) if the file does not exist, is not
readable, or is empty. Otherwise returns the first line utf8 decoded.
"""
try:
with open(path) as f:
# Decode to utf8 to handle any unicode characters. Key data is base64
# encoded so it cannot contain any unicode. Comments may contain unicode,
# but they are ignored in the key file analysis here, so replacing invalid
# chars with ? is OK.
line = f.readline().strip().decode('utf8', 'replace')
if line:
return line
msg = 'is empty'
except IOError as e:
if e.errno == errno.ENOENT:
msg = 'does not exist'
else:
msg = 'is not readable'
log.warn('The %s SSH key file for Google Compute Engine %s.', kind, msg)
return None
def PrepareGCDDataDir(data_dir):
"""Prepares the given directory using gcd create.
Raises:
UnableToPrepareDataDir: If the gcd create execution fails.
Args:
data_dir: str, Path of data directy to be prepared.
"""
if os.path.isdir(data_dir) and os.listdir(data_dir):
log.warn('Reusing existing data in [{0}].'.format(data_dir))
return
gcd_create_args = ['create']
project = properties.VALUES.core.project.Get(required=True)
gcd_create_args.append('--project_id={0}'.format(project))
gcd_create_args.append(data_dir)
exec_args = ArgsForGCDEmulator(*gcd_create_args)
log.status.Print('Executing: {0}'.format(' '.join(exec_args)))
process = util.Exec(exec_args)
util.PrefixOutput(process, DATASTORE)
failed = process.poll()
if failed:
raise UnableToPrepareDataDir()
def Run(self, args):
log.warn('Please use instead [gcloud compute instance-groups '
'managed set-autoscaling].')
messages = self.context['autoscaler_messages_module']
client = self.context['autoscaler-client']
resources = self.context['autoscaler_resources']
autoscaler_ref = resources.Parse(
args.name, collection='autoscaler.autoscalers')
try:
request = messages.AutoscalerAutoscalersInsertRequest()
request.project = autoscaler_ref.project
request.zone = autoscaler_ref.zone
request.autoscaler = util.PrepareAutoscaler(args, messages, resources)
request.autoscaler.name = autoscaler_ref.autoscaler
result = client.autoscalers.Insert(request)
operation_ref = resources.Create(
'autoscaler.zoneOperations',
operation=result.name,
autoscaler=request.autoscaler,
)
if args.async:
return client.zoneOperations.Get(operation_ref.Request())
util.WaitForOperation(client, operation_ref,
'Creating Autoscaler instance')
log.CreatedResource(autoscaler_ref)
return client.autoscalers.Get(autoscaler_ref.Request())
except exceptions.HttpError as error:
raise calliope_exceptions.HttpException(util.GetErrorMessage(error))
except ValueError as error:
raise calliope_exceptions.HttpException(error)
def CreateRequests(self, args):
"""Returns the request necessary for adding the network."""
# TODO(user): after one month, make default auto.
if args.mode is None:
if args.range is not None:
log.warn('You are creating a legacy network. Using --mode=legacy will '
'be required in future releases.')
args.mode = 'legacy'
else:
args.mode = 'auto'
if args.mode != 'legacy' and args.range is not None:
raise exceptions.InvalidArgumentException(
'--range', '--range can only be used if --mode=legacy')
network_ref = self.CreateGlobalReference(
args.name, resource_type='networks')
if args.mode == 'legacy':
return [self.messages.ComputeNetworksInsertRequest(
network=self.messages.Network(
name=network_ref.Name(),
IPv4Range=args.range,
description=args.description),
project=self.project)]
request = self.messages.ComputeNetworksInsertRequest(
network=self.messages.Network(
name=network_ref.Name(),
autoCreateSubnetworks=args.mode == 'auto',
description=args.description),
project=self.project)
return [request]
def CallKubectl(self, c_config, kubectl_args):
"""Shell out to call to kubectl tool.
Args:
c_config: ClusterConfig object for cluster.
kubectl_args: specific args to call kubectl with (not including args
for authentication).
Returns:
(output, error), where
output: str, raw output of the kubectl command.
error: subprocess.CalledProcessError, if the command exited with
non-zero status, None if command exited with success.
"""
base_args = [
'--kubeconfig=%s' % kconfig.Kubeconfig.DefaultPath(),
'--context=%s' % c_config.kube_context,
]
if not c_config.has_certs:
log.warn('No certificate files found in %s. Certificate checking '
'disabled for calls to cluster master.', c_config.config_dir)
args = ['kubectl'] + base_args + kubectl_args
try:
log.debug('Calling \'%s\'', repr(args))
output = subprocess.check_output(args, stderr=subprocess.STDOUT)
return (output, None)
except subprocess.CalledProcessError as error:
return (error.output, error)
def Run(self, args):
"""Run 'service-management operations describe'.
Args:
args: argparse.Namespace, The arguments that this command was invoked
with.
Returns:
The response from the operations.Get API call.
"""
# If a user includes the leading "operations/", just strip it off
if args.operation.startswith(OPTIONAL_PREFIX_TO_STRIP):
args.operation = args.operation[len(OPTIONAL_PREFIX_TO_STRIP):]
request = self.services_messages.ServicemanagementOperationsGetRequest(
operationsId=args.operation,
)
operation = self.services_client.operations.Get(request)
if (sys.getsizeof(str(operation.response)) > MAX_RESPONSE_BYTES and
not args.full):
log.warn('Response portion of Operation redacted. '
'Use --full to see the whole Operation.\n')
operation.response = None
# Set async to True because we don't need to wait for the operation
# to complete to check the status of it.
return services_util.GetProcessedOperationResult(operation, async=True)
def Generate():
try:
with open(name, 'w') as f:
json.dump(json_object, f)
except IOError as e:
log.warn('Could not generate [{0}]: {1}'.format(name, e))
return Cleanup
def CreateAppInteractively(api_client, project):
"""Interactively choose a region and create an App Engine app.
The caller is responsible for calling this method only when the user can be
prompted interactively.
Example interaction:
Which region?
[1] us-east1 (supports standard and flexible)
[2] europe-west (supports standard)
[3] us-central (supports standard and flexible)
[4] cancel
Please enter your numeric choice: 1
Args:
api_client: The App Engine Admin API client
project: The GCP project
Raises:
AppAlreadyExistsError if app already exists
"""
log.status.Print('You are creating an app for project [{}].'.format(project))
log.warn(APP_CREATE_WARNING)
all_regions = sorted(set(api_client.ListRegions()))
idx = console_io.PromptChoice(
all_regions,
message=('Please choose the region where you want your App Engine '
'application located:\n\n'),
cancel_option=True)
region = all_regions[idx]
CreateApp(api_client, project, region.region, suppress_warning=True)
def GetDefaultSshUsername(warn_on_account_user=False):
"""Returns the default username for ssh.
The default username is the local username, unless that username is invalid.
In that case, the default username is the username portion of the current
account.
Emits a warning if it's not using the local account username.
Args:
warn_on_account_user: bool, whether to warn if using the current account
instead of the local username.
Returns:
str, the default SSH username.
"""
user = getpass.getuser()
if not _IsValidSshUsername(user):
full_account = properties.VALUES.core.account.Get(required=True)
account_user = gaia_utils.MapGaiaEmailToDefaultAccountName(full_account)
if warn_on_account_user:
log.warn('Invalid characters in local username [{0}]. '
'Using username corresponding to active account: [{1}]'.format(
user, account_user))
user = account_user
return user
def Run(self, args):
"""Run 'service-management operations describe'.
Args:
args: argparse.Namespace, The arguments that this command was invoked
with.
Returns:
The response from the operations.Get API call.
Raises:
HttpException: An http error response was received while executing api
request.
"""
# If a user includes the leading "operations/", just strip it off
if args.operation.startswith(OPTIONAL_PREFIX_TO_STRIP):
args.operation = args.operation[len(OPTIONAL_PREFIX_TO_STRIP):]
request = self.services_messages.ServicemanagementOperationsGetRequest(
operationsId=args.operation,
)
try:
result = self.services_client.operations.Get(request)
except apitools_exceptions.HttpError as error:
raise exceptions.HttpException(services_util.GetError(error))
if not args.full:
log.warn('Response portion of Operation redacted. '
'Use --full to see the whole Operation.\n')
result.response = None
# Set async to True because we don't need to wait for the operation
# to complete to check the status of it.
return services_util.ProcessOperationResult(result, async=True)
def CreateApp(api_client, project, region, suppress_warning=False):
"""Create an App Engine app in the given region.
Prints info about the app being created and displays a progress tracker.
Args:
api_client: The App Engine Admin API client
project: The GCP project
region: The region to create the app
suppress_warning: True if user doesn't need to be warned this is
irreversible.
Raises:
AppAlreadyExistsError if app already exists
"""
if not suppress_warning:
log.status.Print('You are creating an app for project [{project}].'.format(
project=project))
log.warn(APP_CREATE_WARNING)
try:
api_client.CreateApp(region)
except api_lib_exceptions.ConflictError:
raise AppAlreadyExistsError(
'The project [{project}] already contains an App Engine application. '
'You can deploy your application using `gcloud app deploy`.'.format(
project=project))
def Filter(self, context, args):
"""Context() is a filter function that can update the context.
Args:
context: The current context.
args: The argparse namespace that was specified on the CLI or API.
Returns:
The updated context.
"""
log.warn('This command will be soon removed.')
log.warn('Replica pools have been replaced by instance groups;'
'please use those instead.')
# pylint:disable=g-import-not-at-top, Delaying import for performance.
import apiclient.discovery as discovery
# Create the Replica Pool service client
api_server = properties.VALUES.core.api_host.Get()
# Re-using the compute API overrides from compute, which is not technically
# correct (since this is a different API), but we don't want to introduce
# another property only to remove it.
api_version = (properties.VALUES.api_client_overrides.compute.Get() or
'v1beta1')
discovery_url = ('{server}/discovery/v1/apis/replicapool/{version}/rest'
.format(server=api_server.rstrip('/'),
version=api_version))
http = self.Http()
client = discovery.build(
'replicapool', api_version, http=http,
discoveryServiceUrl=discovery_url)
context['replicapool'] = client
return context
def Run(self, args):
"""Run 'instance-groups delete'.
Args:
args: argparse.Namespace, The arguments that this command was invoked
with.
Raises:
HttpException: A http error response was received while executing api
request.
ToolException: An error other than http error occured while executing
the command.
"""
log.warn('Please use instead [gcloud compute instance-groups '
'unmanaged delete].')
client = self.context['instanceGroupsClient']
project = properties.VALUES.core.project.Get(required=True)
for group_name in args.name:
request = client.delete(
project=project, zone=args.zone, resourceView=group_name)
try:
request.execute()
log.Print('Instance group {0} deleted.'.format(group_name))
except errors.HttpError as error:
raise exceptions.HttpException(util.GetError(error))
except errors.Error as error:
raise exceptions.ToolException(error)
def GetPublicKey(self):
"""Generates an SSH key using ssh-key (if necessary) and returns it."""
public_ssh_key_file = self.ssh_key_file + '.pub'
if (not os.path.exists(self.ssh_key_file) or
not os.path.exists(public_ssh_key_file)):
log.warn('You do not have an SSH key for Google Compute Engine.')
log.warn('[%s] will be executed to generate a key.',
self.ssh_keygen_executable)
ssh_directory = os.path.dirname(public_ssh_key_file)
if not os.path.exists(ssh_directory):
if console_io.PromptContinue(
'This tool needs to create the directory [{0}] before being able '
'to generate SSH keys.'.format(ssh_directory)):
files.MakeDir(ssh_directory, 0700)
else:
raise exceptions.ToolException('SSH key generation aborted by user.')
keygen_args = [
self.ssh_keygen_executable,
'-t', 'rsa',
'-f', self.ssh_key_file,
]
_RunExecutable(keygen_args)
with open(public_ssh_key_file) as f:
return f.readline().strip()
def GetRuntimes(args):
"""Gets a list of unique runtimes that the user is about to run.
Args:
args: A list of arguments (typically sys.argv).
Returns:
A set of runtime strings.
"""
runtimes = set()
for arg in args:
# Check all the arguments to see if they're application yaml files.
if (os.path.isfile(arg) and
os.path.splitext(arg)[1] in _YAML_FILE_EXTENSIONS):
with open(arg) as f:
try:
info = yaml.safe_load(f)
# safe_load can return arbitrary objects, we need a dict.
if not isinstance(info, dict):
continue
# Grab the runtime from the yaml, if it exists.
if 'runtime' in info:
runtime = info.get('runtime')
if type(runtime) == str:
runtimes.add(runtime)
if runtime in _WARNING_RUNTIMES:
log.warn(_WARNING_RUNTIMES[runtime])
except yaml.YAMLError:
continue
return runtimes
def DoPrepareManagedVms(gae_client):
"""Call an API to prepare the for App Engine Flexible."""
try:
message = 'If this is your first deployment, this may take a while'
with progress_tracker.ProgressTracker(message):
# Note: this doesn't actually boot the VM, it just prepares some stuff
# for the project via an undocumented Admin API.
gae_client.PrepareVmRuntime()
log.status.Print()
except util.RPCError as err:
# Any failures later due to an unprepared project will be noisy, so it's
# okay not to fail here.
log.warn((
"We couldn't validate that your project is ready to deploy to App "
'Engine Flexible Environment. If deployment fails, please check the '
'following message and try again:\n') + str(err))
def GetPublicKey(self):
"""Generates an SSH key using ssh-keygen (if necessary) and returns it."""
public_ssh_key_file = self.ssh_key_file + '.pub'
if not _KeyFilesAreValid(private=self.ssh_key_file,
public=public_ssh_key_file):
log.warn('You do not have an SSH key for Google Compute Engine.')
log.warn('[%s] will be executed to generate a key.',
self.ssh_keygen_executable)
ssh_directory = os.path.dirname(public_ssh_key_file)
if not os.path.exists(ssh_directory):
if console_io.PromptContinue(
'This tool needs to create the directory [{0}] before being able '
'to generate SSH keys.'.format(ssh_directory)):
files.MakeDir(ssh_directory, 0700)
else:
raise exceptions.ToolException(
'SSH key generation aborted by user.')
# Remove the private key file to avoid interactive prompts.
try:
os.remove(self.ssh_key_file)
except OSError:
pass
keygen_args = [self.ssh_keygen_executable]
if platforms.OperatingSystem.IsWindows():
# No passphrase in the current implementation.
keygen_args.append(self.ssh_key_file)
else:
if properties.VALUES.core.disable_prompts.GetBool():
# Specify empty passphrase on command line
keygen_args.extend(['-P', ''])
keygen_args.extend([
'-t',
'rsa',
'-f',
self.ssh_key_file,
])
_RunExecutable(keygen_args)
with open(public_ssh_key_file) as f:
# We get back a unicode list of keys for the remaining metadata, so
# convert to unicode. Assume UTF 8, but if we miss a character we can just
# replace it with a '?'. The only source of issues would be the hostnames,
# which are relatively inconsequential.
return f.readline().strip().decode('utf8', 'replace')
def CreateContextFiles(output_dir, source_contexts, overwrite=False,
source_dir=None):
"""Creates source context files in the given directory.
Currently, two files will be produced, source-context.json and
source-contexts.json. The old-style source-context.json file is deprecated,
but will need to be produced until all components are updated to use the new
file. This process may take a while because there are managed VMs which may be
slow to update the debug agent to one that supports the new format.
The new format supports communicating multiple source contexts with labels to
enable the UI to chose the best contexts for a given situation.
Args:
output_dir: (String) The directory to create the files (usually the yaml
directory).
source_contexts: ([ExtendedSourceContext-compatible json dict])
A list of json-serializable dicts containing source contexts. If None
or empty, source context will be inferred from source_dir.
overwrite: (boolean) If true, silently replace any existing file.
source_dir: (String) The location of the source files, for inferring
source contexts when source_contexts is empty or None. If not
specified, output_dir will be used instead.
Returns:
([String]) A list containing the names of the files created.
"""
if not source_contexts:
source_contexts = _GetSourceContexts(source_dir or output_dir)
if not source_contexts:
return []
created = []
try:
for context_filename, context_object in [
(CONTEXT_FILENAME, BestSourceContext(source_contexts, output_dir)),
(EXT_CONTEXT_FILENAME, source_contexts)]:
context_filename = os.path.join(output_dir, context_filename)
if overwrite or not os.path.exists(context_filename):
with open(context_filename, 'w') as f:
json.dump(context_object, f)
created.append(context_filename)
except IOError as e:
log.warn('Could not generate [{0}]: {1}'.format(context_filename, e))
except GenerateSourceContextError as e:
log.warn('Could not select best source context [{0}]: {1}'.format(
source_contexts, e))
return created
def Run(self, args):
client = self.context['dataproc_client']
messages = self.context['dataproc_messages']
cluster_ref = util.ParseCluster(args.name, self.context)
request = messages.DataprocProjectsRegionsClustersDiagnoseRequest(
clusterName=cluster_ref.clusterName,
region=cluster_ref.region,
projectId=cluster_ref.projectId)
try:
operation = client.projects_regions_clusters.Diagnose(request)
# TODO(user): Stream output during polling.
operation = util.WaitForOperation(
operation, self.context,
message='Waiting for cluster diagnose operation')
response = operation.response
except apitools_exceptions.HttpError as error:
raise exceptions.HttpException(util.FormatHttpError(error))
if not response:
raise exceptions.ToolException('Operation is missing response')
properties = encoding.MessageToDict(response)
output_uri = properties['outputUri']
if not output_uri:
raise exceptions.ToolException('Response is missing outputUri')
log.err.Print('Output from diagnostic:')
log.err.Print('-----------------------------------------------')
driver_log_stream = storage_helpers.StorageObjectSeriesStream(
output_uri)
# A single read might not read whole stream. Try a few times.
read_retrier = retry.Retryer(max_retrials=4, jitter_ms=None)
try:
read_retrier.RetryOnResult(
lambda: driver_log_stream.ReadIntoWritable(log.err),
sleep_ms=100,
should_retry_if=lambda *_: driver_log_stream.open)
except retry.MaxRetrialsException:
log.warn(
'Diagnostic finished succesfully, '
'but output did not finish streaming.')
log.err.Print('-----------------------------------------------')
return output_uri
def BuildAndPushDockerImages(module_configs, version_id, client, cli, remote,
implicit_remote_build):
# PrepareVmRuntime only needs to be called once per deployment.
project = properties.VALUES.core.project.Get(required=True)
if any(info.RequiresImage() for info in module_configs.values()):
log.status.Print('Verifying that Managed VMs are enabled and ready.')
message = 'If this is your first deployment, this may take a while'
try:
with console_io.DelayedProgressTracker(message,
_PREPARE_VM_MESSAGE_DELAY):
client.PrepareVmRuntime()
log.status.Print()
except util.RPCError as err:
log.warn('If this is your first deployment, please try again.')
raise err
for registry in constants.ALL_SUPPORTED_REGISTRIES:
docker.UpdateDockerCredentials(registry)
if remote and implicit_remote_build:
# Test for presence of local Docker
try:
with docker_util.DockerHost(cli, version_id,
False) as docker_client:
if os.environ.get('DOCKER_HOST'):
docker_client.ping()
log.warn(
'A hosted build is being performed, but a local Docker '
'was found. Specify `--docker-build=local` to use it, or '
'`--docker-build=remote` to silence this warning.')
except containers.DockerDaemonConnectionError:
pass
with docker_util.DockerHost(cli, version_id, remote) as docker_client:
# Build and push all images.
for (module, info) in module_configs.iteritems():
if info.RequiresImage():
log.status.Print(
'Building and pushing image for module [{module}]'.
format(module=module))
info.UpdateManagedVMConfig()
push.BuildAndPushDockerImage(info.file, project, module,
version_id, info.runtime,
docker_client)
metric_name = _REMOTE_BUILD if remote else _BUILD
metrics.CustomTimedEvent(metric_name)
def UpdateCliTrees(cli=None, commands=None, directory=None,
force=False, verbose=False, warn_on_exceptions=False):
"""(re)generates the CLI trees in directory if non-existent or out ot date.
This function uses the progress tracker because some of the updates can
take ~minutes.
Args:
cli: The default CLI. If not None then the default CLI is also updated.
commands: Update only the commands in this list.
directory: The directory containing the CLI tree JSON files. If None
then the default installation directories are used.
force: Update all exitsing trees by forcing them to be out of date if True.
verbose: Display a status line for up to date CLI trees if True.
warn_on_exceptions: Emits warning messages in lieu of exceptions. Used
during installation.
Raises:
NoCliTreeGeneratorForCommand: A command in commands is not supported
(doesn't have a generator).
"""
directories = _GetDirectories(
directory=directory, warn_on_exceptions=warn_on_exceptions)
if not commands:
commands = set([cli_tree.DEFAULT_CLI_NAME] + GENERATORS.keys())
failed = []
for command in sorted(commands):
if command != cli_tree.DEFAULT_CLI_NAME:
tree = LoadOrGenerate(command,
directories=directories,
force=force,
verbose=verbose,
warn_on_exceptions=warn_on_exceptions)
if not tree:
failed.append(command)
elif cli:
cli_tree.Load(cli=cli,
path=cli_tree.CliTreePath(directory=directories[0]),
force=force, verbose=verbose)
if failed:
message = 'No CLI tree {} for [{}].'.format(
text_utils.Pluralize(len(failed), 'generator'),
', '.join(sorted(failed)))
if not warn_on_exceptions:
raise NoCliTreeGeneratorForCommand(message)
log.warn(message)
def _RunPredict(args):
"""Run ML Engine local prediction."""
instances = predict_utilities.ReadInstancesFromArgs(
args.json_instances, args.text_instances)
sdk_root = config.Paths().sdk_root
if not sdk_root:
raise LocalPredictEnvironmentError(
'You must be running an installed Cloud SDK to perform local '
'prediction.')
env = {'CLOUDSDK_ROOT': sdk_root}
# We want to use whatever the user's Python was, before the Cloud SDK started
# changing the PATH. That's where Tensorflow is installed.
python_executables = files.SearchForExecutableOnPath('python')
if not python_executables:
# This doesn't have to be actionable because things are probably beyond help
# at this point.
raise LocalPredictEnvironmentError(
'Something has gone really wrong; we can\'t find a valid Python '
'executable on your PATH.')
python_executable = python_executables[0]
# Start local prediction in a subprocess.
proc = subprocess.Popen([
python_executable, local_predict.__file__, '--model-dir',
args.model_dir
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
# Pass the instances to the process that actually runs local prediction.
for instance in instances:
proc.stdin.write(json.dumps(instance) + '\n')
proc.stdin.flush()
# Get the results for the local prediction.
output, err = proc.communicate()
if proc.returncode != 0:
raise LocalPredictRuntimeError(err)
if err:
log.warn(err)
try:
return json.loads(output)
except ValueError:
raise InvalidReturnValueError('The output for prediction is not '
'in JSON format: ' + output)
def Run(self, args):
info = info_holder.InfoHolder()
log_data = None
if args.log_file:
try:
log_data = info_holder.LogData.FromFile(args.log_file)
except IOError as err:
log.warn('Error reading the specified file [{0}]: '
'{1}\n'.format(args.log_file, err))
if args.quiet:
_PrintQuiet(str(info), log_data)
else:
log.status.Print(FEEDBACK_MESSAGE)
if not log_data:
log_data = _SuggestIncludeRecentLogs()
if console_io.PromptContinue(prompt_string=FEEDBACK_PROMPT):
_OpenNewIssueInBrowser(info, log_data)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace. All the arguments that were provided to this
command invocation.
Returns:
Some value that we want to have printed later.
"""
regions = args.regions
if regions is None:
log.warn(
'`--regions` flag will soon be required. Please explicitly '
'specify a region. Using [us-central1] by default.')
regions = ['us-central1']
return models.ModelsClient().Create(args.model, regions)
def __init__(self, *args, **kwargs):
"""Creates a new TablePrinter."""
self._rows = []
self._nest = []
super(TablePrinter, self).__init__(*args, by_columns=True,
non_empty_projection_required=True,
**kwargs)
encoding = None
for name in ['ascii', 'utf8', 'win']:
if name in self.attributes:
encoding = name
break
if not self._console_attr:
self._console_attr = console_attr.GetConsoleAttr(encoding=encoding,
out=self._out)
self._rows_per_page = self.attributes.get('page', 0)
if self._rows_per_page:
log.warn('The [page=N] printer attribute is deprecated. '
'Use the --page-size=N flag instead.')
self._page_count = 0
# Check for subformat columns.
self._subformats = []
has_subformats = False
self._aggregate = True
if self.column_attributes:
for col in self.column_attributes.Columns():
if col.attribute.subformat:
has_subformats = True
else:
self._aggregate = False
index = 0
for col in self.column_attributes.Columns():
if col.attribute.subformat:
# This initializes a Printer to a string stream.
out = self._out if self._aggregate else cStringIO.StringIO()
printer = self.Printer(col.attribute.subformat, out=out,
console_attr=self._console_attr)
else:
out = None
printer = None
self._subformats.append(SubFormat(index, printer, out))
index += 1
if not has_subformats:
self._subformats = None
self._aggregate = False
def _ValidateTriggerArgs(args):
"""Check if args related function triggers are valid.
Args:
args: parsed command line arguments.
Raises:
FunctionsError.
"""
if args.trigger_gs_uri is not None:
log.warn('--trigger-gs-uri flag is deprecated. Use --trigger-bucket '
'instead.')
if args.trigger_provider is None and (args.trigger_event is not None or
args.trigger_resource is not None):
raise exceptions.FunctionsError(
'--trigger-event, --trigger-resource, and --trigger-path may only '
'be used with --trigger-provider')
def RunLanguageCommand(feature, content_file=None, content=None,
language=None, content_type=None,
encoding_type=None,
api_version=util.LANGUAGE_GA_VERSION):
"""Runs a gcloud ml language command.
Args:
feature: str, the name of the feature being used, such as analyzeEntities.
content_file: str, the file to be used to analyze text.
content: str, the text to be analyzed.
language: str, the language of the input text.
content_type: str, the format of the input text - 'PLAIN_TEXT' or 'HTML'.
encoding_type: str, the encoding type to be used for calculating word
offsets - 'UTF8', 'UTF16', 'UTF32', 'NONE'.
api_version: str, the API version to use.
Raises:
ContentFileError: if content file can't be found and is not a GCS URL.
ContentError: if content is given but empty.
googlecloudsdk.api_lib.util.exceptions.HttpException: if the API returns
an error.
Returns:
the response from the API (type depends on feature, for example
if feature is analyzeEntities, response would be
messages.AnalyzeEntitiesResponse).
"""
entity_sentiment = True if feature == 'analyzeEntitySentiment' else False
classify_text = True if feature == 'classifyText' else False
client = util.LanguageClient(version=api_version,
entity_sentiment_enabled=entity_sentiment,
classify_text_enabled=classify_text)
source = util.GetContentSource(content, content_file)
try:
return client.SingleFeatureAnnotate(
feature,
source=source,
language=language,
content_type=content_type,
encoding_type=encoding_type)
except HttpError as e:
# Print Service Account Help on Access Denied errors
# as this is most likely cause
if e.status_code == 403:
log.warn('Please Note: {}\n'.format(SERVICE_ACCOUNT_HELP))
raise e
def WarnIfDiskSizeIsTooSmall(size_gb, disk_type):
"""Writes a warning message if the given disk size is too small."""
if not size_gb:
return
if disk_type and 'pd-ssd' in disk_type:
warning_threshold_gb = constants.SSD_DISK_PERFORMANCE_WARNING_GB
else:
warning_threshold_gb = constants.STANDARD_DISK_PERFORMANCE_WARNING_GB
if size_gb < warning_threshold_gb:
log.warn(
'You have selected a disk size of under [%sGB]. This may result in '
'poor I/O performance. For more information, see: '
'https://developers.google.com/compute/docs/disks/persistent-disks'
'#pdperformance.',
warning_threshold_gb)
def _GetCredHelperCommand(uri, full_path=False):
"""Returns the gcloud credential helper command for a remote repository.
The command will be of the form '!gcloud auth-git-helper --account=EMAIL
--ignore-unknown $@`. See https://git-scm.com/docs/git-config. If the
installed version of git or the remote repository does not support
the gcloud credential helper, then returns None.
Args:
uri: str, The uri of the remote repository.
full_path: bool, If true, use the full path to gcloud.
Returns:
str, The credential helper command if it is available.
"""
credentialed_hosts = ['source.developers.google.com']
extra = properties.VALUES.core.credentialed_hosted_repo_domains.Get()
if extra:
credentialed_hosts.extend(extra.split(','))
if any(uri.startswith('https://' + host) for host in credentialed_hosts):
try:
CheckGitVersion(_HELPER_MIN)
except GitVersionException as e:
helper_min = '.'.join(str(i) for i in _HELPER_MIN)
log.warn(
textwrap.dedent("""\
You are using a Google-hosted repository with a
{current} which is older than {min_version}. If you upgrade
to {min_version} or later, gcloud can handle authentication to
this repository. Otherwise, to authenticate, use your Google
account and the password found by running the following command.
$ gcloud auth print-access-token""".format(current=e.cur_version,
min_version=helper_min)))
return None
if _HasSystemCredHelper():
log.warn(
textwrap.dedent("""\
If your system's credential.helper requests a password, choose
cancel."""))
# Use git alias "!shell command" syntax so we can configure
# the helper with options. Also git-credential is not
# prefixed when it starts with "!".
return '!{0} auth git-helper --account={1} --ignore-unknown $@'.format(
_GetGcloudScript(full_path),
properties.VALUES.core.account.Get(required=True))
return None
def _GetImageUri(self, args, client, holder, instance_template_ref):
if (args.IsSpecified('image') or args.IsSpecified('image_family') or
args.IsSpecified('image_project')):
image_expander = image_utils.ImageExpander(client, holder.resources)
image_uri, _ = image_expander.ExpandImageFlag(
user_project=instance_template_ref.project,
image=args.image,
image_family=args.image_family,
image_project=args.image_project)
if holder.resources.Parse(image_uri).project != 'cos-cloud':
log.warn('This container deployment mechanism requires a '
'Container-Optimized OS image in order to work. Select an '
'image from a cos-cloud project (cost-stable, cos-beta, '
'cos-dev image families).')
else:
image_uri = containers_utils.ExpandKonletCosImageFlag(client)
return image_uri
def CreateEmulator(self,
emulator_id,
path,
args,
target_patterns,
resolved_host=None):
"""Creates a new emulator entry.
Args:
emulator_id: (str) The emulator id
path: (str) The path to the emulator binary.
args: (list of str) The command line arguments to the emulator.
target_patterns: (list or str) The regular expressions used to match
input targets for the emulator.
resolved_host: (str) The address to use when resolving the new emulator.
Only specified if the lifetime of this emulator is not managed by
this broker.
Raises:
BrokerNotRunningError: If the broker is not running.
BrokerError: If the creation failed.
"""
if not self.IsRunning():
raise BrokerNotRunningError('Failed to create emulator')
emulator = {
'emulator_id': emulator_id,
'start_command': {
'path': path,
'args': args,
},
'rule': {
'rule_id': emulator_id,
'target_patterns': target_patterns,
}
}
if resolved_host:
emulator['rule']['resolved_host'] = resolved_host
url = _EmulatorPath()
body = json.dumps(emulator)
response, data = self._SendJsonRequest('POST', url, body=body)
if response.status != httplib.OK:
log.warn('Failed to create emulator: {0} ({1})'.format(response.reason,
response.status))
raise BrokerError('Failed to create emulator: %s' % data)
def UploadSource(source_dir, bucket, obj, storage_client):
"""Upload a gzipped tarball of the source directory to GCS.
Note: To provide parity with docker's behavior, we must respect .dockerignore.
Args:
source_dir: the directory to be archived.
bucket: the GCS bucket where the tarball will be stored.
obj: the GCS object where the tarball will be stored, in the above bucket.
storage_client: An instance of the storage_v1.StorageV1 client.
Raises:
UploadFailedError: when the source fails to upload to GCS.
"""
dockerignore = os.path.join(source_dir, '.dockerignore')
exclude = None
if os.path.exists(dockerignore):
with open(dockerignore) as f:
# Read the exclusions, filtering out blank lines.
exclude = set(filter(bool, f.read().splitlines()))
# Remove paths that shouldn't be excluded on the client.
exclude -= set(BLACKLISTED_DOCKERIGNORE_PATHS)
# We can't use tempfile.NamedTemporaryFile here because ... Windows.
# See https://bugs.python.org/issue14243. There are small cleanup races
# during process termination that will leave artifacts on the filesystem.
# eg, CTRL-C on windows leaves both the directory and the file. Unavoidable.
# On Posix, `kill -9` has similar behavior, but CTRL-C allows cleanup.
try:
temp_dir = tempfile.mkdtemp()
f = open(os.path.join(temp_dir, 'src.tgz'), 'w+b')
# We are able to leverage the source archiving code from docker-py;
# however, there are two wrinkles:
# 1) The 3P code doesn't support gzip (it's expecting a local unix socket).
# So we create a GzipFile object and let the 3P code write into that.
# 2) The .seek(0) call at the end of the 3P code causes GzipFile to throw an
# exception. So we use GzipFileIgnoreSeek as a workaround.
with _GzipFileIgnoreSeek(mode='wb', fileobj=f) as gz:
docker.utils.tar(source_dir, exclude, fileobj=gz)
f.close()
cloud_storage.CopyFileToGCS(bucket, f.name, obj, storage_client)
finally:
try:
files.RmTree(temp_dir)
except OSError:
log.warn(
'Could not remove temporary directory [{0}]'.format(temp_dir))
def _GetSslCertificatesList(self, args):
holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
if args.ssl_certificate:
log.warn(
'The --ssl-certificate flag is deprecated and will be removed soon. '
'Use equivalent --ssl-certificates %s flag.',
args.ssl_certificate)
return [
self.SSL_CERTIFICATE_ARG.ResolveAsResource(
args, holder.resources)
]
if args.ssl_certificates:
return self.SSL_CERTIFICATES_ARG.ResolveAsResource(
args, holder.resources)
return []
def _AddFlagToSummary(command, summary, length_per_snippet, location, terms):
"""Adds flag summary, given location such as ['flags']['--myflag']."""
flags = command.get(location[0], {})
lines = []
line = ''
if _FormatHeader(lookup.FLAGS) not in summary:
lines.append(_FormatHeader(lookup.FLAGS))
# Add specific flag if given.
if len(location) > 1:
# Add flag name and description of flag if not added yet.
if _FormatItem(location[1]) not in summary:
lines.append(_FormatItem(location[1]))
desc_line = flags.get(location[1], {}).get(lookup.DESCRIPTION, '')
desc_line = _Snip(desc_line, length_per_snippet, terms)
if desc_line:
line = desc_line
else:
log.warn('Attempted to look up a location [{}] that was not '
'found.'.format(location[1]))
# Add subsections of flag if given.
if len(location) > 2:
if location[2] == lookup.CHOICES:
choices = flags.get(location[1], {}).get(lookup.CHOICES, [])
if choices:
lines.append(line)
line = 'Choices: {}.'.format(', '.join(sorted(choices)))
elif location[2] == lookup.DEFAULT:
default = flags.get(location[1]).get(lookup.DEFAULT)
if default:
lines.append(line)
line = 'Default: {}.'.format(
flags.get(location[1]).get(lookup.DEFAULT))
else:
log.warn('Attempted to look up a location [{}] that was not '
'found.'.format(location[-1]))
# If no specific flag given, get list of all flags.
else:
line = ', '.join(sorted(command.get(location[0], {}).keys()))
line = _Snip(line, length_per_snippet, terms)
if line:
lines.append(line)
summary += lines
def GetPublicKey(self):
"""Generates an SSH key using ssh-key (if necessary) and returns it."""
public_ssh_key_file = self.ssh_key_file + '.pub'
if (not os.path.exists(self.ssh_key_file)
or not os.path.exists(public_ssh_key_file)):
log.warn('You do not have an SSH key for Google Compute Engine.')
log.warn('[%s] will be executed to generate a key.',
self.ssh_keygen_executable)
ssh_directory = os.path.dirname(public_ssh_key_file)
if not os.path.exists(ssh_directory):
if console_io.PromptContinue(
'This tool needs to create the directory [{0}] before being able '
'to generate SSH keys.'.format(ssh_directory)):
files.MakeDir(ssh_directory, 0700)
else:
raise exceptions.ToolException(
'SSH key generation aborted by user.')
keygen_args = [
self.ssh_keygen_executable,
'-t',
'rsa',
'-f',
self.ssh_key_file,
]
if properties.VALUES.core.disable_prompts.GetBool():
# If prompts are disabled, use the default of no passphrase
current_os = platforms.OperatingSystem.Current()
if current_os is platforms.OperatingSystem.WINDOWS:
_GenerateKeyNoPassphraseOnWindows(keygen_args)
else:
# Specify empty passphrase on command line
keygen_args.extend(['-P', ''])
_RunExecutable(keygen_args)
else:
# Prompts are enabled. Run normally.
_RunExecutable(keygen_args)
with open(public_ssh_key_file) as f:
# We get back a unicode list of keys for the remaining metadata, so
# convert to unicode. Assume UTF 8, but if we miss a character we can just
# replace it with a '?'. The only source of issues would be the hostnames,
# which are relatively inconsequential.
return f.readline().strip().decode('utf8', 'replace')
def Filter(self, context, args):
project = properties.VALUES.core.project
resolver = resolvers.FromProperty(project)
resources.SetParamDefault('dns', None, 'project', resolver)
context['dns_client'] = apis.GetClientInstance('dns',
'v1',
http=self.Http())
context['dns_messages'] = apis.GetMessagesModule('dns', 'v1')
context['dns_resources'] = resources
if args.endpoint:
log.warn('The --endpoint flag is deprecated and will be removed. '
'Set the api_endpoint_overrides/dns property instead. '
'e.g. gcloud config set api_endpoint_overrides/dns '
'https://www.googleapis.com/dns/v1beta1')
return context
def ValidateDockerArgs(args):
"""Validates Docker-related args."""
if not args.container_manifest and not args.docker_image:
raise exceptions.RequiredArgumentException(
'--docker-image', 'You must provide Docker image')
if args.container_manifest:
log.warn('--container-manifest flag is deprecated and will be removed. '
'Use --docker-image flag instead.')
if args.run_command:
raise exceptions.InvalidArgumentException(
'--run-command', 'argument --run-command: not allowed with argument '
'--container-manifest')
if args.port_mappings:
raise exceptions.InvalidArgumentException(
'--port-mappings', 'argument --port-mappings: not allowed with '
'argument --container-manifest')
def Update(cli):
"""Create or overwrite help search table.
Args:
cli: Calliope CLI object for generating the help search table.
"""
help_text = HelpIndexGenerator(cli).Walk(hidden=False)
def SerializeCommand(command):
return resource_projector.Compile().Evaluate(command)
help_text = SerializeCommand(help_text)
# Overwrite the help search table file with updated content
table_path = IndexPath()
with open(table_path, 'w') as index_file:
json.dump(help_text, index_file, sort_keys=True)
index_file.write('\n')
log.warn('updated help index at {0}'.format(table_path))
def Run(self, args):
log.warn(
'The download command is deprecated, and will soon be removed.')
if args.output_dir:
output_dir = args.output_dir
if not os.path.isdir(output_dir):
raise exceptions.InvalidArgumentException(
'output-dir', 'The directory does not exist.')
else:
output_dir = os.getcwd()
client = appengine_client.AppengineClient(args.server)
for m in args.modules:
module_dir = os.path.join(output_dir, m)
files.MakeDir(module_dir)
log.status.Print('Downloading module [{module}] to [{dir}]'.format(
module=m, dir=module_dir))
client.DownloadModule(m, args.version, module_dir)
def Run(self, args):
log.warn('Please use instead [gcloud compute instance-groups '
'managed describe].')
client = self.context['autoscaler-client']
messages = self.context['autoscaler_messages_module']
resources = self.context['autoscaler_resources']
autoscaler_ref = resources.Parse(
args.name, collection='autoscaler.autoscalers')
request = messages.AutoscalerAutoscalersGetRequest()
request.project = autoscaler_ref.project
request.zone = autoscaler_ref.zone
request.autoscaler = autoscaler_ref.autoscaler
try:
return client.autoscalers.Get(request)
except apitools_base.exceptions.HttpError as error:
raise calliope_exceptions.HttpException(util.GetErrorMessage(error))
def Run(self, args):
log.warn('This command is deprecated. '
'Please use `gcloud preview app versions list` instead.')
api_client = appengine_api_client.GetApiClient()
services = api_client.ListServices()
versions = api_client.ListVersions(services)
service_versions = []
for version in versions:
if args.modules and version.service not in args.modules:
continue
service_versions.append({
'module': version.service,
'version': version.id,
'traffic_split': version.traffic_split
})
# Sort so the order is deterministic for testing.
return sorted(service_versions)
def Run(self, args):
log.warn('Please use instead [gcloud compute instance-groups '
'managed list].')
client = self.context['autoscaler-client']
messages = self.context['autoscaler_messages_module']
resources = self.context['autoscaler_resources']
try:
request = messages.AutoscalerAutoscalersListRequest()
request.project = properties.VALUES.core.project.Get(required=True)
request.zone = resources.Parse(args.zone,
collection='compute.zones').zone
return apitools_base.YieldFromList(client.autoscalers, request)
except exceptions.HttpError as error:
raise calliope_exceptions.HttpException(
util.GetErrorMessage(error))
except ValueError as error:
raise calliope_exceptions.HttpException(error)
def _CreateProject(project_id, project_ids):
"""Create a project and check that it isn't in the known project IDs."""
if project_ids and project_id in project_ids:
raise ValueError('Attempting to create a project that already exists.')
project_ref = resources.REGISTRY.Create('cloudresourcemanager.projects',
projectId=project_id)
try:
projects_api.Create(project_ref)
except Exception as err: # pylint: disable=broad-except
log.warn('Project creation failed: {err}\n'
'Please make sure to create the project [{project}] using\n'
' $ gcloud projects create {project}\n'
'or change to another project using\n'
' $ gcloud config set project <PROJECT ID>'.format(
err=str(err), project=project_id))
return None
return project_id
def Run(self, args):
holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
client = holder.client.apitools_client
messages = client.MESSAGES_MODULE
project = properties.VALUES.core.project.Get(required=True)
# TODO(b/33298284): remove names and regexp arguments.
filter_uris = []
filter_names = []
if args.names:
log.warn('Name argument for filtering list results is deprecated. '
'Please use --filter flag.')
if args.regexp:
log.warn('--regexp flag for filtering list results is deprecated. '
'Please use --filter flag.')
for name in args.names:
try:
ref = holder.resources.Parse(
name,
params={'project': project},
collection='compute.targetSslProxies')
filter_uris.append(ref.SelfLink())
except resources.UserError:
filter_names.append(name)
request = messages.ComputeTargetSslProxiesListRequest(
project=project,
filter='name eq {0}'.format(args.regexp) if args.regexp else None)
results = list_pager.YieldFromList(client.targetSslProxies,
request,
field='items',
limit=args.limit,
batch_size=None)
for item in results:
if not args.names:
yield item
elif item.selfLink in filter_uris or item.name in filter_names:
yield item
def FileIterator(base, skip_files, separator=os.path.sep):
"""Walks a directory tree, returning all the files. Follows symlinks.
Args:
base: The base path to search for files under.
skip_files: A regular expression object for files/directories to skip.
separator: Path separator used by the running system's platform.
Yields:
Paths of files found, relative to base.
"""
dirs = ['']
contains_skipped_modules = False
while dirs:
current_dir = dirs.pop()
entries = set(os.listdir(os.path.join(base, current_dir)))
for entry in sorted(entries):
name = os.path.join(current_dir, entry)
fullname = os.path.join(base, name)
# On Windows, os.path.join uses the path separator '\' instead of '/'.
# However, the skip_files regular expression always uses '/'.
# To handle this, we'll replace '\' characters with '/' characters.
if separator == '\\':
name = name.replace('\\', '/')
if os.path.isfile(fullname):
if skip_files.match(name):
log.info('Ignoring file [%s]: File matches ignore regex.', name)
contains_skipped_modules = True
else:
yield name
elif os.path.isdir(fullname):
if skip_files.match(name):
log.info('Ignoring directory [%s]: Directory matches ignore regex.',
name)
contains_skipped_modules = True
else:
dirs.append(name)
if contains_skipped_modules:
log.warn('Some files were skipped. Check the gcloud log file or pass '
'`--verbosity=info` to see which ones.')