def testZipDirSingleEmptyDir(self):
with files.TemporaryDirectory() as src_dir:
os.makedirs(os.path.join(src_dir, 'empty_dir'))
name_list = self._MakeZip(src_dir)
self.assertEqual([('empty_dir/', b'')], name_list)
def testZipDirSingleFile(self):
with files.TemporaryDirectory() as src_dir:
with open(os.path.join(src_dir, 'sample.txt'), 'a'):
pass
name_list = self._MakeZip(src_dir)
self.assertEqual([('sample.txt', b'')], name_list)
def _Update(self, restrict):
"""Update() helper method. Returns the number of changed help text files."""
with file_utils.TemporaryDirectory() as temp_dir:
pb = console_io.ProgressBar(label='Generating Help Docs')
walker = walker_util.HelpTextGenerator(self._cli,
temp_dir,
pb.SetProgress,
restrict=restrict)
pb.Start()
walker.Walk(hidden=True)
pb.Finish()
diff = HelpTextAccumulator(restrict=restrict)
DirDiff(self._help_dir, temp_dir, diff)
if diff.invalid_file_count:
# Bail out early on invalid content errors. These must be corrected
# before proceeding.
raise HelpTextUpdateError(
'{0} help text {1} with invalid content must be fixed.'.
format(diff.invalid_file_count,
text.Pluralize(diff.invalid_file_count, 'file')))
ops = {}
for op in ['add', 'delete', 'edit']:
ops[op] = []
changes = 0
for op, path in sorted(diff.GetChanges()):
changes += 1
if not self._test or changes < TEST_CHANGES_DISPLAY_MAX:
log.status.Print('{0} {1}'.format(op, path))
ops[op].append(path)
if self._test:
if changes:
if changes >= TEST_CHANGES_DISPLAY_MAX:
log.status.Print('...')
log.status.Print('{0} help test {1} changed'.format(
changes, text.Pluralize(changes, 'file')))
return changes
op = 'add'
if ops[op]:
for path in ops[op]:
dest_path = os.path.join(self._help_dir, path)
subdir = os.path.dirname(dest_path)
if subdir:
file_utils.MakeDir(subdir)
temp_path = os.path.join(temp_dir, path)
shutil.copyfile(temp_path, dest_path)
op = 'edit'
if ops[op]:
for path in ops[op]:
dest_path = os.path.join(self._help_dir, path)
temp_path = os.path.join(temp_dir, path)
shutil.copyfile(temp_path, dest_path)
op = 'delete'
if ops[op]:
for path in ops[op]:
dest_path = os.path.join(self._help_dir, path)
try:
os.remove(dest_path)
except OSError:
pass
return changes
def __call__(self, parser, namespace, values, option_string=None):
alias_args = ['gcloud']
# subprocess.call() below handles 'shell not found' diagnostics
if values:
shell = values
else:
shell = os.environ.get('SHELL')
if not shell:
# search for a default for SHELL, biased from left to right
for shell in ['bash', 'ksh', 'sh', 'zsh', 'dash']:
path = file_utils.FindExecutableOnPath(shell)
if path:
shell = path
break
else:
shell = 'sh'
for arg in cli.argv:
if arg == '--shell' or arg.startswith('--shell='):
# Only things up to, and not including, the first --shell.
# TODO(user): This search can have false positives. eg,
# $ gcloud --project --shell auth --shell
# If someone somehow had a project "--shell", or if some other flag
# flag value was legitimately "--shell". For now, we'll let this be
# a problematic, but rare, corner case.
break
# TODO(user): Make this quoting more robust.
if ' ' in arg:
arg = '"{arg}"'.format(arg=arg)
alias_args.append(arg)
alias_prefix = ' '.join(alias_args)
prompt = ' '.join(['gcloud'] + alias_args[1:])
interactive = sys.stdin.isatty()
buf = StringIO.StringIO()
GenerateRcFile(alias_prefix, prompt, subcommands, interactive, buf)
exit_code = 0
with file_utils.TemporaryDirectory() as tmpdir:
# link or symlink not available on all targets so we make N copies.
envfile = '.gcloudenv'
for rcfile in ['.bashrc', '.zshrc', envfile]:
path = os.path.join(tmpdir, rcfile)
with open(path, 'w') as f:
f.write(buf.getvalue())
f.flush()
try:
restore = ''
for name in ['HOME', 'ZDOTDIR', 'ENV']:
val = os.environ.get(name)
if val is not None:
restore += ' ' + name + "='" + val + "'"
if restore is not '':
restore = 'export' + restore
env = dict(os.environ)
env['_GCLOUD_RESTORE_'] = restore
env['HOME'] = tmpdir
env['ZDOTDIR'] = tmpdir
if os.sep == '\\':
# Workaround for UWIN ksh(1) PATH lookup on pure windows paths.
# The embeded '/' means "this is literal, don't do PATH lookup".
# Also handles the eval of $ENV that eliminates \'s.
env['ENV'] = '/'.join([tmpdir,
envfile]).replace('\\', '\\\\')
else:
env['ENV'] = path
# Why print terminal escape sequences if stdout is not a terminal?
if not sys.stdout.isatty():
env['TERM'] = 'dumb'
cmd = [shell, '-i']
# not interactive implies batch mode with commands on stdin. Since zsh
# insists on reading from /dev/tty we stuff it in a new sesssion which
# detaches /dev/tty and forces it to read from stdin. bash and dash
# complain about no tty in -i mode, so zsh is special-cased.
if not interactive and os.path.basename(shell).startswith(
'zsh'):
# eventually change preexec_fn=os.setsid to start_new_session=True
exit_code = subprocess.call(cmd,
env=env,
preexec_fn=os.setsid)
else:
exit_code = subprocess.call(cmd, env=env)
except OSError as e:
log.error("""\
could not run the shell [{shell}] -- \
make sure it is installed and on the system PATH [{e}]\
""".format(e=e, shell=shell))
exit_code = 1
sys.exit(exit_code)
def RunDeploy(
args,
api_client,
use_beta_stager=False,
runtime_builder_strategy=runtime_builders.RuntimeBuilderStrategy.NEVER,
parallel_build=True,
flex_image_build_option=FlexImageBuildOptions.ON_CLIENT,
disable_build_cache=False):
"""Perform a deployment based on the given args.
Args:
args: argparse.Namespace, An object that contains the values for the
arguments specified in the ArgsDeploy() function.
api_client: api_lib.app.appengine_api_client.AppengineClient, App Engine
Admin API client.
use_beta_stager: Use the stager registry defined for the beta track rather
than the default stager registry.
runtime_builder_strategy: runtime_builders.RuntimeBuilderStrategy, when to
use the new CloudBuild-based runtime builders (alternative is old
externalized runtimes).
parallel_build: bool, whether to use parallel build and deployment path.
Only supported in v1beta and v1alpha App Engine Admin API.
flex_image_build_option: FlexImageBuildOptions, whether a flex deployment
should upload files so that the server can build the image or build the
image on client.
disable_build_cache: bool, disable the build cache.
Returns:
A dict on the form `{'versions': new_versions, 'configs': updated_configs}`
where new_versions is a list of version_util.Version, and updated_configs
is a list of config file identifiers, see yaml_parsing.ConfigYamlInfo.
"""
project = properties.VALUES.core.project.Get(required=True)
deploy_options = DeployOptions.FromProperties(
runtime_builder_strategy=runtime_builder_strategy,
parallel_build=parallel_build,
flex_image_build_option=flex_image_build_option)
with files.TemporaryDirectory() as staging_area:
stager = _MakeStager(args.skip_staging, use_beta_stager,
args.staging_command, staging_area)
services, configs = deployables.GetDeployables(
args.deployables, stager, deployables.GetPathMatchers())
service_infos = [d.service_info for d in services]
flags.ValidateImageUrl(args.image_url, service_infos)
# pylint: disable=protected-access
log.debug('API endpoint: [{endpoint}], API version: [{version}]'.format(
endpoint=api_client.client.url,
version=api_client.client._VERSION))
# The legacy admin console API client.
# The Admin Console API existed long before the App Engine Admin API, and
# isn't being improved. We're in the process of migrating all of the calls
# over to the Admin API, but a few things (notably config deployments)
# haven't been ported over yet.
# Import only when necessary, to decrease startup time.
# pylint: disable=g-import-not-at-top
from googlecloudsdk.api_lib.app import appengine_client
# pylint: enable=g-import-not-at-top
ac_client = appengine_client.AppengineClient(
args.server, args.ignore_bad_certs)
app = _PossiblyCreateApp(api_client, project)
_RaiseIfStopped(api_client, app)
app = _PossiblyRepairApp(api_client, app)
# Tell the user what is going to happen, and ask them to confirm.
version_id = args.version or util.GenerateVersionId()
deployed_urls = output_helpers.DisplayProposedDeployment(
app, project, services, configs, version_id, deploy_options.promote)
console_io.PromptContinue(cancel_on_no=True)
if service_infos:
# Do generic app setup if deploying any services.
# All deployment paths for a service involve uploading source to GCS.
metrics.CustomTimedEvent(metric_names.GET_CODE_BUCKET_START)
code_bucket_ref = args.bucket or flags.GetCodeBucket(app, project)
metrics.CustomTimedEvent(metric_names.GET_CODE_BUCKET)
log.debug('Using bucket [{b}].'.format(b=code_bucket_ref.ToBucketUrl()))
# Prepare Flex if any service is going to deploy an image.
if any([s.RequiresImage() for s in service_infos]):
if deploy_options.use_service_management:
deploy_command_util.PossiblyEnableFlex(project)
else:
deploy_command_util.DoPrepareManagedVms(ac_client)
all_services = dict([(s.id, s) for s in api_client.ListServices()])
else:
code_bucket_ref = None
all_services = {}
new_versions = []
deployer = ServiceDeployer(api_client, deploy_options)
# Track whether a service has been deployed yet, for metrics.
service_deployed = False
for service in services:
if not service_deployed:
metrics.CustomTimedEvent(metric_names.FIRST_SERVICE_DEPLOY_START)
new_version = version_util.Version(project, service.service_id,
version_id)
deployer.Deploy(
service,
new_version,
code_bucket_ref,
args.image_url,
all_services,
app.gcrDomain,
disable_build_cache=disable_build_cache,
flex_image_build_option=flex_image_build_option)
new_versions.append(new_version)
log.status.Print('Deployed service [{0}] to [{1}]'.format(
service.service_id, deployed_urls[service.service_id]))
if not service_deployed:
metrics.CustomTimedEvent(metric_names.FIRST_SERVICE_DEPLOY)
service_deployed = True
# Deploy config files.
if configs:
metrics.CustomTimedEvent(metric_names.UPDATE_CONFIG_START)
for config in configs:
message = 'Updating config [{config}]'.format(config=config.name)
with progress_tracker.ProgressTracker(message):
ac_client.UpdateConfig(config.name, config.parsed)
metrics.CustomTimedEvent(metric_names.UPDATE_CONFIG)
updated_configs = [c.name for c in configs]
PrintPostDeployHints(new_versions, updated_configs)
# Return all the things that were deployed.
return {
'versions': new_versions,
'configs': updated_configs
}
def SetUp(self):
self.SelectApi(self.api_version)
self.dir = file_utils.TemporaryDirectory()
self._createTestInitialStateFiles()
def testGenerateHelpDocsNoRestrictions(self):
with files.TemporaryDirectory() as devsite_dir:
self.Run(
['meta', 'generate-help-docs', '--devsite-dir=' + devsite_dir])
self.walk.assert_called_once_with(False, [])
def ForcePushFilesToBranch(self,
branch,
base_path,
paths,
dry_run=False,
full_path=False):
"""Force pushes a set of files to a branch on the remote repository.
This is mainly to be used with source captures, where the user wants to
upload files associated with a deployment to view later.
Args:
branch: str, The name of the branch to push to.
base_path: str, The base path to use for the files.
paths: list of str, The paths for the files to upload.
Their paths in the repository will be relative to base_path.
For example, if base_path is '/a/b/c', the path '/a/b/c/d/file1' will
appear as 'd/file1' in the repository.
dry_run: bool, If true do not run but print commands instead.
full_path: bool, If true use the full path to gcloud.
Raises:
CannotPushToRepositoryException: If the operation fails in any way.
"""
# Git treats relative paths strangely with the --work-tree flag.
# git add --work-tree=a a/b will try to look for a/a/b.
# To avoid the lookup, always convert to absolute paths.
paths = [os.path.abspath(p) for p in paths]
# Check for git files and don't allow upload if they are included.
for path in paths:
for segment in path.split(os.sep):
if segment in ('.git', '.gitignore'):
message = (
"Can't upload the file tree. "
'Uploading a directory containing a git repository as a '
'subdirectory is not supported. '
'Please either upload from the top level git repository '
'or any of its subdirectories. '
'Unsupported git file detected: %s' % path)
raise CannotPushToRepositoryException(message)
with files.TemporaryDirectory() as temp_dir:
def RunGitCommand(*args):
git_dir = '--git-dir=' + os.path.join(temp_dir, '.git')
work_tree = '--work-tree=' + base_path
self._RunCommand(['git', git_dir, work_tree] + list(args),
dry_run)
# Init empty repository in a temporary location
self._RunCommand(['git', 'init', temp_dir], dry_run)
# Create new branch and add files
RunGitCommand('checkout', '-b', branch)
args_len = 100
for i in range(0, len(paths), args_len):
RunGitCommand('add', *paths[i:i + args_len])
RunGitCommand('commit', '-m',
'source capture uploaded from gcloud')
# Add remote and force push
cred_helper_command = _GetCredHelperCommand(self._uri,
full_path=full_path)
if cred_helper_command:
RunGitCommand('config', 'credential.helper',
cred_helper_command)
try:
RunGitCommand('remote', 'add', 'origin', self._uri)
RunGitCommand('push', '-f', 'origin', branch)
except subprocess.CalledProcessError as e:
raise CannotPushToRepositoryException(e)
def testFailureConstructUpdateMaskFromPolicy(self):
policy_file_path = self.Touch(files.TemporaryDirectory().path,
'bad',
contents='{foo} bad {{foo}}')
with self.assertRaises(exceptions.Error):
iam_util.ConstructUpdateMaskFromPolicy(policy_file_path)
def Run(self, args):
log.status.Print('Validating input arguments.')
project_id = properties.VALUES.core.project.GetOrFail()
# Validate the args value before generate the RBAC policy file.
rbac_util.ValidateArgs(args)
# Revoke RBAC policy for specified user from cluster.
if args.revoke:
sys.stdout.write(
'Revoking the RBAC policy from cluster with kubeconfig: {}, context: {}\n'
.format(args.kubeconfig, args.context))
with kube_util.KubernetesClient(
kubeconfig=getattr(args, 'kubeconfig', None),
context=getattr(args, 'context', None),
) as kube_client:
# Check Admin permissions.
kube_client.CheckClusterAdminPermissions()
users_list = list()
if args.users:
users_list = args.users.split(',')
elif args.anthos_support:
users_list.append(
rbac_util.GetAnthosSupportUser(project_id))
for user in users_list:
message = ('The RBAC policy for user: {} will be clean up.'
.format(user))
console_io.PromptContinue(message=message,
cancel_on_no=True)
log.status.Print(
'--------------------------------------------')
log.status.Print(
'Start cleaning up RBAC policy for: {}'.format(user))
if kube_client.CleanUpRbacPolicy(args.membership,
args.role, project_id,
user,
args.anthos_support):
log.status.Print(
'Finish clean up the previous RBAC policy for: {}'.
format(user))
return
# Generate the RBAC policy file from args.
generated_rbac = rbac_util.GenerateRBAC(args, project_id)
if args.rbac_output_file:
sys.stdout.write(
'Generated RBAC policy is written to file: {} \n'.format(
args.rbac_output_file))
else:
sys.stdout.write('Generated RBAC policy is: \n')
sys.stdout.write('--------------------------------------------\n')
# Write the generated RBAC policy file to the file provided with
# "--rbac-output-file" specified or print on the screen.
final_rbac_policy = ''
for user in sorted(generated_rbac.keys()):
final_rbac_policy += generated_rbac.get(user)
log.WriteToFileOrStdout(
args.rbac_output_file if args.rbac_output_file else '-',
final_rbac_policy,
overwrite=True,
binary=False,
private=True)
# Apply generated RBAC policy to cluster.
if args.apply:
sys.stdout.write(
'Applying the generate RBAC policy to cluster with kubeconfig: {}, context: {}\n'
.format(args.kubeconfig, args.context))
with kube_util.KubernetesClient(
kubeconfig=getattr(args, 'kubeconfig', None),
context=getattr(args, 'context', None),
) as kube_client:
# Check Admin permissions.
kube_client.CheckClusterAdminPermissions()
for user in generated_rbac.keys():
with file_utils.TemporaryDirectory() as tmp_dir:
file = tmp_dir + '/rbac.yaml'
current_rbac_policy = generated_rbac.get(user)
file_utils.WriteFileContents(file, current_rbac_policy)
# Check whether there are existing RBAC policy for this user, if not,
# will directly apply the new RBAC policy.
if not kube_client.GetRbacPolicy(
args.membership, args.role, project_id, user,
args.anthos_support):
# Check whether there are role confliction, which required clean up.
need_clean_up = False
# Override when proposed RBAC policy has diff with existing one.
override_check = False
# Checking RBAC policy diff, return None, None if there are no diff.
diff, err = kube_client.GetRbacPolicyDiff(file)
if diff is not None:
override_check = True
log.status.Print(
'The new RBAC policy has diff with previous: \n {}'
.format(diff))
if err is not None:
# 'Invalid value' means the clusterrole/role permission has been
# changed. This need to clean up old RBAC policy and then apply
# the new one.
if 'Invalid value' in err:
rbac_policy_name = kube_client.RbacPolicyName(
'permission', project_id,
args.membership, user)
rbac_permission_policy = kube_client.GetRbacPermissionPolicy(
rbac_policy_name, args.role)
log.status.Print(
'The existing RBAC policy has conflict with proposed one:\n{}'
.format(rbac_permission_policy))
need_clean_up = True
override_check = True
else:
raise exceptions.Error(
'Error when get diff for RBAC policy files for user: {}, with error: {}'
.format(user, err))
if override_check:
message = ('The RBAC file will be overridden.')
console_io.PromptContinue(message=message,
cancel_on_no=True)
if need_clean_up:
log.status.Print(
'--------------------------------------------'
)
log.status.Print(
'Start cleaning up previous RBAC policy for: {}'
.format(user))
if kube_client.CleanUpRbacPolicy(
args.membership, args.role, project_id,
user, args.anthos_support):
log.status.Print(
'Finish clean up the previous RBAC policy for: {}'
.format(user))
try:
log.status.Print(
'Writing RBAC policy for user: {} to cluster.'.
format(user))
kube_client.ApplyRbacPolicy(file)
except Exception as e:
log.status.Print(
'Error in applying the RBAC policy to cluster: {}'
.format(e))
raise
log.status.Print(
'Successfully applied the RBAC policy to cluster.')
def SetUp(self):
self.dir = files.TemporaryDirectory()
def CopyFilesToCodeBucket(service, source_dir, bucket_ref,
upload_strategy):
"""Copies application files to the Google Cloud Storage code bucket.
Uses either gsutil, the Cloud Storage API using processes, or the Cloud
Storage API using threads.
Consider the following original structure:
app/
main.py
tools/
foo.py
Assume main.py has SHA1 hash 123 and foo.py has SHA1 hash 456. The resultant
GCS bucket will look like this:
gs://$BUCKET/
123
456
The resulting App Engine API manifest will be:
{
"app/main.py": {
"sourceUrl": "https://storage.googleapis.com/staging-bucket/123",
"sha1Sum": "123"
},
"app/tools/foo.py": {
"sourceUrl": "https://storage.googleapis.com/staging-bucket/456",
"sha1Sum": "456"
}
}
A 'list' call of the bucket is made at the start, and files that hash to
values already present in the bucket will not be uploaded again.
Args:
service: ServiceYamlInfo, The service being deployed.
source_dir: str, path to the service's source directory
bucket_ref: The reference to the bucket files will be placed in.
upload_strategy: The UploadStrategy to use
Returns:
A dictionary representing the manifest.
Raises:
ValueError: if an invalid upload strategy or None is given
"""
if upload_strategy is UploadStrategy.GSUTIL:
manifest = CopyFilesToCodeBucketGsutil(service, source_dir, bucket_ref)
metrics.CustomTimedEvent(metric_names.COPY_APP_FILES)
else:
# Collect a list of files to upload, indexed by the SHA so uploads are
# deduplicated.
with file_utils.TemporaryDirectory() as tmp_dir:
manifest = _BuildDeploymentManifest(service, source_dir, bucket_ref,
tmp_dir)
files_to_upload = _BuildFileUploadMap(
manifest, source_dir, bucket_ref, tmp_dir)
if upload_strategy is UploadStrategy.THREADS:
_UploadFilesThreads(files_to_upload, bucket_ref)
elif upload_strategy is UploadStrategy.PROCESSES:
_UploadFilesProcesses(files_to_upload, bucket_ref)
else:
raise ValueError('Invalid upload strategy ' + str(upload_strategy))
log.status.Print('File upload done.')
log.info('Manifest: [{0}]'.format(manifest))
metrics.CustomTimedEvent(metric_names.COPY_APP_FILES)
return manifest
def LoadOrGenerate(command, directories=None, tarball=None, force=False,
generate=True, ignore_out_of_date=False, verbose=False,
warn_on_exceptions=False):
"""Returns the CLI tree for command, generating it if it does not exist.
Args:
command: The CLI root command. This has the form:
[executable] [tarball_path/]command_name
where [executable] and [tarball_path/] are used only for packaging CLI
trees and may be empty. Examples:
* gcloud
* kubectl/kubectl
* python gsutil/gsutil
directories: The list of directories containing the CLI tree JSON files.
If None then the default installation directories are used.
tarball: For packaging CLI trees. --commands specifies one command that is
a relative path in this tarball. The tarball is extracted to a temporary
directory and the command path is adjusted to point to the temporary
directory.
force: Update all exitsing trees by forcing them to be out of date if True.
generate: Generate the tree if it is out of date or does not exist.
ignore_out_of_date: Ignore out of date trees instead of regenerating.
verbose: Display a status line for up to date CLI trees if True.
warn_on_exceptions: Emits warning messages in lieu of generator exceptions.
Used during installation.
Returns:
The CLI tree for command or None if command not found or there is no
generator.
"""
parts = shlex.split(command)
command_executable_args, command_relative_path = parts[:-1], parts[-1]
_, command_name = os.path.split(command_relative_path)
# Handle package time command names.
if command_name.endswith('_lite'):
command_name = command_name[:-5]
# Don't repeat failed attempts.
if CliTreeGenerator.AlreadyFailed(command_name):
if verbose:
log.status.Print(
'Skipping CLI tree generation for [{}] due to previous '
'failure.'.format(command_name))
return None
def _LoadOrGenerate(command_path, command_name):
"""Helper."""
# Instantiate the appropriate generator.
if command_name in GENERATORS:
command_args = command_executable_args + [command_path]
try:
generator = GENERATORS[command_name](
command_name, root_command_args=command_args)
except CommandInvocationError as e:
if verbose:
log.status.Print('Command [{}] could not be invoked:\n{}'.format(
command, e))
return None
else:
generator = ManPageCliTreeGenerator(command_name)
# Load or (re)generate the CLI tree if possible.
try:
return generator.LoadOrGenerate(
directories=directories,
force=force,
generate=generate,
ignore_out_of_date=ignore_out_of_date,
verbose=verbose,
warn_on_exceptions=warn_on_exceptions)
except NoManPageTextForCommandError:
pass
return None
if command_name == cli_tree.DEFAULT_CLI_NAME:
tree = cli_tree.Load()
elif tarball:
with files.TemporaryDirectory() as tmp:
tar = tarfile.open(tarball)
tar.extractall(tmp)
tree = _LoadOrGenerate(
os.path.join(tmp, command_relative_path),
command_name)
else:
tree = _LoadOrGenerate(command_relative_path, command_name)
if not tree:
CliTreeGenerator.AddFailure(command_name)
return tree
def __init__(self, flags):
"""Constructor for KubernetesClient.
Args:
flags: the flags passed to the enclosing command.
Raises:
exceptions.Error: if the client cannot be configured
calliope_exceptions.MinimumArgumentException: if a kubeconfig file
cannot be deduced from the command line flags or environment
"""
self.kubectl_timeout = '20s'
self.temp_kubeconfig_dir = None
# If the cluster to be registered is a GKE cluster, create a temporary
# directory to store the kubeconfig that will be generated using the
# GKE GetCluster() API
if flags and (flags.gke_uri or flags.gke_cluster):
self.temp_kubeconfig_dir = files.TemporaryDirectory()
self.processor = KubeconfigProcessor()
self.kubeconfig, self.context = self.processor.GetKubeconfigAndContext(
flags, self.temp_kubeconfig_dir)
# If --public-issuer-url is set or if gke_cluster_uri is set, we must not
# attempt to construct the cluster client, because it may use a kubeconfig
# that we don't fully support yet. See: b/152465794.
# TODO(b/149872627): Switch to official client to fully support kubeconfig.
if (hasattr(flags, 'public_issuer_url') and flags.public_issuer_url
) or (hasattr(flags, 'enable_workload_identity')
and self.processor.gke_cluster_uri):
return
# Due to an issue between the gcloud third_party yaml library, which the
# official K8s client depends on, and python3 (b/149872627), we construct
# our own client below. Since this is not as robust a solution as using the
# official client, and this client is currently only used for Workload
# Identity related calls, we also gate it on --enable-workload-identity
# (for the register command) and --manage-workload-identity-bucket
# (for the unregister command).
if (flags and ((hasattr(flags, 'enable_workload_identity')
and flags.enable_workload_identity) or
(hasattr(flags, 'manage_workload_identity_bucket')
and flags.manage_workload_identity_bucket))):
# If processor.GetKubeconfigAndContext returns `None` for the kubeconfig
# path, that indicates we should be using in-cluster config. Otherwise,
# the first return value is the path to the kubeconfig file. Since the
# client we are using for Workload Identity related calls does not support
# in-cluster config yet, this will raise an exception if
# --enable-workload-identity is set in an environment that requires in-
# cluster config.
if self.kubeconfig is not None:
client_config = self.processor.GetClientConfig(
self.kubeconfig, self.context)
self.apiserver = client_config['server']
ca_file = None
cert_file = None
key_file = None
if client_config['insecure']:
cert_reqs = ssl.CERT_NONE
else:
cert_reqs = ssl.CERT_REQUIRED
# Cert info needs to be written to a file so PoolManager can read it.
ca_file = _WriteTempFile(
base64.standard_b64decode(
client_config['cluster_ca_cert']))
cert_file = _WriteTempFile(
base64.standard_b64decode(
client_config['client_cert']))
key_file = _WriteTempFile(
base64.standard_b64decode(client_config['client_key']))
# TODO(b/149872627): If for some reason we don't switch to the official
# client before beta, figure out whether we need to support proxies.
# The official client has Configuration.proxy, which can be a proxy
# URL and causes it to use urllib3.ProxyManager instead of PoolManager.
# Since the official client's kubeconfig loader doesn't set
# Configuration.proxy, and the default is None, it's possible that
# it's not used unless configured in-process. This doesn't seem to be
# an option that can be set via a kubeconfig file, though there is an
# unresolved Open Source issue that was created several years ago
# to allow proxy configuration in kubeconfig:
# https://github.com/kubernetes/client-go/issues/351.
self.cluster_pool_manager = urllib3.PoolManager(
num_pools=4, # Official client's default.
maxsize=4, # Official client's default.
cert_reqs=cert_reqs,
ca_certs=ca_file,
cert_file=cert_file,
key_file=key_file,
**{})
else:
raise exceptions.Error(
'Workload Identity feature does not support '
'constructing a client from in-cluster config')
def LoadOrGenerate(command, directories=None, tarball=None, force=False,
generate=True, ignore_out_of_date=False, verbose=False,
warn_on_exceptions=False):
"""Returns the CLI tree for command, generating it if it does not exist.
Args:
command: The CLI root command name.
directories: The list of directories containing the CLI tree JSON files.
If None then the default installation directories are used.
tarball: For packaging CLI trees. --commands specifies one command that is
a relative path in this tarball. The tarball is extracted to a temporary
directory and the command path is adjusted to point to the temporary
directory.
force: Update all exitsing trees by forcing them to be out of date if True.
generate: Generate the tree if it is out of date or does not exist.
ignore_out_of_date: Ignore out of date trees instead of regenerating.
verbose: Display a status line for up to date CLI trees if True.
warn_on_exceptions: Emits warning messages in lieu of generator exceptions.
Used during installation.
Returns:
The CLI tree for command or None if command not found or there is no
generator.
"""
command_dir, command_name = os.path.split(command)
# Handle package time command names.
if command_name.endswith('_lite'):
command_name = command_name[:-5]
# Don't repeat failed attempts.
if CliTreeGenerator.AlreadyFailed(command_name):
if verbose:
log.status.Print('No CLI tree generator for [{}].'.format(command))
return None
def _LoadOrGenerate(command, command_dir, command_name):
"""Helper."""
# The command must exist.
if (command_dir and not os.path.exists(command) or
not command_dir and not files.FindExecutableOnPath(command_name)):
if verbose:
log.status.Print('Command [{}] not found.'.format(command))
return None
# Instantiate the appropriate generator.
try:
generator = GENERATORS[command_name](
command, command_name=command_name, tarball=tarball)
except KeyError:
generator = ManPageCliTreeGenerator(command_name)
if not generator:
return None
# Load or (re)generate the CLI tree if possible.
try:
return generator.LoadOrGenerate(
directories=directories,
force=force,
generate=generate,
ignore_out_of_date=ignore_out_of_date,
verbose=verbose,
warn_on_exceptions=warn_on_exceptions)
except NoManPageTextForCommand:
pass
return None
if command_name == cli_tree.DEFAULT_CLI_NAME:
tree = cli_tree.Load()
elif tarball:
with files.TemporaryDirectory() as tmp:
tar = tarfile.open(tarball)
tar.extractall(tmp)
command = os.path.join(tmp, command)
command_dir = os.path.join(tmp, command_dir)
tree = _LoadOrGenerate(command, command_dir, command_name)
else:
tree = _LoadOrGenerate(command, command_dir, command_name)
if not tree:
CliTreeGenerator.AddFailure(command_name)
return tree
def __init__(self,
api_adapter=None,
gke_uri=None,
gke_cluster=None,
kubeconfig=None,
internal_ip=False,
cross_connect_subnetwork=None,
private_endpoint_fqdn=None,
context=None,
public_issuer_url=None,
enable_workload_identity=False):
"""Constructor for KubernetesClient.
Args:
api_adapter: the GKE api adapter used for running kubernetes commands
gke_uri: the URI of the GKE cluster; for example,
'https://container.googleapis.com/v1/projects/my-project/locations/us-central1-a/clusters/my-cluster'
gke_cluster: the "location/name" of the GKE cluster. The location can be a
zone or a region for e.g `us-central1-a/my-cluster`
kubeconfig: the kubeconfig path
internal_ip: whether to persist the internal IP of the endpoint.
cross_connect_subnetwork: full path of the cross connect subnet whose
endpoint to persist (optional)
private_endpoint_fqdn: whether to persist the private fqdn.
context: the context to use
public_issuer_url: the public issuer url
enable_workload_identity: whether to enable workload identity
Raises:
exceptions.Error: if the client cannot be configured
calliope_exceptions.MinimumArgumentException: if a kubeconfig file
cannot be deduced from the command line flags or environment
"""
self.kubectl_timeout = '20s'
self.temp_kubeconfig_dir = None
# If the cluster to be registered is a GKE cluster, create a temporary
# directory to store the kubeconfig that will be generated using the
# GKE GetCluster() API
if gke_uri or gke_cluster:
self.temp_kubeconfig_dir = files.TemporaryDirectory()
self.processor = KubeconfigProcessor(
api_adapter=api_adapter,
gke_uri=gke_uri,
gke_cluster=gke_cluster,
kubeconfig=kubeconfig,
internal_ip=internal_ip,
cross_connect_subnetwork=cross_connect_subnetwork,
private_endpoint_fqdn=private_endpoint_fqdn,
context=context)
self.kubeconfig, self.context = self.processor.GetKubeconfigAndContext(
self.temp_kubeconfig_dir)
# This previously fixed b/152465794. It is probably unnecessary now that
# we use the official K8s client, but it's also still true that we don't
# need a K8s client to talk to the cluster in this case. Consider removing
# this check later if we need the K8s client in other scenarios. For now,
# the impact of switching to the official client can be minimized to only
# scenarios where we actually need it.
if public_issuer_url or (enable_workload_identity
and self.processor.gke_cluster_uri):
return
if enable_workload_identity:
self.kube_client = self.processor.GetKubeClient(
self.kubeconfig, self.context)
def BuildPackages(package_path, output_dir):
"""Builds Python packages from the given package source.
That is, builds Python packages from the code in package_path, using its
parent directory (the 'package root') as its context using the setuptools
`sdist` command.
If there is a `setup.py` file in the package root, use that. Otherwise,
use a simple, temporary one made for this package.
We try to be as unobstrustive as possible (see _RunSetupTools for details):
- setuptools writes some files to the package root--we move as many temporary
generated files out of the package root as possible
- the final output gets written to output_dir
- any temporary setup.py file is written outside of the package root.
- if the current directory isn't writable, we silenly make a temporary copy
Args:
package_path: str. Path to the package. This should be the path to
the directory containing the Python code to be built, *not* its parent
(which optionally contains setup.py and other metadata).
output_dir: str, path to a long-lived directory in which the built packages
should be created.
Returns:
list of str. The full local path to all built Python packages.
Raises:
SetuptoolsFailedError: If the setup.py file fails to successfully build.
MissingInitError: If the package doesn't contain an `__init__.py` file.
"""
package_path = os.path.abspath(package_path)
with files.TemporaryDirectory() as temp_dir:
package_root = _CopyIfNotWritable(os.path.dirname(package_path),
temp_dir)
if not os.path.exists(os.path.join(package_path, '__init__.py')):
# We could drop `__init__.py` in here, but it's pretty likely that this
# indicates an incorrect directory or some bigger problem and we don't
# want to obscure that.
#
# Note that we could more strictly validate here by checking each package
# in the `--module-name` argument, but this should catch most issues.
raise MissingInitError(package_path)
setup_py_path = os.path.join(package_root, 'setup.py')
package_name = os.path.basename(package_path)
generated = _GenerateSetupPyIfNeeded(setup_py_path, package_name)
try:
return _RunSetupTools(package_root, setup_py_path, output_dir)
except RuntimeError as err:
raise SetuptoolsFailedError(str(err), generated)
finally:
if generated:
# For some reason, this artifact gets generated in the package root by
# setuptools, even after setting PYTHONDONTWRITEBYTECODE or running
# `python setup.py clean --all`. It's weird to leave someone a .pyc for
# a file they never knew they had, so we clean it up.
pyc_file = os.path.join(package_root, 'setup.pyc')
for path in (setup_py_path, pyc_file):
try:
os.unlink(path)
except OSError:
log.debug(
"Couldn't remove file [%s] (it may never have been created).",
pyc_file)
def testMavenJsonKeyInput(self):
with files.TemporaryDirectory() as tmp_dir:
key_file = os.path.join(tmp_dir, 'path/to/key.json')
cmd = [
'artifacts', 'print-settings', 'mvn', '--repository=my-repo',
'--location=us',
'--json-key=%s' % key_file
]
self.WriteKeyFile(key_file)
self.SetListLocationsExpect('us')
self.SetGetRepositoryExpect(
'us', 'my-repo', self.messages.Repository.FormatValueValuesEnum.MAVEN)
self.Run(cmd)
self.AssertOutputEquals(
"""\
<!-- Insert following snippet into your pom.xml -->
<project>
<distributionManagement>
<snapshotRepository>
<id>artifact-registry</id>
<url>https://us-maven.pkg.dev/fake-project/my-repo</url>
</snapshotRepository>
<repository>
<id>artifact-registry</id>
<url>https://us-maven.pkg.dev/fake-project/my-repo</url>
</repository>
</distributionManagement>
<repositories>
<repository>
<id>artifact-registry</id>
<url>https://us-maven.pkg.dev/fake-project/my-repo</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
</project>
<!-- Insert following snippet into your settings.xml -->
<settings>
<servers>
<server>
<id>artifact-registry</id>
<configuration>
<httpConfiguration>
<get>
<usePreemptive>true</usePreemptive>
</get>
<put>
<params>
<property>
<name>http.protocol.expect-continue</name>
<value>false</value>
</property>
</params>
</put>
</httpConfiguration>
</configuration>
<username>_json_key_base64</username>
<password>eyJhIjoiYiJ9</password>
</server>
</servers>
</settings>
""",
normalize_space=True)
def UploadPythonPackages(packages=(),
package_path=None,
staging_bucket=None,
job_name=None):
"""Uploads Python packages (if necessary), building them as-specified.
A Cloud ML job needs one or more Python packages to run. These Python packages
can be specified in one of three ways:
1. As a path to a local, pre-built Python package file.
2. As a path to a Cloud Storage-hosted, pre-built Python package file (paths
beginning with 'gs://').
3. As a local Python source tree (the `--package-path` flag).
In case 1, we upload the local files to Cloud Storage[1] and provide their
paths. These can then be given to the Cloud ML API, which can fetch these
files.
In case 2, we don't need to do anything. We can just send these paths directly
to the Cloud ML API.
In case 3, we perform a build using setuptools[2], and upload the resulting
artifacts to Cloud Storage[1]. The paths to these artifacts can be given to
the Cloud ML API. See the `BuildPackages` method.
These methods of specifying Python packages may be combined.
[1] Uploads are to a specially-prefixed location in a user-provided Cloud
Storage staging bucket. If the user provides bucket `gs://my-bucket/`, a file
`package.tar.gz` is uploaded to
`gs://my-bucket/<job name>/<checksum>/package.tar.gz`.
[2] setuptools must be installed on the local user system.
Args:
packages: list of str. Path to extra tar.gz packages to upload, if any. If
empty, a package_path must be provided.
package_path: str. Relative path to source directory to be built, if any. If
omitted, one or more packages must be provided.
staging_bucket: storage_util.BucketReference. Bucket to which archives are
uploaded. Not necessary if only remote packages are given.
job_name: str. Name of the Cloud ML Job. Used to prefix uploaded packages.
Not necessary if only remote packages are given.
Returns:
list of str. Fully qualified Cloud Storage URLs (`gs://..`) from uploaded
packages.
Raises:
ValueError: If packages is empty, and building package_path produces no
tar archives.
SetuptoolsFailedError: If the setup.py file fails to successfully build.
MissingInitError: If the package doesn't contain an `__init__.py` file.
DuplicateEntriesError: If multiple files with the same name were provided.
ArgumentError: if no packages were found in the given path.
"""
remote_paths = []
local_paths = []
for package in packages:
if storage_util.ObjectReference.IsStorageUrl(package):
remote_paths.append(package)
else:
local_paths.append(package)
if package_path:
with files.TemporaryDirectory() as temp_dir:
local_paths.extend(
BuildPackages(package_path, os.path.join(temp_dir, 'output')))
remote_paths.extend(
_UploadFilesByPath(local_paths, staging_bucket, job_name))
elif local_paths:
# Can't combine this with above because above requires the temporary
# directory to still be around
remote_paths.extend(
_UploadFilesByPath(local_paths, staging_bucket, job_name))
if not remote_paths:
raise flags.ArgumentError(_NO_PACKAGES_ERROR_MSG)
return remote_paths
def testZipDirEmpty(self):
with files.TemporaryDirectory() as src_dir:
name_list = self._MakeZip(src_dir)
self.assertEqual([], name_list)
def CopyFilesToCodeBucket(modules, bucket_ref, source_contexts,
unused_storage_client):
"""Examines modules and copies files to a Google Cloud Storage bucket.
Args:
modules: [(str, ModuleYamlInfo)] List of pairs of module name, and parsed
module information.
bucket_ref: str A reference to a GCS bucket where the files will be
uploaded.
source_contexts: [dict] List of json-serializable source contexts
associated with the modules.
unused_storage_client: Unused, there to satisfy the same interface as
CopyFilesToCodeBucketNoGsutil
Returns:
A lookup from module name to a dictionary representing the manifest. See
_BuildStagingDirectory.
"""
manifests = {}
with file_utils.TemporaryDirectory() as staging_directory:
for (module, info) in modules:
source_directory = os.path.dirname(info.file)
excluded_files_regex = info.parsed.skip_files.regex
manifest = _BuildStagingDirectory(source_directory,
staging_directory, bucket_ref,
excluded_files_regex,
source_contexts)
manifests[module] = manifest
if any(manifest for manifest in manifests.itervalues()):
log.status.Print('Copying files to Google Cloud Storage...')
log.status.Print(
'Synchronizing files to [{b}].'.format(b=bucket_ref))
try:
log.SetUserOutputEnabled(False)
def _StatusUpdate(result, unused_retry_state):
log.info('Error synchronizing files. Return code: {0}. '
'Retrying.'.format(result))
retryer = retry.Retryer(max_retrials=3,
status_update_func=_StatusUpdate)
def _ShouldRetry(return_code, unused_retry_state):
return return_code != 0
# gsutil expects a trailing /
dest_dir = bucket_ref.ToBucketUrl()
try:
retryer.RetryOnResult(cloud_storage.Rsync,
(staging_directory, dest_dir),
should_retry_if=_ShouldRetry)
except retry.RetryException as e:
raise exceptions.ToolException((
'Could not synchronize files. The gsutil command exited with '
'status [{s}]. Command output is available in [{l}].'
).format(s=e.last_result, l=log.GetLogFilePath()))
finally:
# Reset to the standard log level.
log.SetUserOutputEnabled(None)
log.status.Print('File upload done.')
return manifests
