def _FindExecutable(exe):
"""Finds the path to an executable.
Args:
exe: Name of the executable.
Returns:
Path to the executable.
Raises:
EnvironmentError: The exeuctable can't be found.
"""
path = file_utils.FindExecutableOnPath(exe) or _FindOrInstallComponent(exe)
if not path:
raise EnvironmentError('Unable to locate %s.' % exe)
return path
def _GetGsutilPath():
"""Determines the path to the gsutil binary."""
sdk_bin_path = config.Paths().sdk_bin_path
if not sdk_bin_path:
# Check if gsutil is located on the PATH.
gsutil_path = file_utils.FindExecutableOnPath('gsutil')
if gsutil_path:
log.debug(
'Using gsutil found at [{path}]'.format(path=gsutil_path))
return gsutil_path
else:
raise exceptions.ToolException(
('A SDK root could not be found. Please '
'check your installation.'))
return os.path.join(sdk_bin_path, 'gsutil')
def _GetGsutilPath():
"""Determines the path to the gsutil binary."""
sdk_bin_path = config.Paths().sdk_bin_path
if not sdk_bin_path:
# Check if gsutil is located on the PATH.
gsutil_path = file_utils.FindExecutableOnPath('gsutil')
if gsutil_path:
log.debug(
'Using gsutil found at [{path}]'.format(path=gsutil_path))
return gsutil_path
else:
raise GsutilError(
'A path to the storage client `gsutil` could not be '
'found. Please check your SDK installation.')
return os.path.join(sdk_bin_path, 'gsutil')
def _GetCloudSqlProxyPath():
"""Determines the path to the cloud_sql_proxy binary."""
sdk_bin_path = config.Paths().sdk_bin_path
if not sdk_bin_path:
# Check if cloud_sql_proxy is located on the PATH.
proxy_path = file_utils.FindExecutableOnPath('cloud_sql_proxy')
if proxy_path:
log.debug('Using cloud_sql_proxy found at [{path}]'.format(
path=proxy_path))
return proxy_path
else:
raise exceptions.ToolException(
'A Cloud SQL Proxy SDK root could not be found. Please check your '
'installation.')
return os.path.join(sdk_bin_path, 'cloud_sql_proxy')
def GetGcloudPreferredExecutable(exe):
"""Finds the path to an executable, preferring the gcloud packaged version.
Args:
exe: Name of the executable.
Returns:
Path to the executable.
Raises:
EnvironmentError: The executable can't be found.
"""
path = _FindOrInstallComponent(exe) or file_utils.FindExecutableOnPath(exe)
if not path:
raise EnvironmentError('Unable to locate %s.' % exe)
return path
def Current(cls):
"""Retrieve the current environment.
Returns:
Environment, the active and current environment on this machine.
"""
if platforms.OperatingSystem.IsWindows():
commands = Environment._WINDOWS_COMMANDS
path = _SdkHelperBin()
else:
commands = Environment._NIX_COMMANDS
path = None
env = Environment()
for key, cmd in commands.iteritems():
setattr(env, key, files.FindExecutableOnPath(cmd, path=path))
return env
def _GetGcloudScript():
"""Get name of the gcloud script."""
if (platforms.OperatingSystem.Current() ==
platforms.OperatingSystem.WINDOWS):
gcloud_ext = '.cmd'
else:
gcloud_ext = ''
gcloud_name = 'gcloud'
gcloud = files.FindExecutableOnPath(gcloud_name, pathext=[gcloud_ext])
if not gcloud:
raise GcloudIsNotInPath(
'Could not verify that gcloud is in the PATH. '
'Please make sure the Cloud SDK bin folder is in PATH.')
return gcloud_name + gcloud_ext
def More(contents, out=None, prompt=None, check_pager=True):
"""Run a user specified pager or fall back to the internal pager.
Args:
contents: The entire contents of the text lines to page.
out: The output stream, log.out (effectively) if None.
prompt: The page break prompt.
check_pager: Checks the PAGER env var and uses it if True.
"""
if not IsInteractive(output=True):
if not out:
out = log.out
out.write(contents)
return
if not out:
# Rendered help to the log file.
log.file_only_logger.info(contents)
# Paging shenanigans to stdout.
out = sys.stdout
if check_pager:
pager = encoding.GetEncodedValue(os.environ, 'PAGER', None)
if pager == '-':
# Use the fallback Pager.
pager = None
elif not pager:
# Search for a pager that handles ANSI escapes.
for command in ('less', 'pager'):
if files.FindExecutableOnPath(command):
pager = command
break
if pager:
# If the pager is less(1) then instruct it to display raw ANSI escape
# sequences to enable colors and font embellishments.
less_orig = encoding.GetEncodedValue(os.environ, 'LESS', None)
less = '-R' + (less_orig or '')
encoding.SetEncodedValue(os.environ, 'LESS', less)
p = subprocess.Popen(pager, stdin=subprocess.PIPE, shell=True)
enc = console_attr.GetConsoleAttr().GetEncoding()
p.communicate(input=contents.encode(enc))
p.wait()
if less_orig is None:
encoding.SetEncodedValue(os.environ, 'LESS', None)
return
# Fall back to the internal pager.
console_pager.Pager(contents, out, prompt).Run()
def InstallBinaryNoOverrides(binary_name, prompt):
"""Helper method for installing binary dependencies within command execs."""
console_io.PromptContinue(
message='Pausing command execution:',
prompt_string=prompt,
cancel_on_no=True,
cancel_string='Aborting component install for {} and command execution.'
.format(binary_name))
platform = platforms.Platform.Current()
update_manager_client = update_manager.UpdateManager(platform_filter=platform)
update_manager_client.Install([binary_name])
path_executable = files.FindExecutableOnPath(binary_name)
if path_executable:
return path_executable
raise MissingExecutableException(
binary_name, '{} binary not installed'.format(binary_name))
def _EnsureDockerRunning():
"""Make sure docker is running."""
docker = file_utils.FindExecutableOnPath('docker')
if not docker:
raise RuntimeMissingDependencyError(
'Cannot locate docker on $PATH. Install docker from '
'https://docs.docker.com/get-docker/.')
try:
# docker info returns 0 if it can connect to the docker daemon and
# returns a non-zero error code if it cannot. run_subprocess
# checks raises an error if the process does not return 0.
run_subprocess.Run([docker, 'info'],
timeout_sec=20,
show_output=_IsDebug())
except subprocess.CalledProcessError:
raise RuntimeMissingDependencyError(
'Unable to reach docker daemon. Make sure docker is running '
'and reachable.')
def __init__(self, suite, bin_path=None):
"""Create a new environment by supplying a suite and command directory.
Args:
suite: Suite, the suite for this environment.
bin_path: str, the path where the commands are located. If None, use
standard $PATH.
"""
self.suite = suite
self.bin_path = bin_path
# So pytype is aware of attributes.
self.ssh = None
self.ssh_term = None
self.scp = None
self.keygen = None
for key, cmd in six.iteritems(self.COMMANDS[suite]):
setattr(self, key, files.FindExecutableOnPath(cmd, path=self.bin_path))
self.ssh_exit_code = self.SSH_EXIT_CODES[suite]
def _GetPrimaryNodeName():
"""Get the primary node name.
Returns:
str, the name of the primary node. If running in tensorflow 1.x,
return 'master'. If running in tensorflow 2.x, return 'chief'.
If tensorflow is not installed in local envrionment, it will return
the default name 'master'.
Raises:
RuntimeError: if there is no python executable on the user system.
"""
exe_override = properties.VALUES.ml_engine.local_python.Get()
python_executable = exe_override or files.FindExecutableOnPath('python')
if not python_executable:
raise RuntimeError('No python interpreter found on local machine')
cmd = [
python_executable, '-c',
'import tensorflow as tf; print(tf.version.VERSION)'
]
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
return_code = proc.wait()
if return_code != 0:
log.warning('''
Cannot import tensorflow under path {}. Using "chief" for cluster setting.
If this is not intended, Please check if tensorflow is installed. Please also
verify if the python path used is correct. If not, to change the python path:
use `gcloud config set ml_engine/local_python $python_path`
Eg: gcloud config set ml_engine/local_python /usr/bin/python3'''.format(
python_executable))
return 'chief'
tf_version = proc.stdout.read()
if 'decode' in dir(tf_version):
tf_version = tf_version.decode('utf-8')
if tf_version.startswith('1.'):
return 'master'
elif tf_version.startswith('2.'):
return 'chief'
log.warning(
'Unexpected tensorflow version {}, using the default primary'
' node name, aka "chief" for cluster settings'.format(tf_version))
return 'chief'
def More(contents, out=None, prompt=None, check_pager=True):
"""Run a user specified pager or fall back to the internal pager.
Args:
contents: The entire contents of the text lines to page.
out: The output stream, log.out (effectively) if None.
prompt: The page break prompt.
check_pager: Checks the PAGER env var and uses it if True.
"""
if not IsInteractive(output=True):
if not out:
out = log.out
out.write(contents)
return
if not out:
# Rendered help to the log file.
log.file_only_logger.info(contents)
# Paging shenanigans to stdout.
out = sys.stdout
if check_pager:
pager = os.environ.get('PAGER', None)
if pager == '-':
# Use the fallback Pager.
pager = None
elif not pager:
# Search for a pager that handles ANSI escapes.
for command in ('less', 'pager'):
if files.FindExecutableOnPath(command):
pager = command
break
if pager:
less = os.environ.get('LESS', None)
if less is None:
os.environ['LESS'] = '-R'
p = subprocess.Popen(pager, stdin=subprocess.PIPE, shell=True)
encoding = console_attr.GetConsoleAttr().GetEncoding()
p.communicate(input=contents.encode(encoding))
p.wait()
if less is None:
os.environ.pop('LESS')
return
# Fall back to the internal pager.
console_pager.Pager(contents, out, prompt).Run()
def RequireJavaInstalled(for_text, min_version=7):
"""Require that a certain version of Java is installed.
Args:
for_text: str, the text explaining what Java is necessary for.
min_version: int, the minimum major version to check for.
Raises:
JavaError: if a Java executable is not found or has the wrong version.
Returns:
str, Path to the Java executable.
"""
java_path = files.FindExecutableOnPath('java')
if not java_path:
raise JavaError('To use the {for_text}, a Java {v}+ JRE must be installed '
'and on your system PATH'.format(for_text=for_text,
v=min_version))
try:
output = subprocess.check_output([java_path, '-version'],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
raise JavaError('Unable to execute the java that was found on your PATH.'
' The {for_text} requires a Java {v}+ JRE installed and on '
'your system PATH'.format(for_text=for_text, v=min_version))
java_exec_version_error = JavaError(
'The java executable on your PATH is not a Java {v}+ JRE.'
' The {for_text} requires a Java {v}+ JRE installed and on '
'your system PATH'.format(v=min_version, for_text=for_text))
match = re.search(r'version "(\d+)\.(\d+)\.', output)
if not match:
raise java_exec_version_error
major_version = int(match.group(1))
minor_version = int(match.group(2))
# Java <= 8 used 1.X version format. Java > 8 uses X.* version format.
if ((major_version == 1 and minor_version < min_version) or
(major_version > 1 and major_version < min_version)):
raise java_exec_version_error
return java_path
def _JavaStagingMapper(command_path, descriptor, staging_dir):
"""Map a java staging request to the right args.
Args:
command_path: str, path to the jar tool file.
descriptor: str, path to the `appengine-web.xml`
staging_dir: str, path to the empty staging dir
Raises:
java.JavaError, if Java is not installed.
Returns:
[str], args for executable invocation.
"""
java.CheckIfJavaIsInstalled('local staging for java')
java_bin = files.FindExecutableOnPath('java')
app_dir = os.path.dirname(os.path.dirname(descriptor))
args = ([java_bin, '-classpath', command_path, _JAVA_APPCFG_ENTRY_POINT] +
_JAVA_APPCFG_STAGE_FLAGS + ['stage', app_dir, staging_dir])
return args
def StartEmulatorProxy(args=None):
"""Starts the emulator reverse proxy, as a context manager.
Note that this will SIGKILL the process once the context comes back to
the message, so the caller can and should try to shut down the process
more gracefully.
Args:
args: [str], the arguments to be passed to the relevant script
Yields:
The proxy process.
"""
reverse_proxy_jar = ReverseProxyJar()
java_path = files.FindExecutableOnPath('java')
classname = 'com.google.cloudsdk.emulators.EmulatorProxy'
proc = subprocess.Popen([java_path, '-cp', reverse_proxy_jar, classname] +
args,
stdout=subprocess.PIPE)
yield proc
proc.kill()
def _GetGcloudScript(full_path=False):
"""Get name of the gcloud script.
Args:
full_path: boolean, True if the gcloud full path should be used if free
of spaces.
Returns:
str, command to use to execute gcloud
Raises:
GcloudIsNotInPath: if gcloud is not found in the path
"""
if (platforms.OperatingSystem.Current() ==
platforms.OperatingSystem.WINDOWS):
gcloud_ext = '.cmd'
else:
gcloud_ext = ''
gcloud_name = 'gcloud'
gcloud = files.FindExecutableOnPath(gcloud_name, pathext=[gcloud_ext])
if not gcloud:
raise GcloudIsNotInPath(
'Could not verify that gcloud is in the PATH. '
'Please make sure the Cloud SDK bin folder is in PATH.')
if full_path:
if not re.match(r'[-a-zA-Z0-9_/]+$', gcloud):
log.warning(
textwrap.dedent("""\
You specified the option to use the full gcloud path in the git
credential.helper, but the path contains non alphanumberic characters
so the credential helper may not work correctly."""))
return gcloud
else:
return gcloud_name + gcloud_ext
def RunKubectlCommand(args, out_func=None, err_func=None):
"""Shells out a command to kubectl.
This command should be called within the context of a TemporaryKubeconfig
context manager in order for kubectl to be configured to access the correct
cluster.
Args:
args: list of strings, command line arguments to pass to the kubectl
command. Should omit the kubectl command itself. For example, to
execute 'kubectl get pods', provide ['get', 'pods'].
out_func: str->None, a function to call with the stdout of the kubectl
command
err_func: str->None, a function to call with the stderr of the kubectl
command
Raises:
Error: if kubectl could not be called
KubectlError: if the invocation of kubectl was unsuccessful
"""
kubectl_path = files.FindExecutableOnPath(_KUBECTL_COMPONENT_NAME,
config.Paths().sdk_bin_path)
if kubectl_path is None:
raise Error(MISSING_KUBECTL_MSG)
try:
retval = execution_utils.Exec(
execution_utils.ArgsForExecutableTool(kubectl_path, *args),
no_exit=True,
out_func=out_func,
err_func=err_func,
universal_newlines=True)
except (execution_utils.PermissionError,
execution_utils.InvalidCommandError) as e:
raise KubectlError(six.text_type(e))
if retval:
raise KubectlError('kubectl returned non-zero status code.')
def CheckForInstalledBinary(binary_name,
check_hidden=False,
custom_message=None,
install_if_missing=False):
"""Check if binary is installed and return path or raise error.
Prefer the installed component over any version found on path.
Args:
binary_name: str, name of binary to search for.
check_hidden: bool, whether to check hidden components for the binary.
custom_message: str, custom message to used by
MissingExecutableException if thrown.
install_if_missing: bool, if true will prompt user to install binary if
not found.
Returns:
Path to executable if found on path or installed component.
Raises:
MissingExecutableException: if executable can not be found or can not be
installed as a component.
"""
is_component = CheckBinaryComponentInstalled(binary_name, check_hidden)
if is_component:
return os.path.join(config.Paths().sdk_bin_path, binary_name)
path_executable = files.FindExecutableOnPath(binary_name)
if path_executable:
return path_executable
if install_if_missing:
return InstallBinaryNoOverrides(
binary_name, _INSTALL_MISSING_EXEC_PROMPT.format(binary=binary_name))
raise MissingExecutableException(binary_name, custom_message)
def RunEmulatorProxyClient(log_file=None, env=None):
"""Runs proxy client to test running emulator reverse proxy.
Args:
log_file: int, a file to reroute stdout and stderr to.
env: dict, the env for the subprocess.
Yields:
the calling subprocess
"""
reverse_proxy_jar = proxy_util.ReverseProxyJar()
java_path = files.FindExecutableOnPath('java')
classname = 'com.google.cloudsdk.emulators.Testing$ProtoClient'
stdout = log_file if log_file is not None else subprocess.PIPE
stderr = log_file if log_file is not None else subprocess.PIPE
proc = subprocess.Popen(
[java_path, '-cp', reverse_proxy_jar, classname],
stdout=stdout, stderr=stderr, env=env)
yield proc
try:
proc.kill()
except OSError:
# The caller did our dirty work for us
pass
def _GetManPageCollectorType(cls):
"""Returns the man page collector type."""
if files.FindExecutableOnPath('man'):
return _ManCommandCollector
return _ManUrlCollector
def Run(self, args):
"""Connects to a Cloud SQL instance.
Args:
args: argparse.Namespace, The arguments that this command was invoked
with.
Returns:
If no exception is raised this method does not return. A new process is
started and the original one is killed.
Raises:
HttpException: An http error response was received while executing api
request.
ToolException: An error other than http error occurred while executing the
command.
"""
# TODO(b/62055495): Replace ToolExceptions with specific exceptions.
client = api_util.SqlClient(api_util.API_VERSION_DEFAULT)
sql_client = client.sql_client
sql_messages = client.sql_messages
validate.ValidateInstanceName(args.instance)
instance_ref = client.resource_parser.Parse(
args.instance,
params={'project': properties.VALUES.core.project.GetOrFail},
collection='sql.instances')
acl_name = _WhitelistClientIP(instance_ref, sql_client, sql_messages,
client.resource_parser)
# Get the client IP that the server sees. Sadly we can only do this by
# checking the name of the authorized network rule.
retryer = retry.Retryer(max_retrials=2, exponential_sleep_multiplier=2)
try:
instance_info, client_ip = retryer.RetryOnResult(
_GetClientIP,
[instance_ref, sql_client, acl_name],
should_retry_if=lambda x, s: x[1] is None, # client_ip is None
sleep_ms=500)
except retry.RetryException:
raise exceptions.ToolException('Could not whitelist client IP. Server '
'did not reply with the whitelisted IP.')
# Check for the mysql or psql executable based on the db version.
db_type = instance_info.databaseVersion.split('_')[0]
exe_name = constants.DB_EXE.get(db_type, 'mysql')
exe = files.FindExecutableOnPath(exe_name)
if not exe:
raise exceptions.ToolException(
'{0} client not found. Please install a {1} client and make sure '
'it is in PATH to be able to connect to the database instance.'
.format(exe_name.title(), exe_name))
# Check the version of IP and decide if we need to add ipv4 support.
ip_type = network.GetIpVersion(client_ip)
if ip_type == network.IP_VERSION_4:
if instance_info.settings.ipConfiguration.ipv4Enabled:
ip_address = instance_info.ipAddresses[0].ipAddress
else:
# TODO(b/36049930): ask user if we should enable ipv4 addressing
message = ('It seems your client does not have ipv6 connectivity and '
'the database instance does not have an ipv4 address. '
'Please request an ipv4 address for this database instance.')
raise exceptions.ToolException(message)
elif ip_type == network.IP_VERSION_6:
ip_address = instance_info.ipv6Address
else:
raise exceptions.ToolException('Could not connect to SQL server.')
# Determine what SQL user to connect with.
sql_user = constants.DEFAULT_SQL_USER[exe_name]
if args.user:
sql_user = args.user
# We have everything we need, time to party!
flags = constants.EXE_FLAGS[exe_name]
sql_args = [exe_name, flags['hostname'], ip_address]
sql_args.extend([flags['user'], sql_user])
sql_args.append(flags['password'])
try:
log.status.write(
'Connecting to database with SQL user [{0}].'.format(sql_user))
execution_utils.Exec(sql_args)
except OSError:
log.error('Failed to execute command "{0}"'.format(' '.join(sql_args)))
log.Print(info_holder.InfoHolder())
class UploadDownloadTest(e2e_base.WithServiceAuth):
"""Test uploading a small number of small files to Cloud Storage."""
_GSUTIL_EXECUTABLE = files.FindExecutableOnPath('gsutil')
def SetUp(self):
self.storage_client = storage_api.StorageClient()
self.files_to_upload = []
self.object_path = next(
e2e_utils.GetResourceNameGenerator(prefix='object'))
self.bucket_name = next(
e2e_utils.GetResourceNameGenerator(prefix=BUCKET_PREFIX))
def _AssertFileUploaded(self, bucket_ref, expected_file):
object_ = storage_util.ObjectReference.FromBucketRef(
bucket_ref, expected_file)
try:
self.storage_client.GetObject(object_)
except apitools_exceptions.HttpError as err:
self.fail('Object [{}] not successfully uploaded:\n\n{}'.format(
object_.ToUrl(), str(err)))
def _TestUploadAndDownload(self, contents):
with storage_e2e_util.CloudStorageBucket(self.storage_client,
self.bucket_name,
self.Project()) as bucket:
file_path = self.Touch(self.temp_path,
'test_file',
contents=contents.encode('utf-8'))
target_obj_ref = storage_util.ObjectReference.FromBucketRef(
bucket, self.object_path)
with storage_e2e_util.GcsFile(self.storage_client, file_path,
target_obj_ref):
self._AssertFileUploaded(bucket, self.object_path)
download_path = os.path.join(self.temp_path, 'download_file')
source_obj_ref = storage_util.ObjectReference.FromBucketRef(
bucket, self.object_path)
self.storage_client.CopyFileFromGCS(source_obj_ref,
download_path)
# Now download again using ReadObject, the in-memory version of
# CopyFileFromGCS
object_ref = storage_util.ObjectReference.FromBucketRef(
bucket, self.object_path)
stream = self.storage_client.ReadObject(object_ref)
# Check regular file download
self.AssertFileExists(download_path)
actual_contents = files.ReadFileContents(download_path)
self.assertEqual(contents, actual_contents)
# Check stream download
self.assertEqual(stream.getvalue().decode('utf-8'), contents)
def testCopyFileToAndFromGcs(self):
self._TestUploadAndDownload('test file content.')
def testCopyFileToAndFromGcs_NonAscii(self):
self._TestUploadAndDownload('\u0394')
def testCopyFileToAndFromGcs_LargeFile(self):
"""Tests file uploads that require chunking."""
# Default chunk size isn't available as a constant
# Need 10 multiples until issue with copied stream chunks appears.
file_length = properties.VALUES.storage.chunk_size.GetInt() * 10
# There's a check for large files accidentally left in the temporary
# directory; we're okay with it in this case, since this deliberately tests
# a large file.
self._dirs_size_limit_method = file_length * 2
file_path = self.Touch(self.temp_path,
'test_file',
contents=('.' * file_length))
with storage_e2e_util.CloudStorageBucket(self.storage_client,
self.bucket_name,
self.Project()) as bucket:
target_obj_ref = storage_util.ObjectReference.FromBucketRef(
bucket, self.object_path)
with storage_e2e_util.GcsFile(self.storage_client, file_path,
target_obj_ref):
self._AssertFileUploaded(bucket, self.object_path)
# Don't run the download portion of the test as a time-saving measure
@test_case.Filters.RunOnlyIf(_GSUTIL_EXECUTABLE, 'No gsutil found')
def testRunGsutilCommand(self):
self.assertEqual(0, storage_util.RunGsutilCommand('help'))
@test_case.Filters.RunOnlyIf(_GSUTIL_EXECUTABLE, 'No gsutil found')
def testGsutilCopy(self):
file_length = 1024
file_path = self.Touch(self.temp_path,
'test_file',
contents=('.' * file_length))
with storage_e2e_util.CloudStorageBucket(self.storage_client,
self.bucket_name,
self.Project()) as bucket:
# Test upload
object_uri = 'gs://{bucket}/{object_path}'.format(
bucket=self.bucket_name, object_path=self.object_path)
exit_code = storage_util.RunGsutilCommand('cp',
[file_path, object_uri])
self.assertEqual(0, exit_code)
self._AssertFileUploaded(bucket, self.object_path)
# Test download
download_file_path = os.path.join(self.temp_path, 'download_file')
download_exit_code = storage_util.RunGsutilCommand(
'cp', [object_uri, download_file_path])
self.assertEqual(0, download_exit_code)
self.assertTrue(os.path.exists(download_file_path))
self.assertTrue(filecmp.cmp(file_path, download_file_path))
# Try to clean up
obj_ref = storage_util.ObjectReference.FromUrl(object_uri)
self.storage_client.DeleteObject(obj_ref)
def __call__(self, parser, namespace, values, option_string=None):
alias_args = ['gcloud']
# subprocess.call() below handles 'shell not found' diagnostics
if values:
shell = values
else:
shell = os.environ.get('SHELL')
if not shell:
# search for a default for SHELL, biased from left to right
for shell in ['bash', 'ksh', 'sh', 'zsh', 'dash']:
path = file_utils.FindExecutableOnPath(shell)
if path:
shell = path
break
else:
shell = 'sh'
for arg in cli.argv:
if arg == '--shell' or arg.startswith('--shell='):
# Only things up to, and not including, the first --shell.
# TODO(user): This search can have false positives. eg,
# $ gcloud --project --shell auth --shell
# If someone somehow had a project "--shell", or if some other flag
# flag value was legitimately "--shell". For now, we'll let this be
# a problematic, but rare, corner case.
break
# TODO(user): Make this quoting more robust.
if ' ' in arg:
arg = '"{arg}"'.format(arg=arg)
alias_args.append(arg)
alias_prefix = ' '.join(alias_args)
prompt = ' '.join(['gcloud'] + alias_args[1:])
interactive = sys.stdin.isatty()
buf = StringIO.StringIO()
GenerateRcFile(alias_prefix, prompt, subcommands, interactive, buf)
exit_code = 0
with file_utils.TemporaryDirectory() as tmpdir:
# link or symlink not available on all targets so we make N copies.
envfile = '.gcloudenv'
for rcfile in ['.bashrc', '.zshrc', envfile]:
path = os.path.join(tmpdir, rcfile)
with open(path, 'w') as f:
f.write(buf.getvalue())
f.flush()
try:
restore = ''
for name in ['HOME', 'ZDOTDIR', 'ENV']:
val = os.environ.get(name)
if val is not None:
restore += ' ' + name + "='" + val + "'"
if restore is not '':
restore = 'export' + restore
env = dict(os.environ)
env['_GCLOUD_RESTORE_'] = restore
env['HOME'] = tmpdir
env['ZDOTDIR'] = tmpdir
if os.sep == '\\':
# Workaround for UWIN ksh(1) PATH lookup on pure windows paths.
# The embeded '/' means "this is literal, don't do PATH lookup".
# Also handles the eval of $ENV that eliminates \'s.
env['ENV'] = '/'.join([tmpdir,
envfile]).replace('\\', '\\\\')
else:
env['ENV'] = path
# Why print terminal escape sequences if stdout is not a terminal?
if not sys.stdout.isatty():
env['TERM'] = 'dumb'
cmd = [shell, '-i']
# not interactive implies batch mode with commands on stdin. Since zsh
# insists on reading from /dev/tty we stuff it in a new sesssion which
# detaches /dev/tty and forces it to read from stdin. bash and dash
# complain about no tty in -i mode, so zsh is special-cased.
if not interactive and os.path.basename(shell).startswith(
'zsh'):
# eventually change preexec_fn=os.setsid to start_new_session=True
exit_code = subprocess.call(cmd,
env=env,
preexec_fn=os.setsid)
else:
exit_code = subprocess.call(cmd, env=env)
except OSError as e:
log.error("""\
could not run the shell [{shell}] -- \
make sure it is installed and on the system PATH [{e}]\
""".format(e=e, shell=shell))
exit_code = 1
sys.exit(exit_code)
b'CfesnGxEEzpu66fb4M4S36sCAwEAAQ==\n'
b'-----END PUBLIC KEY-----\n',
b'y78flNI4F4I44vxeFq9BaqBXZGjmkELfatGSlsGwOBP9Xmhwh8jkCijdCgYcMEdp'
b'BWhHFwd50lJyO6wWQPCh6/p3i4nA/mgQ9/3S6EMw+HPKO/IEpv7uOv7j0vT27jDo'
b'yopEDHU5a7IJctWq8Xyc4euyd/xTG6v8X++UdJL1keS5Ftw3/FZfWt4iaDKzl/EP'
b'PNvYlh6YdVevxjmmyuHn5gT+y8Qqj++zSe8wMls5t2/sTCcVi8p5ZP0HQvpmQVyA'
b'wlhNN6WmleE5r2LJpjtfCfHwLOzgGEiS0zGSSAGOSdFwVA7rqCt1xvtYdyrphNXq'
b'i1vVERDvWox7sCypQeyNkqMvYozqwOw1r0FgMVvgj7WTyT6rhfNmRR88o2+k/Fvc'
b'TVZZJ0aJe0Hp3KwShq7kklCwWHlNwdt2jRtCO8aHQiPGfEzpBBzJYAgmLq8xX7Ez'
b'J+ht1MVjMTIC//dkfUSymYgrVcUc7hBOuRpvoxo2Ze5zDsYXUf2Zj5vcBbbM/5bZ'
b'mHwoE42y0NhAsaxilFybBNdDUpiFs0MMg28HcOUCXoUj5Ax3P4WmcUh4PDGqqenr'
b'GqCoIp1vkMmrDmQOIxHPhMJT9edx8sYgzbk2fbB5Fty/byMlr6hlPmm7P8/qfctE'
b'/9b4qoiy+dxDe7rYnP4kCfesnGxEEzpu66fb4M4S36s=')
]
_OPEN_SSL_EXECUTABLE = files.FindExecutableOnPath('openssl')
@test_case.Filters.RunOnlyIf(_OPEN_SSL_EXECUTABLE, 'No openssl found')
@test_case.Filters.DoNotRunOnWindows(
'Windows uses Windows Crypto APIs instead of OpenSSL')
class OpensslEncryptionUtilsTest(test_case.TestCase):
def SetUp(self):
self.crypt = openssl_encryption_utils.OpensslCrypt(_OPEN_SSL_EXECUTABLE)
def testStripKey(self):
expected_output = (
b'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvjrnuNYG8QtewB/mcbS9'
b'uqyN3gCG+RulSv4FZAFVib+nbdFpQemcFHfKVk3tve+J24U3TW87B49mat3tvPPl'
b's5y0N6esYEDyvaynP5ELdQWGSf9V9Kr6b2yBreyq03WpzlMRdLZ7Mj8W6SzK9BpJ'
def Run(self, args):
start = time_utils.CurrentTimeSec()
# Set up Encryption utilities.
openssl_executable = files.FindExecutableOnPath('openssl')
if windows_encryption_utils:
crypt = windows_encryption_utils.WinCrypt()
elif openssl_executable:
crypt = openssl_encryption_utils.OpensslCrypt(openssl_executable)
else:
raise utils.MissingDependencyError(
'Your platform does not support OpenSSL.')
# Get Authenticated email address and default username.
email = gaia_utils.GetAuthenticatedGaiaEmail(self.http)
if args.user:
user = args.user
else:
user = gaia_utils.MapGaiaEmailToDefaultAccountName(email)
if args.name == user:
raise utils.InvalidUserError(
MACHINE_USERNAME_SAME_ERROR.format(user, args.name))
# Warn user (This warning doesn't show for non-interactive sessions).
message = RESET_PASSWORD_WARNING.format(user)
prompt_string = (
'Would you like to set or reset the password for [{0}]'.format(
user))
console_io.PromptContinue(message=message,
prompt_string=prompt_string,
cancel_on_no=True)
log.status.Print(
'Resetting and retrieving password for [{0}] on [{1}]'.format(
user, args.name))
# Get Encryption Keys.
key = crypt.GetKeyPair()
modulus, exponent = crypt.GetModulusExponentFromPublicKey(
crypt.GetPublicKey(key))
# Create Windows key entry.
self.windows_key_entry = self._ConstructWindowsKeyEntry(
user, modulus, exponent, email)
# Call ReadWriteCommad.Run() which will fetch the instance and update
# the metadata (using the data in self.windows_key_entry).
objects = super(ResetWindowsPassword, self).Run(args)
updated_instance = list(objects)[0]
# Retrieve and Decrypt the password from the serial console.
enc_password = self._GetEncryptedPasswordFromSerialPort(modulus)
password = crypt.DecryptMessage(key, enc_password)
# Get External IP address.
try:
access_configs = updated_instance['networkInterfaces'][0][
'accessConfigs']
external_ip_address = access_configs[0]['natIP']
except KeyError:
log.warn(NO_IP_WARNING.format(updated_instance['name']))
external_ip_address = None
# Check for old Windows credentials.
if self.old_metadata_keys:
log.warn(
OLD_KEYS_WARNING.format(self.ref.Name(), self.ref.Name(),
self.ref.zone,
','.join(self.old_metadata_keys)))
log.info('Total Elapsed Time: {0}'.format(time_utils.CurrentTimeSec() -
start))
# The connection info resource.
connection_info = {
'username': user,
'password': password,
'ip_address': external_ip_address
}
return connection_info
def __init__(self, cli, args):
super(UnknownReferenceMapper, self).__init__(cli, args)
self.known = files.FindExecutableOnPath(args[0])
def CheckKubectlInstalled():
"""Verify that the kubectl component is installed or print a warning."""
if (not file_utils.FindExecutableOnPath(_KUBECTL_COMPONENT_NAME)
and not _KubectlInstalledAsComponent()):
log.warn(MISSING_KUBECTL_MSG)
def Run(self, args):
message = (
'A personal authentication session will propagate your personal '
'credentials to the cluster, so make sure you trust the cluster '
'and the user who created it.')
console_io.PromptContinue(
message=message,
cancel_on_no=True,
cancel_string='Enabling session aborted by user')
dataproc = dp.Dataproc(self.ReleaseTrack())
cluster_ref = args.CONCEPTS.cluster.Parse()
project = cluster_ref.projectId
region = cluster_ref.region
cluster_name = cluster_ref.clusterName
get_request = dataproc.messages.DataprocProjectsRegionsClustersGetRequest(
projectId=project, region=region, clusterName=cluster_name)
cluster = dataproc.client.projects_regions_clusters.Get(get_request)
cluster_uuid = cluster.clusterUuid
if args.access_boundary:
with files.FileReader(args.access_boundary, mode='r') as abf:
access_boundary_json = abf.read()
else:
access_boundary_json = flags.ProjectGcsObjectsAccessBoundary(
project)
openssl_executable = args.openssl_command
if not openssl_executable:
try:
openssl_executable = files.FindExecutableOnPath('openssl')
except ValueError:
log.fatal(
'Could not find openssl on your system. The enable-session '
'command requires openssl to be installed.')
operation_poller = waiter.CloudOperationPollerNoResources(
dataproc.client.projects_regions_operations,
lambda operation: operation.name)
try:
cluster_key = clusters.ClusterKey(cluster)
if not cluster_key:
raise exceptions.PersonalAuthError(
'The cluster {} does not support personal auth.'.format(
cluster_name))
with progress_tracker.ProgressTracker(
'Injecting initial credentials into the cluster {}'.format(
cluster_name),
autotick=True):
self.inject_credentials(dataproc, project, region,
cluster_name, cluster_uuid,
cluster_key, access_boundary_json,
openssl_executable, operation_poller)
if not args.refresh_credentials:
return
update_message = (
'Periodically refreshing credentials for cluster {}. This'
' will continue running until the command is interrupted'
).format(cluster_name)
with progress_tracker.ProgressTracker(update_message,
autotick=True):
try:
# Cluster keys are periodically regenerated, so fetch the latest
# each time we inject credentials.
cluster = dataproc.client.projects_regions_clusters.Get(
get_request)
cluster_key = clusters.ClusterKey(cluster)
if not cluster_key:
raise exceptions.PersonalAuthError(
'The cluster {} does not support personal auth.'.
format(cluster_name))
failure_count = 0
while failure_count < 3:
try:
time.sleep(30)
self.inject_credentials(dataproc, project, region,
cluster_name, cluster_uuid,
cluster_key,
access_boundary_json,
openssl_executable,
operation_poller)
failure_count = 0
except ValueError as err:
log.error(err)
failure_count += 1
raise exceptions.PersonalAuthError(
'Credential injection failed three times in a row, giving up...'
)
except (console_io.OperationCancelledError, KeyboardInterrupt):
return
except exceptions.PersonalAuthError as err:
log.error(err)
return
def MakeProcess(module_name,
package_root,
args=None,
cluster=None,
task_type=None,
index=None,
**extra_popen_args):
"""Make a Popen object that runs the module, with the correct env.
If task_type is primary instead replaces the current process with the
subprocess via execution_utils.Exec
Args:
module_name: str. Name of the module to run, e.g. trainer.task
package_root: str. Absolute path to the package root for the module.
used as CWD for the subprocess.
args: [str]. Additional user args. Any relative paths will not work.
cluster: dict. Cluster configuration dictionary. Suitable for passing to
tf.train.ClusterSpec.
task_type: str. Task type of this process. Only relevant if cluster is
specified.
index: int. Task index of this process.
**extra_popen_args: extra args passed to Popen. Used for testing.
Returns:
a subprocess.Popen object corresponding to the subprocesses or an int
corresponding to the return value of the subprocess
(if task_type is primary)
Raises:
RuntimeError: if there is no python executable on the user system
"""
if args is None:
args = []
exe_override = properties.VALUES.ml_engine.local_python.Get()
python_executable = exe_override or files.FindExecutableOnPath('python')
if not python_executable:
raise RuntimeError('No python interpreter found on local machine')
cmd = [python_executable, '-m', module_name] + args
config = {
'job': {
'job_name': module_name,
'args': args
},
'task': {
'type': task_type,
'index': index
} if cluster else {},
'cluster': cluster or {},
'environment': 'cloud'
}
log.info(('launching training process:\n'
'command: {cmd}\n config: {config}').format(cmd=' '.join(cmd),
config=json.dumps(
config,
indent=2,
sort_keys=True)))
env = os.environ.copy()
# the tf_config environment variable is used to pass the tensorflow
# configuration options to the training module. the module specific
# arguments are passed as command line arguments.
env['TF_CONFIG'] = json.dumps(config)
if task_type == _GetPrimaryNodeName():
return execution_utils.Exec(cmd,
env=env,
no_exit=True,
cwd=package_root,
**extra_popen_args)
else:
env = encoding.EncodeEnv(env)
task = subprocess.Popen(cmd,
env=env,
cwd=package_root,
**extra_popen_args)
atexit.register(execution_utils.KillSubprocess, task)
return task