def ReportMetrics(self, wait_for_report=False):
"""Reports the collected metrics using a separate async process."""
if not self._metrics:
return
temp_metrics_file = tempfile.NamedTemporaryFile(delete=False)
with temp_metrics_file:
pickle.dump(self._metrics, temp_metrics_file)
self._metrics = []
# TODO(user): make this not depend on the file.
this_file = console_attr.DecodeFromInput(__file__)
reporting_script_path = os.path.realpath(
os.path.join(os.path.dirname(this_file), 'metrics_reporter.py'))
execution_args = execution_utils.ArgsForPythonTool(
reporting_script_path, temp_metrics_file.name)
exec_env = os.environ.copy()
exec_env['PYTHONPATH'] = os.pathsep.join(sys.path)
try:
p = subprocess.Popen(execution_args, env=exec_env,
**self._async_popen_args)
log.debug('Metrics reporting process started...')
except OSError:
# This can happen specifically if the Python executable moves between the
# start of this process and now.
log.debug('Metrics reporting process failed to start.')
if wait_for_report:
# NOTE: p.wait() can cause a deadlock. p.communicate() is recommended.
# See python docs for more information.
p.communicate()
log.debug('Metrics reporting process finished.')
def ReportMetrics(self, wait_for_report=False):
"""Reports the collected metrics using a separate async process."""
if not self._metrics:
return
temp_metrics_file = tempfile.NamedTemporaryFile(delete=False)
with temp_metrics_file:
pickle.dump(self._metrics, temp_metrics_file)
self._metrics = []
# TODO(cherba): make this not depend on the file.
reporting_script_path = os.path.join(os.path.dirname(__file__),
'metrics_reporter.py')
execution_args = execution_utils.ArgsForPythonTool(
reporting_script_path, temp_metrics_file.name)
exec_env = os.environ.copy()
exec_env['PYTHONPATH'] = os.pathsep.join(sys.path)
p = subprocess.Popen(execution_args,
env=exec_env,
**self._async_popen_args)
if wait_for_report:
# NOTE: p.wait() can cause a deadlock. p.communicate() is recommended.
# See python docs for more information.
p.communicate()
log.debug('Metrics reporting process started...')
def Start(self, *positional, **flags):
"""Start the dev_appserver.
Args:
*positional: str, The positional arguments to be passed to dev_appserver.
**flags: str, The flags to be passed to dev_appserver.
Raises:
DevappserverExecutionError: If devappserver execution returns an error.
"""
all_flags = dict(self._global_flags)
all_flags.update(flags)
# Don't include the script name in argv because we are hijacking the parse
# method.
argv = (self._GenerateFlags(all_flags) +
[arg for arg in positional if arg is not None])
log.debug('Running [dev_appserver.py] with: {cmd}'.format(
cmd=' '.join(argv)))
run_args = execution_utils.ArgsForPythonTool(self.__executable_path,
*argv)
# TODO(markpell): Take this out (b/19485297). This is because the
# devappserver is depending on our pythonpath right now (it should not in
# the future (b/19443812). We need to do this because if something invokes
# gcloud.py directly, the sys.path is updated but is never put in the env.
# If you call gcloud.sh then it does put it in the env so this is not
# required.
env = dict(os.environ)
env['PYTHONPATH'] = os.pathsep.join(sys.path)
return_code = subprocess.call(run_args, env=env)
if return_code != 0:
raise DevappserverExecutionError(return_code, argv)
def ExecutePythonTool(tool_dir, exec_name, *args):
"""Execute the given python script with the given args and command line.
Args:
tool_dir: the directory the tool is located in
exec_name: additional path to the executable under the tool_dir
*args: args for the command
"""
_ExecuteTool(
execution_utils.ArgsForPythonTool(_FullPath(tool_dir, exec_name), *args))
def GetArgs(self):
try:
return execution_utils.ArgsForPythonTool(self.setup_py_path,
*self.setup_py_args)
except ValueError:
# The most common cause of ValueError for ArgsForPythonTool is a missing
# executable; we want to display a more specific error in this case.
if not sys.executable:
raise SysExecutableMissingError()
raise
def ExecutePythonTool(tool_dir, exec_name, *args):
"""Execute the given python script with the given args and command line.
Args:
tool_dir: the directory the tool is located in
exec_name: additional path to the executable under the tool_dir
*args: args for the command
"""
py_path = None # Let execution_utils resolve the path.
# Gsutil allows users to set the desired Python interpreter using a separate
# environment variable, so as to allow users to run gsutil using Python 3
# without forcing the rest of the Cloud SDK to use Python 3 (as it would
# likely break at the time this comment was written).
extra_popen_kwargs = {}
if exec_name == 'gsutil':
gsutil_py = encoding.GetEncodedValue(os.environ,
'CLOUDSDK_GSUTIL_PYTHON')
# Since PY3, Python closes open FDs in child processes, since we need them
# open for completions to work we set the close_fds kwarg to Popen.
extra_popen_kwargs['close_fds'] = False
if gsutil_py:
py_path = gsutil_py
if exec_name == 'bq.py':
bq_py = encoding.GetEncodedValue(os.environ, 'CLOUDSDK_BQ_PYTHON')
if bq_py:
py_path = bq_py
if exec_name == 'dev_appserver.py':
if platforms.OperatingSystem.Current(
) != platforms.OperatingSystem.WINDOWS:
# Hard code the python to 'python2' which should exist on most Unix based
# systems.
py_path = 'python2'
else:
# On windows attempt to use the bundled PY2 component.
bundled_py2 = os.path.join(SDK_ROOT, 'platform', 'bundledpython2',
'python.exe')
if os.path.exists(bundled_py2):
py_path = bundled_py2
# Set this ENV var to explicitly select the python binary to use for
# dev_appserver.py
devapp_env_py = encoding.GetEncodedValue(
os.environ, 'CLOUDSDK_DEVAPPSERVER_PYTHON')
if devapp_env_py:
py_path = devapp_env_py
_ExecuteTool(
execution_utils.ArgsForPythonTool(_FullPath(tool_dir, exec_name),
*args,
python=py_path),
**extra_popen_kwargs)
def RestartCommand(command=None, python=None, block=True):
"""Calls command again with the same arguments as this invocation and exit.
Args:
command: str, the command to run (full path to Python file). If not
specified, defaults to current `gcloud` installation.
python: str or None, the path to the Python interpreter to use for the new
command invocation (if None, uses the current Python interpreter)
block: bool, whether to wait for the restarted command invocation to
terminate before continuing.
"""
command = command or os.path.join(os.path.dirname(googlecloudsdk.__file__),
'gcloud', 'gcloud.py')
command_args = sys.argv[1:]
args = execution_utils.ArgsForPythonTool(command,
*command_args,
python=python)
short_command = os.path.basename(command)
if short_command == 'gcloud.py':
short_command = 'gcloud'
log.status.Print('Restarting command:\n $ {command} {args}\n'.format(
command=short_command, args=' '.join(command_args)))
log.debug('Restarting command: %s %s', command, args)
log.out.flush()
log.err.flush()
if block:
execution_utils.Exec(args)
else:
if platforms.OperatingSystem.Current(
) is platforms.OperatingSystem.WINDOWS:
# Open in a new cmd window, and wait for the user to hit enter at the end.
# Otherwise, the output is either lost (without `pause`) or comes out
# asynchronously over the next commands (without the new window).
def Quote(s):
return '"' + s + '"'
args = 'cmd.exe /c "{0} && pause"'.format(' '.join(map(
Quote, args)))
subprocess.Popen(args,
shell=True,
**platforms.Platform.Current().AsyncPopenArgs())
def ExecutePythonTool(tool_dir, exec_name, *args):
"""Execute the given python script with the given args and command line.
Args:
tool_dir: the directory the tool is located in
exec_name: additional path to the executable under the tool_dir
*args: args for the command
"""
py_path = None # Let execution_utils resolve the path.
# Gsutil allows users to set the desired Python interpreter using a separate
# environment variable, so as to allow users to run gsutil using Python 3
# without forcing the rest of the Cloud SDK to use Python 3 (as it would
# likely break at the time this comment was written).
if exec_name == 'gsutil':
gsutil_py = encoding.GetEncodedValue(os.environ,
'CLOUDSDK_GSUTIL_PYTHON')
if gsutil_py:
py_path = gsutil_py
if exec_name == 'bq.py':
bq_py = encoding.GetEncodedValue(os.environ, 'CLOUDSDK_BQ_PYTHON')
if bq_py:
py_path = bq_py
if exec_name == 'dev_appserver.py':
# Hard code the python to 'python2' which should exist on most Unix based
# systems, on windows let python be discovered in execution_utils
if platforms.OperatingSystem.Current(
) != platforms.OperatingSystem.WINDOWS:
py_path = 'python2'
# Set this ENV var if python2 is not the name of the python2 binary on the
# system.
devapp_env_py = encoding.GetEncodedValue(
os.environ, 'CLOUDSDK_DEVAPPSERVER_PYTHON')
if devapp_env_py:
py_path = devapp_env_py
_ExecuteTool(
execution_utils.ArgsForPythonTool(_FullPath(tool_dir, exec_name),
*args,
python=py_path))
def ReportMetrics(self, wait_for_report=False):
"""Reports the collected metrics using a separate async process."""
if not self._metrics:
return
temp_metrics_file = tempfile.NamedTemporaryFile(delete=False)
with temp_metrics_file:
pickle.dump(self._metrics, temp_metrics_file)
self._metrics = []
this_file = encoding.Decode(__file__)
reporting_script_path = os.path.realpath(
os.path.join(os.path.dirname(this_file), 'metrics_reporter.py'))
execution_args = execution_utils.ArgsForPythonTool(
reporting_script_path, temp_metrics_file.name)
# On Python 2.x on Windows, the first arg can't be unicode. We encode
# encode it anyway because there is really nothing else we can do if
# that happens.
# https://bugs.python.org/issue19264
execution_args = [encoding.Encode(a) for a in execution_args]
exec_env = os.environ.copy()
encoding.SetEncodedValue(exec_env, 'PYTHONPATH',
os.pathsep.join(sys.path))
try:
p = subprocess.Popen(execution_args,
env=exec_env,
**self._async_popen_args)
log.debug('Metrics reporting process started...')
except OSError:
# This can happen specifically if the Python executable moves between the
# start of this process and now.
log.debug('Metrics reporting process failed to start.')
if wait_for_report:
# NOTE: p.wait() can cause a deadlock. p.communicate() is recommended.
# See python docs for more information.
p.communicate()
log.debug('Metrics reporting process finished.')
def ReportMetrics(self, wait_for_report=False):
"""Reports the collected metrics using a separate async process."""
if not self._metrics:
return
temp_metrics_file = tempfile.NamedTemporaryFile(delete=False)
with temp_metrics_file:
pickle.dump(self._metrics, temp_metrics_file)
self._metrics = []
this_file = encoding.Decode(__file__)
reporting_script_path = os.path.realpath(
os.path.join(os.path.dirname(this_file), 'metrics_reporter.py'))
execution_args = execution_utils.ArgsForPythonTool(
reporting_script_path, temp_metrics_file.name)
# popen is silly when it comes to non-ascii args. The executable has to be
# _unencoded_, while the rest of the args have to be _encoded_.
execution_args = execution_args[0:1] + [
console_attr.EncodeForConsole(a) for a in execution_args[1:]
]
exec_env = os.environ.copy()
exec_env['PYTHONPATH'] = os.pathsep.join(sys.path)
try:
p = subprocess.Popen(execution_args,
env=exec_env,
**self._async_popen_args)
log.debug('Metrics reporting process started...')
except OSError:
# This can happen specifically if the Python executable moves between the
# start of this process and now.
log.debug('Metrics reporting process failed to start.')
if wait_for_report:
# NOTE: p.wait() can cause a deadlock. p.communicate() is recommended.
# See python docs for more information.
p.communicate()
log.debug('Metrics reporting process finished.')
def ExecutePythonTool(tool_dir, exec_name, *args):
"""Execute the given python script with the given args and command line.
Args:
tool_dir: the directory the tool is located in
exec_name: additional path to the executable under the tool_dir
*args: args for the command
"""
py_path = None # Let execution_utils resolve the path.
# Gsutil allows users to set the desired Python interpreter using a separate
# environment variable, so as to allow users to run gsutil using Python 3
# without forcing the rest of the Cloud SDK to use Python 3 (as it would
# likely break at the time this comment was written).
if exec_name == 'gsutil':
gsutil_py = encoding.GetEncodedValue(os.environ,
'CLOUDSDK_GSUTIL_PYTHON')
if gsutil_py:
py_path = gsutil_py
_ExecuteTool(
execution_utils.ArgsForPythonTool(_FullPath(tool_dir, exec_name),
*args,
python=py_path))
def GetArgs(self):
return execution_utils.ArgsForPythonTool(self.setup_py_path,
*self.setup_py_args,
python=GetPythonExecutable())
def testArgsForPythonToolPython(self):
self.StartEnvPatch({}, clear=True)
self.assertEqual(
execution_utils.ArgsForPythonTool('foo.py',
python='/other/python'),
['/other/python', 'foo.py'])
def testArgsForPythonToolEnvironArgs(self):
self.StartEnvPatch({'CLOUDSDK_PYTHON_ARGS': '--bar'}, clear=True)
self.assertEqual(execution_utils.ArgsForPythonTool('foo.py'),
['/path/to/python', '--bar', 'foo.py'])
def testArgsForPythonToolArgs(self):
self.StartEnvPatch({}, clear=True)
self.assertEqual(execution_utils.ArgsForPythonTool('foo.py', 'a', 'b'),
['/path/to/python', 'foo.py', 'a', 'b'])
def GetArgsForLegacyScript(script_name, args, interpreter=None):
"""Returns a list of args for execution of the given script.
Args:
script_name: The name of the script to run. The extension determines which
interpreter will be used for execution. Python will be used for .py.
If there is no extension it is assumed it is just a binary file and is
executed directly. .exe files are executed directly on Windows, or
error raised if not on Windows. If it ends with a single dot, it sees
what OS it is running on and either executes .cmd or .sh.
args: The arguments for the script itself
interpreter: an interpreter to use rather than trying to derive it from
the extension name. The value is the same as an extension, right now
only implemented for 'py'.
Returns:
a list of args for the process executor
Raises:
ValueError: if the script type cannot be determined or is invalid for OS
"""
(_, tail) = os.path.splitext(script_name)
# No extension, just try to execute it
if not tail and not interpreter:
# In our tests, users generally just refer to 'gcloud'. But in windows,
# there is no 'gcloud' file, there is gcloud.cmd. This isn't an issue on the
# shell because the shell, given <name>, is smart enough to detect that
# there is a file <name>.cmd. subprocess.Popen is not smart enough to know
# this, however, unless shell=True is given. We'd like to avoid that if
# possible, however.
if _IsOnWindows() and script_name == 'gcloud':
script_name = 'gcloud.cmd'
return execution_utils.ArgsForExecutableTool(FullPath(script_name), *args)
# Strip the '.'
if interpreter:
ext = interpreter
else:
ext = tail[1:]
# Python, same across platforms
if ext == 'py':
return execution_utils.ArgsForPythonTool(FullPath(script_name), *args)
# .exe, windows only
if ext == 'exe':
if not _IsOnWindows():
raise ValueError('Extention for {0} is only valid for WINDOWS'.format(
script_name))
return execution_utils.ArgsForExecutableTool(FullPath(script_name), *args)
# ending in a '.'
if not ext:
if _IsOnWindows():
return execution_utils.ArgsForCMDTool(
FullPath(script_name + 'cmd'), *args)
else:
return execution_utils.ArgsForExecutableTool(
FullPath(script_name + 'sh'), *args)
else:
raise ValueError('Unknown script type: {0}'.format(script_name))
