def _GetToolEnv():
env = dict(os.environ)
encoding.SetEncodedValue(env, 'CLOUDSDK_WRAPPER', '1')
encoding.SetEncodedValue(env, 'CLOUDSDK_VERSION', config.CLOUD_SDK_VERSION)
encoding.SetEncodedValue(env, 'CLOUDSDK_PYTHON',
execution_utils.GetPythonExecutable())
return env
def GetPythonExecutable():
python_executable = None
try:
python_executable = execution_utils.GetPythonExecutable()
except ValueError:
raise SysExecutableMissingError()
return python_executable
def Run(self, args):
if util.IsPy2() and not args.IsSpecified('python_to_use'):
log.error('Virtual env support requires Python 3.')
raise exceptions.ExitCodeNoError(exit_code=3)
if util.IsWindows():
log.error('Virtual env support not enabled on Windows.')
raise exceptions.ExitCodeNoError(exit_code=4)
if args.IsSpecified('python_to_use'):
python = args.python_to_use
else:
try:
python = execution_utils.GetPythonExecutable()
except ValueError:
log.error('Failed to resolve python to use for virtual env.')
raise exceptions.ExitCodeNoError(exit_code=5)
ve_dir = config.Paths().virtualenv_dir
if util.VirtualEnvExists(ve_dir):
log.error('Virtual env setup {} already exists.'.format(ve_dir))
raise exceptions.ExitCodeNoError(exit_code=5)
succeeded_making_venv = False
try:
log.status.Print('Creating virtualenv...')
# python -m venv is preferred as it aligns the python used with
# the current in used Python.
ec = execution_utils.Exec([python, '-m', 'venv', ve_dir],
no_exit=True,
err_func=log.file_only_logger.debug,
out_func=log.file_only_logger.debug)
if ec != 0:
# Many linux vendors havea a history of having a broken python-venv
# package that will not work correctly, debian for example. If -m venv
# failed above we will attempt to use the virtualenv tool if it is
# installed and exists in $PATH.
ec = execution_utils.Exec(
['virtualenv', '-q', '-p', python, ve_dir], no_exit=True)
if ec != 0:
log.error('Virtual env setup failed.')
raise exceptions.ExitCodeNoError(exit_code=ec)
log.status.Print('Installing modules...')
install_modules = [
'{}/bin/pip3'.format(ve_dir), 'install', '--log',
'{}/install_module.log'.format(ve_dir), '-q',
'--disable-pip-version-check'
]
install_modules.extend(util.MODULES)
ec = execution_utils.Exec(install_modules, no_exit=True)
if ec == 0:
# prevent the cleanup that occurs in finally block
succeeded_making_venv = True
else:
log.error('Virtual env setup failed.')
raise exceptions.ExitCodeNoError(exit_code=ec)
finally:
# If something went wrong we clean up any partial created ve_dir
if not succeeded_making_venv:
if util.VirtualEnvExists(ve_dir):
files.RmTree(ve_dir)
def MakeProcess(module_name,
package_root,
args=None,
cluster=None,
task_type=None,
index=None,
**extra_popen_args):
"""Make a Popen object that runs the module, with the correct env.
If task_type is 'master' instead replaces the current process with the
subprocess via execution_utils.Exec
Args:
module_name: str. Name of the module to run, e.g. trainer.task
package_root: str. Absolute path to the package root for the module.
used as CWD for the subprocess.
args: [str]. Additional user args. Any relative paths will not work.
cluster: dict. Cluster configuration dictionary. Suitable for passing to
tf.train.ClusterSpec.
task_type: str. Task type of this process. Only relevant if cluster is
specified.
index: int. Task index of this process.
**extra_popen_args: extra args passed to Popen. Used for testing.
Returns:
a subprocess.Popen object corresponding to the subprocesses or an int
corresponding to the return value of the subprocess
(if task_type is 'master')
"""
if args is None:
args = []
python = execution_utils.GetPythonExecutable()
cmd = [python, '-m', module_name] + args
config = {
'job': {'job_name': module_name, 'args': args},
'task': {'type': task_type, 'index': index} if cluster else {},
'cluster': cluster or {},
'environment': 'cloud'
}
log.info(('launching training process:\n'
'command: {cmd}\n config: {config}').format(
cmd=' '.join(cmd),
config=json.dumps(config, indent=2, sort_keys=True)))
env = os.environ.copy()
# the tf_config environment variable is used to pass the tensorflow
# configuration options to the training module. the module specific
# arguments are passed as comand line arguments.
env['TF_CONFIG'] = json.dumps(config)
if task_type == 'master':
return execution_utils.Exec(
cmd, env=env, no_exit=True, cwd=package_root, **extra_popen_args)
else:
task = subprocess.Popen(
cmd,
env=env,
cwd=package_root,
**extra_popen_args
)
atexit.register(execution_utils.KillSubprocess, task)
return task
def GetPrimaryNodeName():
"""Get the primary node name.
Returns:
str, the name of the primary node. If running in tensorflow 1.x,
return 'master'. If running in tensorflow 2.x, return 'chief'.
If tensorflow is not installed in local envrionment, it will return
the default name 'chief'.
Raises:
ValueError: if there is no python executable on the user system thrown by
execution_utils.GetPythonExecutable.
"""
exe_override = properties.VALUES.ml_engine.local_python.Get()
python_executable = (
exe_override or files.FindExecutableOnPath('python') or
execution_utils.GetPythonExecutable())
cmd = [python_executable,
'-c',
'import tensorflow as tf; print(tf.version.VERSION)']
with files.FileWriter(os.devnull) as f:
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=f)
return_code = proc.wait()
if return_code != 0:
log.warning('''
Cannot import tensorflow under path {}. Using "chief" for cluster setting.
If this is not intended, Please check if tensorflow is installed. Please also
verify if the python path used is correct. If not, to change the python path:
use `gcloud config set ml_engine/local_python $python_path`
Eg: gcloud config set ml_engine/local_python /usr/bin/python3'''.format(
python_executable))
return 'chief'
tf_version = proc.stdout.read()
if 'decode' in dir(tf_version):
tf_version = tf_version.decode('utf-8')
if tf_version.startswith('1.'):
return 'master'
elif tf_version.startswith('2.'):
return 'chief'
log.warning(
'Unexpected tensorflow version {}, using the default primary'
' node name, aka "chief" for cluster settings'.format(tf_version))
return 'chief'
def GetPythonExecutable(self):
return execution_utils.GetPythonExecutable()
def _GetToolEnv(): env = dict(os.environ) env['CLOUDSDK_WRAPPER'] = '1' env['CLOUDSDK_VERSION'] = config.CLOUD_SDK_VERSION env['CLOUDSDK_PYTHON'] = execution_utils.GetPythonExecutable() return env
def testGetPythonExecutableNoEnvironment(self):
self.StartEnvPatch({}, clear=True)
self.StartObjectPatch(sys, 'executable', new='/current/python')
self.assertEqual(execution_utils.GetPythonExecutable(),
'/current/python')
def testGetPythonExecutableEnvironment(self):
self.StartEnvPatch({'CLOUDSDK_PYTHON': '/env/python'}, clear=True)
self.assertEqual(execution_utils.GetPythonExecutable(), '/env/python')
def RunPlugin(self,
section_name,
plugin_spec,
params,
args=None,
valid_exit_codes=(0, ),
runtime_data=None):
"""Run a plugin.
Args:
section_name: (str) Name of the config section that the plugin spec is
from.
plugin_spec: ({str: str, ...}) A dictionary mapping plugin locales to
script names
params: (fingerprinting.Params or None) Parameters for the plugin.
args: ([str, ...] or None) Command line arguments for the plugin.
valid_exit_codes: (int, ...) Exit codes that will be accepted without
raising an exception.
runtime_data: ({str: object, ...}) A dictionary of runtime data passed
back from detect.
Returns:
(PluginResult) A bundle of the exit code and data produced by the plugin.
Raises:
PluginInvocationFailed: The plugin terminated with a non-zero exit code.
"""
# TODO(user): Support other script types.
if plugin_spec.has_key('python'):
normalized_path = _NormalizePath(self.root, plugin_spec['python'])
# We're sharing 'result' with the output collection thread, we can get
# away with this without locking because we pass it into the thread at
# creation and do not use it again until after we've joined the thread.
result = PluginResult()
p = subprocess.Popen(
[execution_utils.GetPythonExecutable(), normalized_path] +
(args if args else []),
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
stderr_thread = threading.Thread(target=self._ProcessPluginStderr,
args=(
section_name,
p.stderr,
))
stderr_thread.start()
stdout_thread = threading.Thread(target=self._ProcessPluginPipes,
args=(section_name, p, result,
params, runtime_data))
stdout_thread.start()
stderr_thread.join()
stdout_thread.join()
exit_code = p.wait()
result.exit_code = exit_code
if exit_code not in valid_exit_codes:
raise PluginInvocationFailed(
'Failed during execution of plugin %s '
'for section %s of runtime %s. rc = %s' %
(normalized_path, section_name,
self.config.get('name', 'unknown'), exit_code))
return result
else:
log.error('No usable plugin type found for %s' % section_name)
