def Run(self, args): """Run the configure-docker command.""" if not file_utils.SearchForExecutableOnPath( 'docker-credential-gcloud'): log.warning( '`docker-credential-gcloud` not in system PATH.\n' 'gcloud\'s Docker credential helper can be configured but ' 'it will not work until this is corrected.') current_config = cred_utils.Configuration.ReadFromDisk() if file_utils.SearchForExecutableOnPath('docker'): if not current_config.SupportsRegistryHelpers(): raise ConfigureDockerError( 'Invalid Docker version: The version of your Docker client is ' '[{}]; version [{}] or higher is required to support Docker ' 'credential helpers.'.format( current_config.DockerVersion(), cred_utils.MIN_DOCKER_CONFIG_HELPER_VERSION)) else: log.warning( '`docker` not in system PATH.\n' '`docker` and `docker-credential-gcloud` need to be in the same PATH ' 'in order to work correctly together.\n' 'gcloud\'s Docker credential helper can be configured but ' 'it will not work until this is corrected.') current_helpers = current_config.GetRegisteredCredentialHelpers() new_helpers = cred_utils.GetGcloudCredentialHelperConfig() if new_helpers == current_helpers: log.status.Print('gcloud credential helpers ' 'already registered correctly.') return if current_helpers: log.warning( 'Your config file at [{0}] contains these credential helper ' 'entries:\n\n{1}\nThese will be overwritten.'.format( current_config.path, json.dumps(current_helpers, indent=2))) console_io.PromptContinue( message='The following settings will be added to your Docker ' 'config file located at [{0}]:\n {1}'.format( current_config.path, json.dumps(new_helpers, indent=2)), cancel_on_no=True) current_config.RegisterCredentialHelpers() log.status.Print('Docker configuration file updated.')
def __init__(self): self.sdk_root = config.Paths().sdk_root self.release_channel = config.INSTALLATION_CONFIG.release_channel self.repo_url = config.INSTALLATION_CONFIG.snapshot_url repos = properties.VALUES.component_manager.additional_repositories.Get( validate=False) self.additional_repos = repos.split(',') if repos else [] # Keep it as array for structured output. self.path = encoding.GetEncodedValue(os.environ, 'PATH', '').split(os.pathsep) self.python_path = [ encoding.Decode(path_elem) for path_elem in sys.path ] if self.sdk_root: manager = update_manager.UpdateManager() self.components = manager.GetCurrentVersionsInformation() self.old_tool_paths = manager.FindAllOldToolsOnPath() self.duplicate_tool_paths = manager.FindAllDuplicateToolsOnPath() paths = [os.path.realpath(p) for p in self.path] this_path = os.path.realpath( os.path.join(self.sdk_root, update_manager.UpdateManager.BIN_DIR_NAME)) # TODO(b/36055867): Validate symlinks in /usr/local/bin when we start # creating them. self.on_path = this_path in paths else: self.components = {} self.old_tool_paths = [] self.duplicate_tool_paths = [] self.on_path = False self.kubectl = file_utils.SearchForExecutableOnPath('kubectl') if self.kubectl: self.kubectl = self.kubectl[0]
def FindAllOldToolsOnPath(self, path=None): """Searches the PATH for any old Cloud SDK tools. Args: path: str, A path to use instead of the PATH environment variable. Returns: {str}, The old executable paths. """ bin_dir = os.path.realpath( os.path.join(self.__sdk_root, UpdateManager.BIN_DIR_NAME)) bad_commands = set() if not os.path.exists(bin_dir): return bad_commands commands = [ f for f in os.listdir(bin_dir) if os.path.isfile(os.path.join(bin_dir, f)) and not f.startswith('.') ] for command in commands: existing_paths = file_utils.SearchForExecutableOnPath(command, path=path) if existing_paths: this_tool = os.path.join(bin_dir, command) bad_commands.update( set(os.path.realpath(f) for f in existing_paths) - set([this_tool])) return bad_commands
def RunPredict(model_dir, json_instances=None, text_instances=None, framework='tensorflow', signature_name=None): """Run ML Engine local prediction.""" instances = predict_utilities.ReadInstancesFromArgs(json_instances, text_instances) sdk_root = config.Paths().sdk_root if not sdk_root: raise LocalPredictEnvironmentError( 'You must be running an installed Cloud SDK to perform local ' 'prediction.') # Inheriting the environment preserves important variables in the child # process. In particular, LD_LIBRARY_PATH under linux and PATH under windows # could be used to point to non-standard install locations of CUDA and CUDNN. # If not inherited, the child process could fail to initialize Tensorflow. env = os.environ.copy() env['CLOUDSDK_ROOT'] = sdk_root # We want to use whatever the user's Python was, before the Cloud SDK started # changing the PATH. That's where Tensorflow is installed. python_executables = files.SearchForExecutableOnPath('python') # Need to ensure that ml_sdk is in PYTHONPATH for the import in # local_predict to succeed. orig_py_path = ':' + env.get('PYTHONPATH') if env.get('PYTHONPATH') else '' env['PYTHONPATH'] = (os.path.join(sdk_root, 'lib', 'third_party', 'ml_sdk') + orig_py_path) if not python_executables: # This doesn't have to be actionable because things are probably beyond help # at this point. raise LocalPredictEnvironmentError( 'Something has gone really wrong; we can\'t find a valid Python ' 'executable on your PATH.') # Use python found on PATH or local_python override if set python_executable = (properties.VALUES.ml_engine.local_python.Get() or python_executables[0]) predict_args = ['--model-dir', model_dir, '--framework', framework] if signature_name: predict_args += ['--signature-name', signature_name] # Start local prediction in a subprocess. proc = subprocess.Popen( [python_executable, local_predict.__file__] + predict_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) # Pass the instances to the process that actually runs local prediction. for instance in instances: proc.stdin.write(json.dumps(instance) + '\n') proc.stdin.flush() # Get the results for the local prediction. output, err = proc.communicate() if proc.returncode != 0: raise LocalPredictRuntimeError(err) if err: log.warning(err) try: return json.loads(output) except ValueError: raise InvalidReturnValueError('The output for prediction is not ' 'in JSON format: ' + output)
def RunPredict(model_dir, json_instances=None, text_instances=None): """Run ML Engine local prediction.""" instances = predict_utilities.ReadInstancesFromArgs(json_instances, text_instances) sdk_root = config.Paths().sdk_root if not sdk_root: raise LocalPredictEnvironmentError( 'You must be running an installed Cloud SDK to perform local ' 'prediction.') # Inheriting the environment preserves important variables in the child # process. In particular, LD_LIBRARY_PATH under linux and PATH under windows # could be used to point to non-standard install locations of CUDA and CUDNN. # If not inherited, the child process could fail to initialize Tensorflow. env = os.environ.copy() env['CLOUDSDK_ROOT'] = sdk_root # We want to use whatever the user's Python was, before the Cloud SDK started # changing the PATH. That's where Tensorflow is installed. python_executables = files.SearchForExecutableOnPath('python') if not python_executables: # This doesn't have to be actionable because things are probably beyond help # at this point. raise LocalPredictEnvironmentError( 'Something has gone really wrong; we can\'t find a valid Python ' 'executable on your PATH.') python_executable = python_executables[0] # Start local prediction in a subprocess. proc = subprocess.Popen( [python_executable, local_predict.__file__, '--model-dir', model_dir], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) # Pass the instances to the process that actually runs local prediction. for instance in instances: proc.stdin.write(json.dumps(instance) + '\n') proc.stdin.flush() # Get the results for the local prediction. output, err = proc.communicate() if proc.returncode != 0: raise LocalPredictRuntimeError(err) if err: log.warn(err) try: return json.loads(output) except ValueError: raise InvalidReturnValueError('The output for prediction is not ' 'in JSON format: ' + output)
def _RunPredict(args): """Run ML Engine local prediction.""" instances = predict_utilities.ReadInstancesFromArgs( args.json_instances, args.text_instances) sdk_root = config.Paths().sdk_root if not sdk_root: raise LocalPredictEnvironmentError( 'You must be running an installed Cloud SDK to perform local ' 'prediction.') env = {'CLOUDSDK_ROOT': sdk_root} # We want to use whatever the user's Python was, before the Cloud SDK started # changing the PATH. That's where Tensorflow is installed. python_executables = files.SearchForExecutableOnPath('python') if not python_executables: # This doesn't have to be actionable because things are probably beyond help # at this point. raise LocalPredictEnvironmentError( 'Something has gone really wrong; we can\'t find a valid Python ' 'executable on your PATH.') python_executable = python_executables[0] # Start local prediction in a subprocess. proc = subprocess.Popen([ python_executable, local_predict.__file__, '--model-dir', args.model_dir ], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) # Pass the instances to the process that actually runs local prediction. for instance in instances: proc.stdin.write(json.dumps(instance) + '\n') proc.stdin.flush() # Get the results for the local prediction. output, err = proc.communicate() if proc.returncode != 0: raise LocalPredictRuntimeError(err) if err: log.warn(err) try: return json.loads(output) except ValueError: raise InvalidReturnValueError('The output for prediction is not ' 'in JSON format: ' + output)
def __init__(self, anonymizer=None): anonymizer = anonymizer or NoopAnonymizer() self.sdk_root = anonymizer.ProcessPath(config.Paths().sdk_root) self.release_channel = config.INSTALLATION_CONFIG.release_channel self.repo_url = anonymizer.ProcessURL( config.INSTALLATION_CONFIG.snapshot_url) repos = properties.VALUES.component_manager.additional_repositories.Get( validate=False) self.additional_repos = (map(anonymizer.ProcessURL, repos.split(',')) if repos else []) # Keep it as array for structured output. path = encoding.GetEncodedValue(os.environ, 'PATH', '').split(os.pathsep) self.python_path = [ anonymizer.ProcessPath(encoding.Decode(path_elem)) for path_elem in sys.path ] if self.sdk_root: manager = update_manager.UpdateManager() self.components = manager.GetCurrentVersionsInformation() self.other_tool_paths = [ anonymizer.ProcessPath(p) for p in manager.FindAllOtherToolsOnPath() ] self.duplicate_tool_paths = [ anonymizer.ProcessPath(p) for p in manager.FindAllDuplicateToolsOnPath() ] paths = [os.path.realpath(p) for p in path] this_path = os.path.realpath( os.path.join(self.sdk_root, update_manager.UpdateManager.BIN_DIR_NAME)) self.on_path = this_path in paths else: self.components = {} self.other_tool_paths = [] self.duplicate_tool_paths = [] self.on_path = False self.path = [anonymizer.ProcessPath(p) for p in path] self.kubectl = file_utils.SearchForExecutableOnPath('kubectl') if self.kubectl: self.kubectl = anonymizer.ProcessPath(self.kubectl[0])
def _VerifyLibIsInstalled(lib_name): """Checks whether a python library (module) needed for a test is installed. Args: lib_name: name of the library, e.g. `tensorflow`. Returns: A tuple for test skip decorators, consisting of two elements: - a boolean value to indicate whether the test should be skipped - a string with the reason for the skip, if any """ python_executables = files.SearchForExecutableOnPath('python') if not python_executables: return False, 'No python executable available' python_executable = python_executables[0] command = [python_executable, '-c', 'import {}'.format(lib_name)] proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = proc.communicate() if proc.returncode: # Something went wrong during module import return ( False, ( 'Could not verify {lib} install.\n' 'Python location: {python}\n' 'Command to test: {command}\n' '----------------stdout----------------\n' '{stdout}' '----------------stderr----------------' '{stderr}'.format(lib=lib_name, python=python_executable, command=command, stdout=stdout, stderr=stderr) ) ) return True, ''
def DockerExists(self): return file_utils.SearchForExecutableOnPath( 'docker') or file_utils.SearchForExecutableOnPath('docker.exe')
def DockerCredentialGcloudExists(self): return file_utils.SearchForExecutableOnPath( 'docker-credential-gcloud' ) or file_utils.SearchForExecutableOnPath( 'docker-credential-gcloud.cmd')