def WriteFileContents(path,
contents,
overwrite=True,
private=False,
create_path=True):
"""Writes the given text contents to a file at the given path.
Args:
path: str, The file path to write to.
contents: str, The text string to write.
overwrite: bool, False to error out if the file already exists.
private: bool, True to make the file have 0o600 permissions.
create_path: bool, True to create intermediate directories, if needed.
Raises:
Error: If the file cannot be written.
"""
try:
_CheckOverwrite(path, overwrite)
with FileWriter(path, private=private, create_path=create_path) as f:
# This decode is here because a lot of libraries on Python 2 can return
# both text or bytes depending on if unicode is present. If you truly
# pass binary data to this, the decode will fail (as it should). If you
# pass an ascii string (that you got from json.dumps() for example), this
# will prevent it from crashing.
f.write(encoding_util.Decode(contents))
except EnvironmentError as e:
# EnvironmentError is parent of IOError, OSError and WindowsError.
# Raised when file does not exist or can't be opened/read.
raise Error('Unable to write file [{0}]: {1}'.format(path, e))
def ReportMetrics(self, wait_for_report=False):
"""Reports the collected metrics using a separate async process."""
if not self._metrics:
return
temp_metrics_file = tempfile.NamedTemporaryFile(delete=False)
with temp_metrics_file:
pickle.dump(self._metrics, temp_metrics_file)
self._metrics = []
this_file = encoding.Decode(__file__)
reporting_script_path = os.path.realpath(
os.path.join(os.path.dirname(this_file), 'metrics_reporter.py'))
execution_args = execution_utils.ArgsForPythonTool(
reporting_script_path, temp_metrics_file.name)
exec_env = os.environ.copy()
exec_env['PYTHONPATH'] = os.pathsep.join(sys.path)
try:
p = subprocess.Popen(execution_args, env=exec_env,
**self._async_popen_args)
log.debug('Metrics reporting process started...')
except OSError:
# This can happen specifically if the Python executable moves between the
# start of this process and now.
log.debug('Metrics reporting process failed to start.')
if wait_for_report:
# NOTE: p.wait() can cause a deadlock. p.communicate() is recommended.
# See python docs for more information.
p.communicate()
log.debug('Metrics reporting process finished.')
def GenerateConnectAgentManifest(self, option):
"""Generate the YAML manifest to deploy the GKE Connect agent.
Args:
option: an instance of ConnectAgentOption.
Returns:
A slice of connect agent manifest resources.
Raises:
Error: if the API call to generate connect agent manifest failed.
"""
# Can't directly use the generated API client given that it currently
# doesn't support nested messages. See the discussion here:
# https://groups.google.com/a/google.com/forum/#!msg/cloud-sdk-eng/hwdwUTEmvlw/fRdrvK26AAAJ
query_params = [('connectAgent.namespace', option.namespace),
('connectAgent.proxy', option.proxy),
('isUpgrade', option.is_upgrade),
('version', option.version),
('registry', option.registry),
('image_pull_secret_content',
option.image_pull_secret_content)]
base_url = self.client.url
url = '{}/{}/{}:generateConnectManifest?{}'.format(
base_url, self.api_version, option.membership_ref,
urllib.parse.urlencode(query_params))
response, raw_content = http.Http().request(uri=url)
content = core_encoding.Decode(raw_content)
status_code = response.get('status')
if status_code != '200':
msg = self._HTTP_ERROR_FORMAT.format(status_code, content)
raise exceptions.HttpException(msg)
return json.loads(content).get('manifest')
def testDescribeRegional(self):
self.mock_client.regionInstanceGroupManagers.Get.Expect(
messages.ComputeRegionInstanceGroupManagersGetRequest(
instanceGroupManager='group-1',
project='fake-project',
region='region-1'),
test_resources.MakeInstanceGroupManagers(
API_VERSION, scope_type='region', scope_name='region-1')[0]
)
self._MockAutoscalerRequest([])
self.Run("""
compute instance-groups managed describe group-1 --region region-1
""")
self.assertMultiLineEqual(
encoding.Decode(self.stdout.getvalue()),
textwrap.dedent("""\
baseInstanceName: test-instance-name-1
creationTimestamp: '2013-09-06T17:54:10.636-07:00'
description: Test description.
fingerprint: MTIzNA==
instanceGroup: {0}/projects/my-project/regions/region-1/instanceGroups/group-1
instanceTemplate: {0}/projects/my-project/global/instanceTemplates/template-1
name: group-1
region: {0}/projects/my-project/regions/region-1
selfLink: {0}/projects/my-project/regions/region-1/instanceGroupManagers/group-1
targetSize: 1
""".format(self.compute_uri)))
def Resource(*args):
"""Gets the path to a resource under googlecloudsdk."""
# Make sure to find parent of tests directory, without assuming it is
# googlecloudsdk, as tests package can be remapped to different location.
return os.path.join(
os.path.dirname(os.path.dirname(encoding.Decode(tests.__file__))),
*args)
def GetHttpErrorMessage(error):
"""Returns a human readable string representation from the http response.
Args:
error: HttpException representing the error response.
Returns:
A human readable string representation of the error.
"""
status = error.response.status
code = error.response.reason
message = ''
try:
data = json.loads(error.content)
if 'error' in data:
error_info = data['error']
if 'message' in error_info:
message = error_info['message']
violations = _GetViolationsFromError(error)
if violations:
message += '\nProblems:\n' + violations
if status == 403:
permission_issues = _GetPermissionErrorDetails(error_info)
if permission_issues:
message += '\nPermission Details:\n' + permission_issues
except (ValueError, TypeError):
message = error.content
return 'ResponseError: status=[{0}], code=[{1}], message=[{2}]'.format(
status, code, encoding.Decode(message))
def __exit__(self, prev_exc_type, prev_exc_val, prev_exc_trace):
try:
self.Close()
except: # pylint: disable=bare-except
if not prev_exc_type:
raise
message = (
'Got exception {0}'
'while another exception was active {1} [{2}]'
.format(
encoding_util.Decode(traceback.format_exc()),
prev_exc_type,
encoding_util.Decode(prev_exc_val)))
exceptions.reraise(prev_exc_type(message), tb=prev_exc_trace)
# always return False so any exceptions will be re-raised
return False
def _Stringize(value):
"""Returns the unicode string representation for value."""
if value is None:
return u'null'
if not isinstance(value, basestring):
value = repr(value)
return unicode(encoding.Decode(value))
def _WaitForOperation(client, operation, resource_type):
"""Waits for an operation to complete.
Args:
client: GAPIC API client, client used to make requests.
operation: run_apps.v1alpha1.operation, object to wait for.
resource_type: type, the expected type of resource response
Returns:
The resulting resource of input paramater resource_type.
"""
poller = waiter.CloudOperationPoller(resource_type,
client.projects_locations_operations)
operation_ref = resources.REGISTRY.ParseRelativeName(
operation.name,
collection='{}.projects.locations.operations'.format(API_NAME))
try:
return poller.GetResult(waiter.PollUntilDone(
poller,
operation_ref,
max_wait_ms=_POLLING_TIMEOUT_MS,
wait_ceiling_ms=_RETRY_TIMEOUT_MS))
except waiter.OperationError:
operation = poller.Poll(operation_ref)
raise exceptions.IntegrationsOperationError(
'OperationError: code={0}, message={1}'.format(
operation.error.code, encoding.Decode(operation.error.message)))
def PrivatizeFile(path):
"""Makes an existing file private or creates a new, empty private file.
In theory it would be better to return the open file descriptor so that it
could be used directly. The issue that we would need to pass an encoding to
os.fdopen() and on Python 2. This is not supported. Instead we just create
the empty file and then we will just open it normally later to do the write.
Args:
path: str, The path of the file to create or privatize.
"""
try:
if os.path.exists(path):
os.chmod(path, 0o600)
else:
parent_dir_path, _ = os.path.split(path)
full_parent_dir_path = encoding_util.Decode(
os.path.realpath(os.path.expanduser(parent_dir_path)))
MakeDir(full_parent_dir_path, mode=0o700)
flags = os.O_RDWR | os.O_CREAT | os.O_TRUNC
# Accommodate Windows; stolen from python2.6/tempfile.py.
if hasattr(os, 'O_NOINHERIT'):
flags |= os.O_NOINHERIT
fd = os.open(path, flags, 0o600)
os.close(fd)
except EnvironmentError as e:
# EnvironmentError is parent of IOError, OSError and WindowsError.
# Raised when file does not exist or can't be opened/read.
raise Error('Unable to create private file [{0}]: {1}'.format(path, e))
def OpenForWritingPrivate(path, binary=False):
"""Open a file for writing, with the right permissions for user-private files.
Args:
path: str, The full path to the file.
binary: bool, If true forces binary mode, this only affects Windows.
Returns:
A file context manager.
"""
parent_dir_path, _ = os.path.split(path)
full_parent_dir_path = encoding.Decode(
os.path.realpath(os.path.expanduser(parent_dir_path)))
MakeDir(full_parent_dir_path, mode=0o700)
flags = os.O_RDWR | os.O_CREAT | os.O_TRUNC
# Accommodate Windows; stolen from python2.6/tempfile.py.
if hasattr(os, 'O_NOINHERIT'):
flags |= os.O_NOINHERIT
if binary:
flags |= os.O_BINARY
fd = os.open(path, flags, 0o600)
return os.fdopen(fd, 'w')
def RunParserCompletion(self, parser, command, expected):
"""Runs arcomplete.autocomplete on parser+command, avoiding CLI baggage."""
os.environ['_ARGCOMPLETE'] = '1'
os.environ['_ARGCOMPLETE_IFS'] = '\t'
os.environ['_ARGCOMPLETE_TRACE'] = 'info'
os.environ['COMP_LINE'] = command
os.environ['COMP_POINT'] = str(len(command))
if '_ARC_DEBUG' in os.environ:
del os.environ['_ARC_DEBUG']
def MockExit(*unused_args, **unused_kwargs):
raise StopExecutionException()
try:
output_stream = io.BytesIO()
with self.assertRaises(StopExecutionException):
# pylint: disable=protected-access
cli._ArgComplete(parser,
exit_method=MockExit,
output_stream=output_stream)
raw_output = encoding.Decode(output_stream.getvalue())
completions = [s.strip() for s in raw_output.split('\t')]
self.assertEqual(set(expected),
set(completions),
msg='Completions did not match.')
finally:
del os.environ['_ARGCOMPLETE']
del os.environ['_ARGCOMPLETE_IFS']
del os.environ['_ARGCOMPLETE_TRACE']
del os.environ['COMP_LINE']
del os.environ['COMP_POINT']
def _Stringize(value):
"""Returns the unicode string representation for value."""
if value is None:
return 'null'
if not isinstance(value, six.string_types):
value = repr(value)
return six.text_type(encoding.Decode(value))
def FindDirectoryContaining(starting_dir_path, directory_entry_name):
"""Searches directories upwards until it finds one with the given contents.
This can be used to find the directory above you that contains the given
entry. It is useful for things like finding the workspace root you are under
that contains a configuration directory.
Args:
starting_dir_path: str, The path of the directory to start searching
upwards from.
directory_entry_name: str, The name of the directory that must be present
in order to return the current directory.
Returns:
str, The full path to the directory above the starting dir that contains the
given entry, or None if the root of the file system was hit without finding
it.
"""
prev_path = None
path = encoding_util.Decode(os.path.realpath(starting_dir_path))
while path != prev_path:
search_dir = os.path.join(path, directory_entry_name)
if os.path.isdir(search_dir):
return path
prev_path = path
path, _ = os.path.split(path)
return None
def _GetEncryptedPasswordFromSerialPort(self, client, instance_ref,
search_modulus):
"""Returns the decrypted password from the data in the serial port."""
encrypted_password_data = {}
start_time = time_util.CurrentTimeSec()
count = 1
agent_ready = False
while not encrypted_password_data:
log.debug('Get Serial Port Output, Try {0}'.format(count))
if (time_util.CurrentTimeSec() >
(start_time + WINDOWS_PASSWORD_TIMEOUT_SEC)):
raise utils.TimeoutError(
TIMEOUT_ERROR.format(time_util.CurrentDatetimeUtc()))
serial_port_output = self._GetSerialPortOutput(client,
instance_ref,
port=4).split('\n')
for line in reversed(serial_port_output):
try:
encrypted_password_dict = json.loads(line)
# Sometimes the serial port output only contains a partial entry.
except ValueError:
continue
modulus = encrypted_password_dict.get('modulus')
if modulus or encrypted_password_dict.get('ready'):
agent_ready = True
# Ignore any output that doesn't contain an encrypted password.
if not encrypted_password_dict.get('encryptedPassword'):
continue
if (core_encoding.Decode(search_modulus) ==
core_encoding.Decode(modulus)):
encrypted_password_data = encrypted_password_dict
break
if not agent_ready:
if self.old_metadata_keys:
message = OLD_WINDOWS_BUILD_ERROR.format(
instance_ref.instance, instance_ref.zone)
raise utils.WrongInstanceTypeError(message)
else:
message = NOT_READY_ERROR
raise utils.InstanceNotReadyError(message)
time_util.Sleep(POLLING_SEC)
count += 1
encrypted_password = encrypted_password_data['encryptedPassword']
return encrypted_password
def _AddRecord(self, record, delimit=False):
"""Prints the current record as CSV.
Printer attributes:
noheading: bool, Disable the initial key name heading record.
Args:
record: A list of JSON-serializable object columns.
delimit: bool, Print resource delimiters -- ignored.
Raises:
ToolException: A data value has a type error.
"""
# The CSV heading has three states:
# 1: No heading, used by ValuePrinter and CSV when 2. and 3. are empty.
# 2: Heading via AddHeading().
# 3: Default heading from format labels, if specified.
if not self._heading_printed:
self._heading_printed = True
if 'no-heading' not in self.attributes:
if self._heading:
labels = self._heading
else:
labels = self.column_attributes.Labels()
if labels:
labels = [x.lower() for x in labels]
if labels:
self._out.write(self._separator.join(
[self._QuoteField(label) for label in labels]) + self._terminator)
line = []
for col in record:
if col is None:
val = ''
elif isinstance(col, dict):
val = self._delimiter.join(
[self._QuoteField(u'{0}={1}'.format(
encoding.Decode(k), encoding.Decode(v)))
for k, v in sorted(col.iteritems())])
elif isinstance(col, list):
val = self._delimiter.join(
[self._QuoteField(encoding.Decode(x)) for x in col])
elif isinstance(col, float):
val = self._QuoteField(resource_transform.TransformFloat(col))
else:
val = self._QuoteField(encoding.Decode(col))
line.append(val)
self._out.write(self._separator.join(line) + self._terminator)
def WriteNewDockerConfig(self, full_cfg):
new_cfg, unused_true = client_lib.GetDockerConfigPath(force_new=True)
directory = os.path.dirname(new_cfg)
if not os.path.exists(directory):
os.makedirs(directory)
files.WriteFileContents(new_cfg,
encoding.Decode(json.dumps(full_cfg)),
private=True)
def testDecodeCp500DefaultEncoding(self):
"""This is a mutant killer."""
# We have to make sure getfilesystemencoding fails on all systems.
self.StartObjectPatch(sys, 'getfilesystemencoding', return_value='ascii')
self.StartObjectPatch(sys, 'getdefaultencoding', return_value='cp500')
expected = _CP500.decode('cp500')
actual = encoding.Decode(_CP500)
self.assertEqual(expected, actual)
def _GetSurveyContentDirectory():
"""Get the directory containing all surveys in yaml format.
Returns:
Path to the surveys directory, i.e.
$CLOUDSDKROOT/lib/googlecloudsdk/command_lib/survey/contents
"""
return os.path.join(os.path.dirname(encoding.Decode(__file__)), 'contents')
def _GetUserHomeDir():
if platforms.OperatingSystem.Current(
) == platforms.OperatingSystem.WINDOWS:
# %HOME% has precedence over %USERPROFILE% for files.GetHomeDir().
# The Docker config resides under %USERPROFILE% on Windows
return encoding.Decode(os.path.expandvars('%USERPROFILE%'))
else:
return files.GetHomeDir()
def testDecodeCp500FileSystemEncoding(self):
"""This is a mutant killer."""
self.StartObjectPatch(sys,
'getfilesystemencoding',
return_value='cp500')
expected = _CP500.decode('cp500')
actual = encoding.Decode(_CP500)
self.assertEqual(expected, actual)
def Poll(self, is_last=False):
"""Poll the GCS object and print any new bytes to the console.
Args:
is_last: True if this is the last poll operation.
Raises:
api_exceptions.HttpError: if there is trouble connecting to GCS.
"""
(res, body) = self.http.request(
self.url, method='GET',
headers={'Range': 'bytes={0}-'.format(self.cursor)})
if res.status == 404: # Not Found
# Logfile hasn't been written yet (ie, build hasn't started).
log.debug('Reading GCS logfile: 404 (no log yet; keep polling)')
return
if res.status == 416: # Requested Range Not Satisfiable
# We have consumed all available data. We'll get this a lot as we poll.
log.debug('Reading GCS logfile: 416 (no new content; keep polling)')
if is_last:
self._PrintLastLine()
return
if res.status == 206 or res.status == 200: # Partial Content
# New content available. Print it!
log.debug('Reading GCS logfile: {code} (read {count} bytes)'.format(
code=res.status,
count=len(body)))
if self.cursor == 0:
self._PrintFirstLine()
self.cursor += len(body)
body = encoding.Decode(body)
self._PrintLogLine(body.rstrip('\n'))
if is_last:
self._PrintLastLine()
return
# For 429/503, there isn't much to do other than retry on the next poll.
# If we get a 429 after the build has completed, the user may get incomplete
# logs. This is expected to be rare enough to not justify building a complex
# exponential retry system.
if res.status == 429: # Too Many Requests
log.warning('Reading GCS logfile: 429 (server is throttling us)')
if is_last:
self._PrintLastLine(self.LOG_OUTPUT_INCOMPLETE)
return
if res.status >= 500 and res.status < 600: # Server Error
log.warning('Reading GCS logfile: got {0}, retrying'.format(res.status))
if is_last:
self._PrintLastLine(self.LOG_OUTPUT_INCOMPLETE)
return
# Default: any other codes are treated as errors.
raise api_exceptions.HttpError(res, body, self.url)
def __init__(self):
platform = platforms.Platform.Current()
self.version = config.CLOUD_SDK_VERSION
self.operating_system = platform.operating_system
self.architecture = platform.architecture
self.python_location = sys.executable and encoding.Decode(
sys.executable)
self.python_version = sys.version
self.site_packages = 'site' in sys.modules
def MakeAnalyzeIamPolicyHttpRequests(args, api_version=V1P4ALPHA1_API_VERSION):
"""Manually make the get assets history request."""
http_client = http.Http()
parent = asset_utils.GetParentNameForAnalyzeIamPolicy(args.organization)
url_base = '{0}/{1}/{2}:{3}'.format(BASE_URL, api_version, parent,
'analyzeIamPolicy')
params = []
if args.IsSpecified('full_resource_name'):
params.extend([('resourceSelector.fullResourceName',
args.full_resource_name)])
if args.IsSpecified('identity'):
params.extend([('identitySelector.identity', args.identity)])
if args.IsSpecified('roles'):
params.extend([('accessSelector.roles', r) for r in args.roles])
if args.IsSpecified('permissions'):
params.extend([('accessSelector.permissions', p)
for p in args.permissions])
if args.IsSpecified('expand_groups'):
params.extend([('options.expandGroups', args.expand_groups)])
if args.IsSpecified('expand_resources'):
params.extend([('options.expandResources', args.expand_resources)])
if args.IsSpecified('expand_roles'):
params.extend([('options.expandRoles', args.expand_roles)])
if args.IsSpecified('output_resource_edges'):
params.extend([('options.outputResourceEdges',
args.output_resource_edges)])
if args.IsSpecified('output_group_edges'):
params.extend([('options.outputGroupEdges', args.output_group_edges)])
if args.IsSpecified('output_partial_result_before_timeout'):
params.extend([('options.outputPartialResultBeforeTimeout',
args.output_partial_result_before_timeout)])
url_query = six.moves.urllib.parse.urlencode(params)
url = '?'.join([url_base, url_query])
response, raw_content = http_client.request(uri=url, headers=_HEADERS)
content = core_encoding.Decode(raw_content)
if response['status'] != '200':
http_error = api_exceptions.HttpError(response, content, url)
raise exceptions.HttpException(http_error)
response_message_class = GetMessages(api_version).AnalyzeIamPolicyResponse
try:
response = encoding.JsonToMessage(response_message_class, content)
except ValueError as e:
err_msg = ('Failed receiving proper response from server, cannot'
'parse received assets. Error details: ' + six.text_type(e))
raise MessageDecodeError(err_msg)
return response
def Load():
"""Initializes the object with values from the config file.
Returns:
InstallationSpecificData: The loaded data.
"""
data = json.loads(
encoding.Decode(pkg_resources.GetResource(__name__, 'config.json')))
return InstallationConfig(**data)
def sdk_root(self):
"""Searches for the Cloud SDK root directory.
Returns:
str, The path to the root of the Cloud SDK or None if it could not be
found.
"""
return file_utils.FindDirectoryContaining(
os.path.dirname(encoding.Decode(__file__)),
Paths.CLOUDSDK_STATE_DIR)
def _GetRawManPageText(self):
"""Returns the raw man page text."""
try:
with files.FileWriter(os.devnull) as f:
return encoding.Decode(subprocess.check_output(['man', self.command],
stderr=f))
except (OSError, subprocess.CalledProcessError):
raise NoManPageTextForCommand(
'Cannot get man(1) command man page text for [{}].'.format(
self.command))
def Run(self, cmd):
"""Runs cmd and returns the output as a string."""
try:
output = subprocess.check_output([self.command] + cmd[1:])
except subprocess.CalledProcessError as e:
# bq exit code is 1 for help and --help. How do you know if help failed?
if e.returncode != 1:
raise
output = e.output
return encoding.Decode(output).replace('bq.py', 'bq')
def PushFromArgs(self, args):
"""Parse the args and add the value that was found to the top of the stack.
Args:
args: [str], The command line args for this invocation.
"""
# TODO(b/138858862): Decoding the arguments won't be necessary here long
# term once sys.argv is always decoded at the place where it is accessed.
args = [encoding.Decode(a) for a in args]
self.Push(_FlagOverrideStack._FindFlagValue(args))
def WrappedRequest(*args, **kwargs):
"""Replacement http.request() method."""
modified_args = list(args)
if not six.PY2:
# httplib2 needs text under Python 3.
if modified_args:
modified_args[0] = encoding.Decode(modified_args[0])
if 'uri' in kwargs:
kwargs['uri'] = encoding.Decode(kwargs['uri'])
# We need to make a copy here because if we don't we will be modifying the
# dictionary that people pass in.
# TODO(b/37281703): Copy the entire dictionary. This is blocked on making
# sure anything that comes through is actually copyable.
if 'headers' in kwargs:
kwargs['headers'] = Modifiers._EncodeHeaders(kwargs['headers'])
modifier_data = []
for handler in handlers:
modifier_result = handler.request(modified_args, kwargs)
if modifier_result.args:
modified_args = modifier_result.args
modifier_data.append(modifier_result.data)
try:
response = orig_request(*modified_args, **kwargs)
except exc_type as e: # pylint: disable=broad-except
response = None
if exc_handler:
exc_handler(e)
else:
raise
if response_encoding is not None:
response = Modifiers._DecodeResponse(response,
response_encoding)
for handler, data in zip(handlers, modifier_data):
if handler.response:
handler.response(response, data)
return response
