def _PrintInfoAboutBucketListingRef(self, bucket_listing_ref):
"""Print listing info for given bucket_listing_ref.
Args:
bucket_listing_ref: BucketListing being listed.
Returns:
Tuple (number of objects, object size)
Raises:
Exception: if calling bug encountered.
"""
obj = bucket_listing_ref.root_object
url_str = bucket_listing_ref.url_string
if (obj.metadata and S3_DELETE_MARKER_GUID
in obj.metadata.additionalProperties):
size_string = '0'
num_bytes = 0
num_objs = 0
url_str += '<DeleteMarker>'
else:
size_string = (MakeHumanReadable(obj.size)
if self.human_readable else str(obj.size))
num_bytes = obj.size
num_objs = 1
if not self.summary_only:
url_detail = '{size:<11} {url}{ending}'.format(
size=size_string,
url=six.ensure_text(url_str),
ending=six.ensure_text(self.line_ending))
print_to_fd(url_detail, file=sys.stdout, end='')
return (num_objs, num_bytes)
def _EncryptionSetKey(self, bucket_metadata, bucket_url,
svc_acct_for_project_num):
"""Sets defaultKmsKeyName on a Cloud Storage bucket.
Args:
bucket_metadata: (apitools_messages.Bucket) Metadata for the given bucket.
bucket_url: (gslib.storage_url.StorageUrl) StorageUrl of the given bucket.
svc_acct_for_project_num: (Dict[int, str]) Mapping of project numbers to
their corresponding service account.
"""
bucket_project_number = bucket_metadata.projectNumber
try:
# newly_authorized will always be False if the project number is in our
# cache dict, since we've already called _AuthorizeProject on it.
service_account, newly_authorized = (
svc_acct_for_project_num[bucket_project_number], False)
except KeyError:
service_account, newly_authorized = self._AuthorizeProject(
bucket_project_number, self.kms_key)
svc_acct_for_project_num[bucket_project_number] = service_account
if newly_authorized:
text_util.print_to_fd('Authorized service account %s to use key:\n%s' %
(service_account, self.kms_key))
bucket_metadata.encryption = apitools_messages.Bucket.EncryptionValue(
defaultKmsKeyName=self.kms_key)
print('Setting default KMS key for bucket %s...' %
str(bucket_url).rstrip('/'))
self.gsutil_api.PatchBucket(bucket_url.bucket_name,
bucket_metadata,
fields=['encryption'],
provider=bucket_url.scheme)
def PrintDir(bucket_listing_ref):
"""Default function for printing buckets or prefixes.
Args:
bucket_listing_ref: BucketListingRef of type BUCKET or PREFIX.
"""
text_util.print_to_fd(bucket_listing_ref.url_string)
def _PrintLongListing(self, bucket_listing_ref):
"""Prints an object with ListingStyle.LONG."""
obj = bucket_listing_ref.root_object
url_str = bucket_listing_ref.url_string
if (obj.metadata and
S3_DELETE_MARKER_GUID in obj.metadata.additionalProperties):
size_string = '0'
num_bytes = 0
num_objs = 0
url_str += '<DeleteMarker>'
else:
size_string = (MakeHumanReadable(obj.size)
if self.human_readable else str(obj.size))
num_bytes = obj.size
num_objs = 1
timestamp = JSON_TIMESTAMP_RE.sub(r'\1T\2Z', str(obj.timeCreated))
printstr = '%(size)10s %(timestamp)s %(url)s'
encoded_etag = None
encoded_metagen = None
if self.all_versions:
printstr += ' metageneration=%(metageneration)s'
encoded_metagen = str(obj.metageneration)
if self.include_etag:
printstr += ' etag=%(etag)s'
encoded_etag = obj.etag
format_args = {
'size': size_string,
'timestamp': timestamp,
'url': url_str,
'metageneration': encoded_metagen,
'etag': encoded_etag
}
text_util.print_to_fd(printstr % format_args)
return (num_objs, num_bytes)
def _OutputHelp(self, help_str):
"""Outputs simply formatted string.
This function paginates if the string is too long, PAGER is defined, and
the output is a tty.
Args:
help_str: String to format.
"""
# Remove <B> and </B> tags and replace them with ANSI control codes if
# writing to a compatible tty.
if IS_WINDOWS or not IsRunningInteractively():
help_str = re.sub('<B>', '', help_str)
help_str = re.sub('</B>', '', help_str)
text_util.print_to_fd(help_str)
return
help_str = re.sub('<B>', '\033[1m', help_str)
help_str = re.sub('</B>', '\033[0;0m', help_str)
num_lines = len(help_str.split('\n'))
if 'PAGER' in os.environ and num_lines >= GetTermLines():
# Use -r option for less to make bolding work right.
pager = os.environ['PAGER'].split(' ')
if pager[0].endswith('less'):
pager.append('-r')
try:
Popen(pager, stdin=PIPE,
universal_newlines=True).communicate(input=help_str)
except OSError as e:
raise CommandException('Unable to open pager (%s): %s' %
(' '.join(pager), e))
else:
text_util.print_to_fd(help_str)
def _PrintLongListing(self, bucket_listing_ref):
"""Prints an object with ListingStyle.LONG."""
obj = bucket_listing_ref.root_object
url_str = bucket_listing_ref.url_string
if (obj.metadata and S3_DELETE_MARKER_GUID
in obj.metadata.additionalProperties):
size_string = '0'
num_bytes = 0
num_objs = 0
url_str += '<DeleteMarker>'
else:
size_string = (MakeHumanReadable(obj.size)
if self.human_readable else str(obj.size))
num_bytes = obj.size
num_objs = 1
timestamp = JSON_TIMESTAMP_RE.sub(r'\1T\2Z', str(obj.timeCreated))
printstr = '%(size)10s %(timestamp)s %(url)s'
encoded_etag = None
encoded_metagen = None
if self.all_versions:
printstr += ' metageneration=%(metageneration)s'
encoded_metagen = str(obj.metageneration)
if self.include_etag:
printstr += ' etag=%(etag)s'
encoded_etag = obj.etag
format_args = {
'size': size_string,
'timestamp': timestamp,
'url': url_str,
'metageneration': encoded_metagen,
'etag': encoded_etag
}
text_util.print_to_fd(printstr % format_args)
return (num_objs, num_bytes)
def _PrintInfoAboutBucketListingRef(self, bucket_listing_ref):
"""Print listing info for given bucket_listing_ref.
Args:
bucket_listing_ref: BucketListing being listed.
Returns:
Tuple (number of objects, object size)
Raises:
Exception: if calling bug encountered.
"""
obj = bucket_listing_ref.root_object
url_str = bucket_listing_ref.url_string
if (obj.metadata and
S3_DELETE_MARKER_GUID in obj.metadata.additionalProperties):
size_string = '0'
num_bytes = 0
num_objs = 0
url_str += '<DeleteMarker>'
else:
size_string = (MakeHumanReadable(obj.size)
if self.human_readable else str(obj.size))
num_bytes = obj.size
num_objs = 1
if not self.summary_only:
url_detail = '{size:<11} {url}{ending}'.format(
size=size_string,
url=six.ensure_text(url_str),
ending=six.ensure_text(self.line_ending))
print_to_fd(url_detail, file=sys.stdout, end='')
return (num_objs, num_bytes)
def PrintDir(bucket_listing_ref):
"""Default function for printing buckets or prefixes.
Args:
bucket_listing_ref: BucketListingRef of type BUCKET or PREFIX.
"""
text_util.print_to_fd(bucket_listing_ref.url_string)
def PrintDirHeader(bucket_listing_ref):
"""Default function for printing headers for prefixes.
Header is printed prior to listing the contents of the prefix.
Args:
bucket_listing_ref: BucketListingRef of type PREFIX.
"""
text_util.print_to_fd('{}:'.format(bucket_listing_ref.url_string))
def PrintDirHeader(bucket_listing_ref):
"""Default function for printing headers for prefixes.
Header is printed prior to listing the contents of the prefix.
Args:
bucket_listing_ref: BucketListingRef of type PREFIX.
"""
text_util.print_to_fd('{}:'.format(bucket_listing_ref.url_string))
def _AuthorizeProject(self, project_id, kms_key):
"""Authorizes a project's service account to be used with a KMS key.
Authorizes the Cloud Storage-owned service account for project_id to be used
with kms_key.
Args:
project_id: (str) Project id string (not number).
kms_key: (str) Fully qualified resource name for the KMS key.
Returns:
(str, bool) A 2-tuple consisting of:
1) The email address for the service account associated with the project,
which is authorized to encrypt/decrypt with the specified key.
2) A bool value - True if we had to grant the service account permission
to encrypt/decrypt with the given key; False if the required permission
was already present.
"""
# Request the Cloud Storage-owned service account for project_id, creating
# it if it does not exist.
service_account = self.gsutil_api.GetProjectServiceAccount(
project_id, provider='gs').email_address
kms_api = KmsApi(logger=self.logger)
self.logger.debug('Getting IAM policy for %s', kms_key)
try:
policy = kms_api.GetKeyIamPolicy(kms_key)
self.logger.debug('Current policy is %s', policy)
# Check if the required binding is already present; if not, add it and
# update the key's IAM policy.
added_new_binding = False
binding = Binding(
role='roles/cloudkms.cryptoKeyEncrypterDecrypter',
members=['serviceAccount:%s' % service_account])
if binding not in policy.bindings:
policy.bindings.append(binding)
kms_api.SetKeyIamPolicy(kms_key, policy)
added_new_binding = True
return (service_account, added_new_binding)
except AccessDeniedException:
if self.warn_on_key_authorize_failure:
text_util.print_to_fd('\n'.join(
textwrap.wrap(
'Warning: Unable to check the IAM policy for the specified '
'encryption key. Check that your Cloud Platform project\'s '
'service account has the '
'"cloudkms.cryptoKeyEncrypterDecrypter" role for the '
'specified key. Without this role, you may not be able to '
'encrypt or decrypt objects using the key which will '
'prevent you from uploading or downloading objects.')))
return (service_account, False)
else:
raise
def PrintObject(bucket_listing_ref):
"""Default printing function for objects.
Args:
bucket_listing_ref: BucketListingRef of type OBJECT.
Returns:
(num_objects, num_bytes).
"""
try:
text_util.print_to_fd(bucket_listing_ref.url_string)
except IOError as e:
# Windows throws an IOError 0 here for object names containing Unicode
# chars. Ignore it.
if not (IS_WINDOWS and e.errno == 0):
raise
return (1, 0)
def PrintObject(bucket_listing_ref):
"""Default printing function for objects.
Args:
bucket_listing_ref: BucketListingRef of type OBJECT.
Returns:
(num_objects, num_bytes).
"""
try:
text_util.print_to_fd(bucket_listing_ref.url_string)
except IOError as e:
# Windows throws an IOError 0 here for object names containing Unicode
# chars. Ignore it.
if not (IS_WINDOWS and e.errno == 0):
raise
return (1, 0)
def MaybePromptForPythonUpdate(self, command_name):
"""Alert the user that they should install Python 3.
Args:
command_name: The name of the command being run.
Returns:
True if a prompt was output.
"""
if (sys.version_info.major != 2 or self.SkipUpdateCheck() or
command_name not in ('update', 'ver', 'version') or
boto.config.getbool('GSUtil', 'skip_python_update_prompt', False)):
return False
# Notify the user about Python 2 deprecation.
print_to_fd(
'Gsutil 5 drops Python 2 support. Please install Python 3 to update '
'to the latest version of gsutil. https://goo.gle/py3\n')
return True
def _OutputAndExit(message, exception=None):
"""Outputs message to stderr and exits gsutil with code 1.
This function should only be called in single-process, single-threaded mode.
Args:
message: Message to print to stderr.
exception: The exception that caused gsutil to fail.
"""
if debug_level >= constants.DEBUGLEVEL_DUMP_REQUESTS or test_exception_traces:
stack_trace = traceback.format_exc()
err = ('DEBUG: Exception stack trace:\n %s\n%s\n' %
(re.sub('\\n', '\n ', stack_trace), message))
else:
err = '%s\n' % message
try:
text_util.print_to_fd(err, end='', file=sys.stderr)
except UnicodeDecodeError:
# Can happen when outputting invalid Unicode filenames.
sys.stderr.write(err)
if exception:
metrics.LogFatalError(exception)
sys.exit(1)
def _OutputAndExit(message, exception=None):
"""Outputs message to stderr and exits gsutil with code 1.
This function should only be called in single-process, single-threaded mode.
Args:
message: Message to print to stderr.
exception: The exception that caused gsutil to fail.
"""
if debug_level >= constants.DEBUGLEVEL_DUMP_REQUESTS or test_exception_traces:
stack_trace = traceback.format_exc()
err = ('DEBUG: Exception stack trace:\n %s\n%s\n' %
(re.sub('\\n', '\n ', stack_trace), message))
else:
err = '%s\n' % message
try:
text_util.print_to_fd(err, end='', file=sys.stderr)
except UnicodeDecodeError:
# Can happen when outputting invalid Unicode filenames.
sys.stderr.write(err)
if exception:
metrics.LogFatalError(exception)
sys.exit(1)
def _Get(self):
"""Gets logging configuration for a bucket."""
bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg(
self.args[0], bucket_fields=['logging'])
if bucket_url.scheme == 's3':
text_util.print_to_fd(self.gsutil_api.XmlPassThroughGetLogging(
bucket_url, provider=bucket_url.scheme),
end='')
else:
if (bucket_metadata.logging and bucket_metadata.logging.logBucket and
bucket_metadata.logging.logObjectPrefix):
text_util.print_to_fd(
str(encoding.MessageToJson(bucket_metadata.logging)))
else:
text_util.print_to_fd('%s has no logging configuration.' % bucket_url)
return 0
def _PrintSummaryLine(self, num_bytes, name):
size_string = (MakeHumanReadable(num_bytes)
if self.human_readable else six.text_type(num_bytes))
text_util.print_to_fd('{size:<11} {name}'.format(
size=size_string, name=six.ensure_text(name)),
end=self.line_ending)
def _PrintBucketInfo(self, bucket_blr, listing_style):
"""Print listing info for given bucket.
Args:
bucket_blr: BucketListingReference for the bucket being listed
listing_style: ListingStyle enum describing type of output desired.
Returns:
Tuple (total objects, total bytes) in the bucket.
"""
if (listing_style == ListingStyle.SHORT
or listing_style == ListingStyle.LONG):
text_util.print_to_fd(bucket_blr)
return
# listing_style == ListingStyle.LONG_LONG:
# We're guaranteed by the caller that the root object is populated.
bucket = bucket_blr.root_object
location_constraint = bucket.location
storage_class = bucket.storageClass
fields = {
'bucket': bucket_blr.url_string,
'storage_class': storage_class,
'location_constraint': location_constraint,
'acl': AclTranslation.JsonFromMessage(bucket.acl),
'default_acl':
AclTranslation.JsonFromMessage(bucket.defaultObjectAcl),
'versioning': bucket.versioning and bucket.versioning.enabled,
'website_config': 'Present' if bucket.website else 'None',
'logging_config': 'Present' if bucket.logging else 'None',
'cors_config': 'Present' if bucket.cors else 'None',
'lifecycle_config': 'Present' if bucket.lifecycle else 'None',
'requester_pays': bucket.billing and bucket.billing.requesterPays
}
if bucket.retentionPolicy:
fields['retention_policy'] = 'Present'
if bucket.labels:
fields['labels'] = LabelTranslation.JsonFromMessage(
bucket.labels, pretty_print=True)
else:
fields['labels'] = 'None'
if bucket.encryption and bucket.encryption.defaultKmsKeyName:
fields['default_kms_key'] = bucket.encryption.defaultKmsKeyName
else:
fields['default_kms_key'] = 'None'
fields[
'encryption_config'] = 'Present' if bucket.encryption else 'None'
# Fields not available in all APIs (e.g. the XML API)
if bucket.locationType:
fields['location_type'] = bucket.locationType
if bucket.metageneration:
fields['metageneration'] = bucket.metageneration
if bucket.timeCreated:
fields['time_created'] = bucket.timeCreated.strftime(
'%a, %d %b %Y %H:%M:%S GMT')
if bucket.updated:
fields['updated'] = bucket.updated.strftime(
'%a, %d %b %Y %H:%M:%S GMT')
if bucket.defaultEventBasedHold:
fields['default_eventbased_hold'] = bucket.defaultEventBasedHold
if bucket.iamConfiguration and bucket.iamConfiguration.bucketPolicyOnly:
enabled = bucket.iamConfiguration.bucketPolicyOnly.enabled
fields['bucket_policy_only_enabled'] = enabled
# For field values that are multiline, add indenting to make it look
# prettier.
for key in fields:
previous_value = fields[key]
if (not isinstance(previous_value, six.string_types)
or '\n' not in previous_value):
continue
new_value = previous_value.replace('\n', '\n\t ')
# Start multiline values on a new line if they aren't already.
if not new_value.startswith('\n'):
new_value = '\n\t ' + new_value
fields[key] = new_value
# Only display certain properties if the given API returned them (JSON API
# returns many fields that the XML API does not).
location_type_line = ''
metageneration_line = ''
time_created_line = ''
time_updated_line = ''
default_eventbased_hold_line = ''
retention_policy_line = ''
bucket_policy_only_enabled_line = ''
if 'location_type' in fields:
location_type_line = '\tLocation type:\t\t\t{location_type}\n'
if 'metageneration' in fields:
metageneration_line = '\tMetageneration:\t\t\t{metageneration}\n'
if 'time_created' in fields:
time_created_line = '\tTime created:\t\t\t{time_created}\n'
if 'updated' in fields:
time_updated_line = '\tTime updated:\t\t\t{updated}\n'
if 'default_eventbased_hold' in fields:
default_eventbased_hold_line = (
'\tDefault Event-Based Hold:\t{default_eventbased_hold}\n')
if 'retention_policy' in fields:
retention_policy_line = '\tRetention Policy:\t\t{retention_policy}\n'
if 'bucket_policy_only_enabled' in fields:
bucket_policy_only_enabled_line = (
'\tBucket Policy Only enabled:\t'
'{bucket_policy_only_enabled}\n')
text_util.print_to_fd(
('{bucket} :\n'
'\tStorage class:\t\t\t{storage_class}\n' + location_type_line +
'\tLocation constraint:\t\t{location_constraint}\n'
'\tVersioning enabled:\t\t{versioning}\n'
'\tLogging configuration:\t\t{logging_config}\n'
'\tWebsite configuration:\t\t{website_config}\n'
'\tCORS configuration: \t\t{cors_config}\n'
'\tLifecycle configuration:\t{lifecycle_config}\n'
'\tRequester Pays enabled:\t\t{requester_pays}\n' +
retention_policy_line + default_eventbased_hold_line +
'\tLabels:\t\t\t\t{labels}\n' +
'\tDefault KMS key:\t\t{default_kms_key}\n' + time_created_line +
time_updated_line + metageneration_line +
bucket_policy_only_enabled_line + '\tACL:\t\t\t\t{acl}\n'
'\tDefault ACL:\t\t\t{default_acl}').format(**fields))
if bucket_blr.storage_url.scheme == 's3':
text_util.print_to_fd(
'Note: this is an S3 bucket so configuration values may be '
'blank. To retrieve bucket configuration values, use '
'individual configuration commands such as gsutil acl get '
'<bucket>.')
def RunCommand(self):
"""Command entry point for the hash command."""
(calc_crc32c, calc_md5, format_func, cloud_format_func,
output_format) = (self._ParseOpts(self.sub_opts, self.logger))
matched_one = False
for url_str in self.args:
for file_ref in self.WildcardIterator(url_str).IterObjects(
bucket_listing_fields=[
'crc32c',
'customerEncryption',
'md5Hash',
'size',
]):
matched_one = True
url = StorageUrlFromString(url_str)
file_name = file_ref.storage_url.object_name
if StorageUrlFromString(url_str).IsFileUrl():
file_size = os.path.getsize(file_name)
self.gsutil_api.status_queue.put(
FileMessage(url,
None,
time.time(),
size=file_size,
finished=False,
message_type=FileMessage.FILE_HASH))
callback_processor = ProgressCallbackWithTimeout(
file_size,
FileProgressCallbackHandler(self.gsutil_api.status_queue,
src_url=StorageUrlFromString(url_str),
operation_name='Hashing').call)
hash_dict = self._GetHashClassesFromArgs(calc_crc32c, calc_md5)
with open(file_name, 'rb') as fp:
hashing_helper.CalculateHashesFromContents(
fp, hash_dict, callback_processor=callback_processor)
self.gsutil_api.status_queue.put(
FileMessage(url,
None,
time.time(),
size=file_size,
finished=True,
message_type=FileMessage.FILE_HASH))
else:
hash_dict = {}
obj_metadata = file_ref.root_object
file_size = obj_metadata.size
md5_present = obj_metadata.md5Hash is not None
crc32c_present = obj_metadata.crc32c is not None
if not md5_present and not crc32c_present:
logging.getLogger().warn('No hashes present for %s', url_str)
continue
if md5_present:
hash_dict['md5'] = obj_metadata.md5Hash
if crc32c_present:
hash_dict['crc32c'] = obj_metadata.crc32c
text_util.print_to_fd('Hashes [%s] for %s:' %
(output_format, file_name))
for name, digest in six.iteritems(hash_dict):
text_util.print_to_fd('\tHash (%s):\t\t%s' %
(name, (format_func(digest) if url.IsFileUrl()
else cloud_format_func(digest))))
if not matched_one:
raise CommandException('No files matched')
_PutToQueueWithTimeout(self.gsutil_api.status_queue,
FinalMessage(time.time()))
return 0
def PrintNewLine():
"""Default function for printing new lines between directories."""
text_util.print_to_fd()
def RunCommand(self,
command_name,
args=None,
headers=None,
debug=0,
return_stdout=False,
return_stderr=False,
return_log_handler=False,
cwd=None):
"""Method for calling gslib.command_runner.CommandRunner.
Passes parallel_operations=False for all tests, optionally saving/returning
stdout output. We run all tests multi-threaded, to exercise those more
complicated code paths.
TODO: Change to run with parallel_operations=True for all tests. At
present when you do this it causes many test failures.
Args:
command_name: The name of the command being run.
args: Command-line args (arg0 = actual arg, not command name ala bash).
headers: Dictionary containing optional HTTP headers to pass to boto.
debug: Debug level to pass in to boto connection (range 0..3).
return_stdout: If True, will save and return stdout produced by command.
return_stderr: If True, will save and return stderr produced by command.
return_log_handler: If True, will return a MockLoggingHandler instance
that was attached to the command's logger while running.
cwd: The working directory that should be switched to before running the
command. The working directory will be reset back to its original
value after running the command. If not specified, the working
directory is left unchanged.
Returns:
One or a tuple of requested return values, depending on whether
return_stdout, return_stderr, and/or return_log_handler were specified.
Return Types:
stdout - str (binary in Py2, text in Py3)
stderr - str (binary in Py2, text in Py3)
log_handler - MockLoggingHandler
"""
args = args or []
command_line = six.ensure_text(' '.join([command_name] + args))
if self.is_debugging:
print_to_fd('\nRunCommand of {}\n'.format(command_line),
file=self.stderr_save)
# Save and truncate stdout and stderr for the lifetime of RunCommand. This
# way, we can return just the stdout and stderr that was output during the
# RunNamedCommand call below.
sys.stdout.seek(0)
sys.stderr.seek(0)
stdout = sys.stdout.read()
stderr = sys.stderr.read()
if stdout:
self.accumulated_stdout.append(stdout)
if stderr:
self.accumulated_stderr.append(stderr)
sys.stdout.seek(0)
sys.stderr.seek(0)
sys.stdout.truncate()
sys.stderr.truncate()
mock_log_handler = MockLoggingHandler()
logging.getLogger(command_name).addHandler(mock_log_handler)
if debug:
logging.getLogger(command_name).setLevel(logging.DEBUG)
try:
with WorkingDirectory(cwd):
self.command_runner.RunNamedCommand(command_name,
args=args,
headers=headers,
debug=debug,
parallel_operations=False,
do_shutdown=False)
finally:
sys.stdout.seek(0)
sys.stderr.seek(0)
if six.PY2:
stdout = sys.stdout.read()
stderr = sys.stderr.read()
else:
try:
stdout = sys.stdout.read()
stderr = sys.stderr.read()
except UnicodeDecodeError:
sys.stdout.seek(0)
sys.stderr.seek(0)
stdout = sys.stdout.buffer.read()
stderr = sys.stderr.buffer.read()
logging.getLogger(command_name).removeHandler(mock_log_handler)
mock_log_handler.close()
log_output = '\n'.join(
'%s:\n ' % level + '\n '.join(records)
for level, records in six.iteritems(mock_log_handler.messages)
if records)
_id = six.ensure_text(self.id())
if self.is_debugging and log_output:
print_to_fd('==== logging RunCommand {} {} ====\n'.format(
_id, command_line),
file=self.stderr_save)
print_to_fd(log_output, file=self.stderr_save)
print_to_fd('\n==== end logging ====\n', file=self.stderr_save)
if self.is_debugging and stdout:
print_to_fd('==== stdout RunCommand {} {} ====\n'.format(
_id, command_line),
file=self.stderr_save)
print_to_fd(stdout, file=self.stderr_save)
print_to_fd('==== end stdout ====\n', file=self.stderr_save)
if self.is_debugging and stderr:
print_to_fd('==== stderr RunCommand {} {} ====\n'.format(
_id, command_line),
file=self.stderr_save)
print_to_fd(stderr, file=self.stderr_save)
print_to_fd('==== end stderr ====\n', file=self.stderr_save)
# Reset stdout and stderr files, so that we won't print them out again
# in tearDown if debugging is enabled.
sys.stdout.seek(0)
sys.stderr.seek(0)
sys.stdout.truncate()
sys.stderr.truncate()
to_return = []
if return_stdout:
to_return.append(stdout)
if return_stderr:
to_return.append(stderr)
if return_log_handler:
to_return.append(mock_log_handler)
if len(to_return) == 1:
return to_return[0]
return tuple(to_return)
def _PrintPrefixLong(blr):
text_util.print_to_fd(
'%-33s%s' % ('', six.ensure_text(blr.url_string)))
def _PrintPrefixLong(blr):
text_util.print_to_fd('%-33s%s' %
('', blr.url_string.decode(UTF8)))
def PrintFullInfoAboutObject(bucket_listing_ref, incl_acl=True):
"""Print full info for given object (like what displays for gsutil ls -L).
Args:
bucket_listing_ref: BucketListingRef being listed.
Must have ref_type OBJECT and a populated root_object
with the desired fields.
incl_acl: True if ACL info should be output.
Returns:
Tuple (number of objects, object_length)
Raises:
Exception: if calling bug encountered.
"""
url_str = bucket_listing_ref.url_string
storage_url = StorageUrlFromString(url_str)
obj = bucket_listing_ref.root_object
if (obj.metadata and
S3_DELETE_MARKER_GUID in obj.metadata.additionalProperties):
num_bytes = 0
num_objs = 0
url_str += '<DeleteMarker>'
else:
num_bytes = obj.size
num_objs = 1
text_util.print_to_fd('{}:'.format(url_str))
if obj.timeCreated:
text_util.print_to_fd(
MakeMetadataLine('Creation time',
obj.timeCreated.strftime('%a, %d %b %Y %H:%M:%S GMT')))
if obj.updated:
text_util.print_to_fd(
MakeMetadataLine('Update time',
obj.updated.strftime('%a, %d %b %Y %H:%M:%S GMT')))
if (obj.timeStorageClassUpdated and
obj.timeStorageClassUpdated != obj.timeCreated):
text_util.print_to_fd(
MakeMetadataLine(
'Storage class update time',
obj.timeStorageClassUpdated.strftime('%a, %d %b %Y %H:%M:%S GMT')))
if obj.storageClass:
text_util.print_to_fd(MakeMetadataLine('Storage class', obj.storageClass))
if obj.temporaryHold:
text_util.print_to_fd(MakeMetadataLine('Temporary Hold', 'Enabled'))
if obj.eventBasedHold:
text_util.print_to_fd(MakeMetadataLine('Event-Based Hold', 'Enabled'))
if obj.retentionExpirationTime:
text_util.print_to_fd(
MakeMetadataLine(
'Retention Expiration',
obj.retentionExpirationTime.strftime('%a, %d %b %Y %H:%M:%S GMT')))
if obj.kmsKeyName:
text_util.print_to_fd(MakeMetadataLine('KMS key', obj.kmsKeyName))
if obj.cacheControl:
text_util.print_to_fd(MakeMetadataLine('Cache-Control', obj.cacheControl))
if obj.contentDisposition:
text_util.print_to_fd(
MakeMetadataLine('Content-Disposition', obj.contentDisposition))
if obj.contentEncoding:
text_util.print_to_fd(
MakeMetadataLine('Content-Encoding', obj.contentEncoding))
if obj.contentLanguage:
text_util.print_to_fd(
MakeMetadataLine('Content-Language', obj.contentLanguage))
text_util.print_to_fd(MakeMetadataLine('Content-Length', obj.size))
text_util.print_to_fd(MakeMetadataLine('Content-Type', obj.contentType))
if obj.componentCount:
text_util.print_to_fd(
MakeMetadataLine('Component-Count', obj.componentCount))
if obj.timeDeleted:
text_util.print_to_fd(
MakeMetadataLine('Archived time',
obj.timeDeleted.strftime('%a, %d %b %Y %H:%M:%S GMT')))
marker_props = {}
if obj.metadata and obj.metadata.additionalProperties:
non_marker_props = []
for add_prop in obj.metadata.additionalProperties:
if add_prop.key not in S3_MARKER_GUIDS:
non_marker_props.append(add_prop)
else:
marker_props[add_prop.key] = add_prop.value
if non_marker_props:
text_util.print_to_fd(MakeMetadataLine('Metadata', ''))
for ap in non_marker_props:
ap_key = '{}'.format(ap.key)
ap_value = '{}'.format(ap.value)
meta_data_line = MakeMetadataLine(ap_key, ap_value, indent=2)
text_util.print_to_fd(meta_data_line)
if obj.customerEncryption:
if not obj.crc32c:
text_util.print_to_fd(MakeMetadataLine('Hash (crc32c)', 'encrypted'))
if not obj.md5Hash:
text_util.print_to_fd(MakeMetadataLine('Hash (md5)', 'encrypted'))
text_util.print_to_fd(
MakeMetadataLine('Encryption algorithm',
obj.customerEncryption.encryptionAlgorithm))
text_util.print_to_fd(
MakeMetadataLine('Encryption key SHA256',
obj.customerEncryption.keySha256))
if obj.crc32c:
text_util.print_to_fd(MakeMetadataLine('Hash (crc32c)', obj.crc32c))
if obj.md5Hash:
text_util.print_to_fd(MakeMetadataLine('Hash (md5)', obj.md5Hash))
text_util.print_to_fd(MakeMetadataLine('ETag', obj.etag.strip('"\'')))
if obj.generation:
generation_str = GenerationFromUrlAndString(storage_url, obj.generation)
text_util.print_to_fd(MakeMetadataLine('Generation', generation_str))
if obj.metageneration:
text_util.print_to_fd(MakeMetadataLine('Metageneration',
obj.metageneration))
if incl_acl:
# JSON API won't return acls as part of the response unless we have
# full control scope
if obj.acl:
text_util.print_to_fd(
MakeMetadataLine('ACL', AclTranslation.JsonFromMessage(obj.acl)))
elif S3_ACL_MARKER_GUID in marker_props:
text_util.print_to_fd(
MakeMetadataLine('ACL', marker_props[S3_ACL_MARKER_GUID]))
else:
# Empty ACLs are possible with Bucket Policy Only and no longer imply
# ACCESS DENIED anymore.
text_util.print_to_fd(MakeMetadataLine('ACL', '[]'))
return (num_objs, num_bytes)
def _PrintBucketInfo(self, bucket_blr, listing_style):
"""Print listing info for given bucket.
Args:
bucket_blr: BucketListingReference for the bucket being listed
listing_style: ListingStyle enum describing type of output desired.
Returns:
Tuple (total objects, total bytes) in the bucket.
"""
if (listing_style == ListingStyle.SHORT or
listing_style == ListingStyle.LONG):
text_util.print_to_fd(bucket_blr)
return
# listing_style == ListingStyle.LONG_LONG:
# We're guaranteed by the caller that the root object is populated.
bucket = bucket_blr.root_object
location_constraint = bucket.location
storage_class = bucket.storageClass
fields = {
'bucket': bucket_blr.url_string,
'storage_class': storage_class,
'location_constraint': location_constraint,
'acl': AclTranslation.JsonFromMessage(bucket.acl),
'default_acl': AclTranslation.JsonFromMessage(bucket.defaultObjectAcl),
'versioning': bucket.versioning and bucket.versioning.enabled,
'website_config': 'Present' if bucket.website else 'None',
'logging_config': 'Present' if bucket.logging else 'None',
'cors_config': 'Present' if bucket.cors else 'None',
'lifecycle_config': 'Present' if bucket.lifecycle else 'None',
'requester_pays': bucket.billing and bucket.billing.requesterPays
}
if bucket.retentionPolicy:
fields['retention_policy'] = 'Present'
if bucket.labels:
fields['labels'] = LabelTranslation.JsonFromMessage(bucket.labels,
pretty_print=True)
else:
fields['labels'] = 'None'
if bucket.encryption and bucket.encryption.defaultKmsKeyName:
fields['default_kms_key'] = bucket.encryption.defaultKmsKeyName
else:
fields['default_kms_key'] = 'None'
fields['encryption_config'] = 'Present' if bucket.encryption else 'None'
# Fields not available in all APIs (e.g. the XML API)
if bucket.metageneration:
fields['metageneration'] = bucket.metageneration
if bucket.timeCreated:
fields['time_created'] = bucket.timeCreated.strftime(
'%a, %d %b %Y %H:%M:%S GMT')
if bucket.updated:
fields['updated'] = bucket.updated.strftime('%a, %d %b %Y %H:%M:%S GMT')
if bucket.defaultEventBasedHold:
fields['default_eventbased_hold'] = bucket.defaultEventBasedHold
if bucket.iamConfiguration and bucket.iamConfiguration.bucketPolicyOnly:
enabled = bucket.iamConfiguration.bucketPolicyOnly.enabled
fields['bucket_policy_only_enabled'] = enabled
# For field values that are multiline, add indenting to make it look
# prettier.
for key in fields:
previous_value = fields[key]
if (not isinstance(previous_value, six.string_types) or
'\n' not in previous_value):
continue
new_value = previous_value.replace('\n', '\n\t ')
# Start multiline values on a new line if they aren't already.
if not new_value.startswith('\n'):
new_value = '\n\t ' + new_value
fields[key] = new_value
# Only display certain properties if the given API returned them (JSON API
# returns many fields that the XML API does not).
metageneration_line = ''
time_created_line = ''
time_updated_line = ''
default_eventbased_hold_line = ''
retention_policy_line = ''
bucket_policy_only_enabled_line = ''
if 'metageneration' in fields:
metageneration_line = '\tMetageneration:\t\t\t{metageneration}\n'
if 'time_created' in fields:
time_created_line = '\tTime created:\t\t\t{time_created}\n'
if 'updated' in fields:
time_updated_line = '\tTime updated:\t\t\t{updated}\n'
if 'default_eventbased_hold' in fields:
default_eventbased_hold_line = (
'\tDefault Event-Based Hold:\t{default_eventbased_hold}\n')
if 'retention_policy' in fields:
retention_policy_line = '\tRetention Policy:\t\t{retention_policy}\n'
if 'bucket_policy_only_enabled' in fields:
bucket_policy_only_enabled_line = ('\tBucket Policy Only enabled:\t'
'{bucket_policy_only_enabled}\n')
text_util.print_to_fd(
('{bucket} :\n'
'\tStorage class:\t\t\t{storage_class}\n'
'\tLocation constraint:\t\t{location_constraint}\n'
'\tVersioning enabled:\t\t{versioning}\n'
'\tLogging configuration:\t\t{logging_config}\n'
'\tWebsite configuration:\t\t{website_config}\n'
'\tCORS configuration: \t\t{cors_config}\n'
'\tLifecycle configuration:\t{lifecycle_config}\n'
'\tRequester Pays enabled:\t\t{requester_pays}\n' +
retention_policy_line + default_eventbased_hold_line +
'\tLabels:\t\t\t\t{labels}\n' +
'\tDefault KMS key:\t\t{default_kms_key}\n' + time_created_line +
time_updated_line + metageneration_line +
bucket_policy_only_enabled_line + '\tACL:\t\t\t\t{acl}\n'
'\tDefault ACL:\t\t\t{default_acl}').format(**fields))
if bucket_blr.storage_url.scheme == 's3':
text_util.print_to_fd(
'Note: this is an S3 bucket so configuration values may be '
'blank. To retrieve bucket configuration values, use '
'individual configuration commands such as gsutil acl get '
'<bucket>.')
def _PrintSummaryLine(self, num_bytes, name):
size_string = (MakeHumanReadable(num_bytes)
if self.human_readable else six.text_type(num_bytes))
text_util.print_to_fd('{size:<11} {name}'.format(
size=size_string, name=six.ensure_text(name)),
end=self.line_ending)
def MaybePrintBucketHeader(blr):
if len(self.args) > 1:
text_util.print_to_fd('%s:' % blr.url_string.decode(UTF8))
def PrintFullInfoAboutObject(bucket_listing_ref, incl_acl=True):
"""Print full info for given object (like what displays for gsutil ls -L).
Args:
bucket_listing_ref: BucketListingRef being listed.
Must have ref_type OBJECT and a populated root_object
with the desired fields.
incl_acl: True if ACL info should be output.
Returns:
Tuple (number of objects, object_length)
Raises:
Exception: if calling bug encountered.
"""
url_str = bucket_listing_ref.url_string
storage_url = StorageUrlFromString(url_str)
obj = bucket_listing_ref.root_object
if (obj.metadata
and S3_DELETE_MARKER_GUID in obj.metadata.additionalProperties):
num_bytes = 0
num_objs = 0
url_str += '<DeleteMarker>'
else:
num_bytes = obj.size
num_objs = 1
text_util.print_to_fd('{}:'.format(url_str))
if obj.timeCreated:
text_util.print_to_fd(
MakeMetadataLine(
'Creation time',
obj.timeCreated.strftime('%a, %d %b %Y %H:%M:%S GMT')))
if obj.updated:
text_util.print_to_fd(
MakeMetadataLine(
'Update time',
obj.updated.strftime('%a, %d %b %Y %H:%M:%S GMT')))
if (obj.timeStorageClassUpdated
and obj.timeStorageClassUpdated != obj.timeCreated):
text_util.print_to_fd(
MakeMetadataLine(
'Storage class update time',
obj.timeStorageClassUpdated.strftime(
'%a, %d %b %Y %H:%M:%S GMT')))
if obj.storageClass:
text_util.print_to_fd(
MakeMetadataLine('Storage class', obj.storageClass))
if obj.temporaryHold:
text_util.print_to_fd(MakeMetadataLine('Temporary Hold', 'Enabled'))
if obj.eventBasedHold:
text_util.print_to_fd(MakeMetadataLine('Event-Based Hold', 'Enabled'))
if obj.retentionExpirationTime:
text_util.print_to_fd(
MakeMetadataLine(
'Retention Expiration',
obj.retentionExpirationTime.strftime(
'%a, %d %b %Y %H:%M:%S GMT')))
if obj.kmsKeyName:
text_util.print_to_fd(MakeMetadataLine('KMS key', obj.kmsKeyName))
if obj.cacheControl:
text_util.print_to_fd(
MakeMetadataLine('Cache-Control', obj.cacheControl))
if obj.contentDisposition:
text_util.print_to_fd(
MakeMetadataLine('Content-Disposition', obj.contentDisposition))
if obj.contentEncoding:
text_util.print_to_fd(
MakeMetadataLine('Content-Encoding', obj.contentEncoding))
if obj.contentLanguage:
text_util.print_to_fd(
MakeMetadataLine('Content-Language', obj.contentLanguage))
text_util.print_to_fd(MakeMetadataLine('Content-Length', obj.size))
text_util.print_to_fd(MakeMetadataLine('Content-Type', obj.contentType))
if obj.componentCount:
text_util.print_to_fd(
MakeMetadataLine('Component-Count', obj.componentCount))
if obj.timeDeleted:
text_util.print_to_fd(
MakeMetadataLine(
'Archived time',
obj.timeDeleted.strftime('%a, %d %b %Y %H:%M:%S GMT')))
marker_props = {}
if obj.metadata and obj.metadata.additionalProperties:
non_marker_props = []
for add_prop in obj.metadata.additionalProperties:
if add_prop.key not in S3_MARKER_GUIDS:
non_marker_props.append(add_prop)
else:
marker_props[add_prop.key] = add_prop.value
if non_marker_props:
text_util.print_to_fd(MakeMetadataLine('Metadata', ''))
for ap in non_marker_props:
ap_key = '{}'.format(ap.key)
ap_value = '{}'.format(ap.value)
meta_data_line = MakeMetadataLine(ap_key, ap_value, indent=2)
text_util.print_to_fd(meta_data_line)
if obj.customerEncryption:
if not obj.crc32c:
text_util.print_to_fd(
MakeMetadataLine('Hash (crc32c)', 'encrypted'))
if not obj.md5Hash:
text_util.print_to_fd(MakeMetadataLine('Hash (md5)', 'encrypted'))
text_util.print_to_fd(
MakeMetadataLine('Encryption algorithm',
obj.customerEncryption.encryptionAlgorithm))
text_util.print_to_fd(
MakeMetadataLine('Encryption key SHA256',
obj.customerEncryption.keySha256))
if obj.crc32c:
text_util.print_to_fd(MakeMetadataLine('Hash (crc32c)', obj.crc32c))
if obj.md5Hash:
text_util.print_to_fd(MakeMetadataLine('Hash (md5)', obj.md5Hash))
text_util.print_to_fd(MakeMetadataLine('ETag', obj.etag.strip('"\'')))
if obj.generation:
generation_str = GenerationFromUrlAndString(storage_url,
obj.generation)
text_util.print_to_fd(MakeMetadataLine('Generation', generation_str))
if obj.metageneration:
text_util.print_to_fd(
MakeMetadataLine('Metageneration', obj.metageneration))
if incl_acl:
# JSON API won't return acls as part of the response unless we have
# full control scope
if obj.acl:
text_util.print_to_fd(
MakeMetadataLine('ACL',
AclTranslation.JsonFromMessage(obj.acl)))
elif S3_ACL_MARKER_GUID in marker_props:
text_util.print_to_fd(
MakeMetadataLine('ACL', marker_props[S3_ACL_MARKER_GUID]))
else:
# Empty ACLs are possible with Bucket Policy Only and no longer imply
# ACCESS DENIED anymore.
text_util.print_to_fd(MakeMetadataLine('ACL', '[]'))
return (num_objs, num_bytes)
def RunCommand(self):
"""Command entry point for the ls command."""
got_nomatch_errors = False
got_bucket_nomatch_errors = False
listing_style = ListingStyle.SHORT
get_bucket_info = False
self.recursion_requested = False
self.all_versions = False
self.include_etag = False
self.human_readable = False
self.list_subdir_contents = True
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-a':
self.all_versions = True
elif o == '-e':
self.include_etag = True
elif o == '-b':
get_bucket_info = True
elif o == '-h':
self.human_readable = True
elif o == '-l':
listing_style = ListingStyle.LONG
elif o == '-L':
listing_style = ListingStyle.LONG_LONG
elif o == '-p':
# Project IDs are sent as header values when using gs and s3 XML APIs.
InsistAscii(
a, 'Invalid non-ASCII character found in project ID')
self.project_id = a
elif o == '-r' or o == '-R':
self.recursion_requested = True
elif o == '-d':
self.list_subdir_contents = False
if not self.args:
# default to listing all gs buckets
self.args = ['gs://']
total_objs = 0
total_bytes = 0
def MaybePrintBucketHeader(blr):
if len(self.args) > 1:
text_util.print_to_fd('%s:' % six.ensure_text(blr.url_string))
print_bucket_header = MaybePrintBucketHeader
for url_str in self.args:
storage_url = StorageUrlFromString(url_str)
if storage_url.IsFileUrl():
raise CommandException('Only cloud URLs are supported for %s' %
self.command_name)
bucket_fields = None
if (listing_style == ListingStyle.SHORT
or listing_style == ListingStyle.LONG):
bucket_fields = ['id']
elif listing_style == ListingStyle.LONG_LONG:
bucket_fields = [
'acl',
'billing',
'cors',
'defaultObjectAcl',
'encryption',
'iamConfiguration',
'labels',
'location',
'locationType',
'logging',
'lifecycle',
'metageneration',
'retentionPolicy',
'defaultEventBasedHold',
'storageClass',
'timeCreated',
'updated',
'versioning',
'website',
]
if storage_url.IsProvider():
# Provider URL: use bucket wildcard to list buckets.
for blr in self.WildcardIterator(
'%s://*' % storage_url.scheme).IterBuckets(
bucket_fields=bucket_fields):
self._PrintBucketInfo(blr, listing_style)
elif storage_url.IsBucket() and get_bucket_info:
# ls -b bucket listing request: List info about bucket(s).
total_buckets = 0
for blr in self.WildcardIterator(url_str).IterBuckets(
bucket_fields=bucket_fields):
if not ContainsWildcard(url_str) and not blr.root_object:
# Iterator does not make an HTTP call for non-wildcarded
# listings with fields=='id'. Ensure the bucket exists by calling
# GetBucket.
self.gsutil_api.GetBucket(blr.storage_url.bucket_name,
fields=['id'],
provider=storage_url.scheme)
self._PrintBucketInfo(blr, listing_style)
total_buckets += 1
if not ContainsWildcard(url_str) and not total_buckets:
got_bucket_nomatch_errors = True
else:
# URL names a bucket, object, or object subdir ->
# list matching object(s) / subdirs.
def _PrintPrefixLong(blr):
text_util.print_to_fd(
'%-33s%s' % ('', six.ensure_text(blr.url_string)))
if listing_style == ListingStyle.SHORT:
# ls helper by default readies us for a short listing.
listing_helper = LsHelper(
self.WildcardIterator,
self.logger,
all_versions=self.all_versions,
print_bucket_header_func=print_bucket_header,
should_recurse=self.recursion_requested,
list_subdir_contents=self.list_subdir_contents)
elif listing_style == ListingStyle.LONG:
bucket_listing_fields = [
'name',
'size',
'timeCreated',
'updated',
]
if self.all_versions:
bucket_listing_fields.extend([
'generation',
'metageneration',
])
if self.include_etag:
bucket_listing_fields.append('etag')
listing_helper = LsHelper(
self.WildcardIterator,
self.logger,
print_object_func=self._PrintLongListing,
print_dir_func=_PrintPrefixLong,
print_bucket_header_func=print_bucket_header,
all_versions=self.all_versions,
should_recurse=self.recursion_requested,
fields=bucket_listing_fields,
list_subdir_contents=self.list_subdir_contents)
elif listing_style == ListingStyle.LONG_LONG:
# List all fields
bucket_listing_fields = (UNENCRYPTED_FULL_LISTING_FIELDS +
ENCRYPTED_FIELDS)
listing_helper = LsHelper(
self.WildcardIterator,
self.logger,
print_object_func=PrintFullInfoAboutObject,
print_dir_func=_PrintPrefixLong,
print_bucket_header_func=print_bucket_header,
all_versions=self.all_versions,
should_recurse=self.recursion_requested,
fields=bucket_listing_fields,
list_subdir_contents=self.list_subdir_contents)
else:
raise CommandException('Unknown listing style: %s' %
listing_style)
exp_dirs, exp_objs, exp_bytes = (
listing_helper.ExpandUrlAndPrint(storage_url))
if storage_url.IsObject() and exp_objs == 0 and exp_dirs == 0:
got_nomatch_errors = True
total_bytes += exp_bytes
total_objs += exp_objs
if total_objs and listing_style != ListingStyle.SHORT:
text_util.print_to_fd('TOTAL: %d objects, %d bytes (%s)' %
(total_objs, total_bytes,
MakeHumanReadable(float(total_bytes))))
if got_nomatch_errors:
raise CommandException('One or more URLs matched no objects.')
if got_bucket_nomatch_errors:
raise NotFoundException(
'One or more bucket URLs matched no buckets.')
return 0
def MaybePrintBucketHeader(blr):
if len(self.args) > 1:
text_util.print_to_fd('%s:' % six.ensure_text(blr.url_string))
def MaybeCheckForAndOfferSoftwareUpdate(self, command_name, debug):
"""Checks the last time we checked for an update and offers one if needed.
Offer is made if the time since the last update check is longer
than the configured threshold offers the user to update gsutil.
Args:
command_name: The name of the command being run.
debug: Debug level to pass in to boto connection (range 0..3).
Returns:
True if the user decides to update.
"""
# Don't try to interact with user if:
# - gsutil is not connected to a tty (e.g., if being run from cron);
# - user is running gsutil -q
# - user is running the config command (which could otherwise attempt to
# check for an update for a user running behind a proxy, who has not yet
# configured gsutil to go through the proxy; for such users we need the
# first connection attempt to be made by the gsutil config command).
# - user is running the version command (which gets run when using
# gsutil -D, which would prevent users with proxy config problems from
# sending us gsutil -D output).
# - user is running the update command (which could otherwise cause an
# additional note that an update is available when user is already trying
# to perform an update);
# - user specified gs_host (which could be a non-production different
# service instance, in which case credentials won't work for checking
# gsutil tarball).
# - user is using a Cloud SDK install (which should only be updated via
# gcloud components update)
logger = logging.getLogger()
if (not system_util.IsRunningInteractively() or
command_name in ('config', 'update', 'ver', 'version') or
not logger.isEnabledFor(logging.INFO) or HAS_NON_DEFAULT_GS_HOST or
system_util.InvokedViaCloudSdk()):
return False
software_update_check_period = boto.config.getint(
'GSUtil', 'software_update_check_period', 30)
# Setting software_update_check_period to 0 means periodic software
# update checking is disabled.
if software_update_check_period == 0:
return False
last_checked_for_gsutil_update_timestamp_file = (
boto_util.GetLastCheckedForGsutilUpdateTimestampFile())
cur_ts = int(time.time())
if not os.path.isfile(last_checked_for_gsutil_update_timestamp_file):
# Set last_checked_ts from date of VERSION file, so if the user installed
# an old copy of gsutil it will get noticed (and an update offered) the
# first time they try to run it.
last_checked_ts = gslib.GetGsutilVersionModifiedTime()
with open(last_checked_for_gsutil_update_timestamp_file, 'w') as f:
f.write(str(last_checked_ts))
else:
try:
with open(last_checked_for_gsutil_update_timestamp_file, 'r') as f:
last_checked_ts = int(f.readline())
except (TypeError, ValueError):
return False
if (cur_ts - last_checked_ts >
software_update_check_period * SECONDS_PER_DAY):
# Create a credential-less gsutil API to check for the public
# update tarball.
gsutil_api = GcsJsonApi(self.bucket_storage_uri_class,
logger,
DiscardMessagesQueue(),
credentials=NoOpCredentials(),
debug=debug)
cur_ver = LookUpGsutilVersion(gsutil_api, GSUTIL_PUB_TARBALL)
with open(last_checked_for_gsutil_update_timestamp_file, 'w') as f:
f.write(str(cur_ts))
(g, m) = CompareVersions(cur_ver, gslib.VERSION)
if m:
print_to_fd('\n'.join(
textwrap.wrap(
'A newer version of gsutil (%s) is available than the version you '
'are running (%s). NOTE: This is a major new version, so it is '
'strongly recommended that you review the release note details at '
'%s before updating to this version, especially if you use gsutil '
'in scripts.' % (cur_ver, gslib.VERSION, RELEASE_NOTES_URL))))
if gslib.IS_PACKAGE_INSTALL:
return False
print_to_fd('\n')
answer = input('Would you like to update [y/N]? ')
return answer and answer.lower()[0] == 'y'
elif g:
print_to_fd('\n'.join(
textwrap.wrap(
'A newer version of gsutil (%s) is available than the version you '
'are running (%s). A detailed log of gsutil release changes is '
'available at %s if you would like to read them before updating.'
% (cur_ver, gslib.VERSION, RELEASE_NOTES_URL))))
if gslib.IS_PACKAGE_INSTALL:
return False
print_to_fd('\n')
answer = input('Would you like to update [Y/n]? ')
return not answer or answer.lower()[0] != 'n'
return False
def tearDown(self):
super(GsUtilUnitTestCase, self).tearDown()
self.root_logger.handlers = self.log_handlers_save
self.temp_log_handler.flush()
self.temp_log_handler.close()
self.log_handler_stream.seek(0)
log_output = self.log_handler_stream.read()
self.log_handler_stream.close()
os.unlink(self.log_handler_file)
sys.stdout.seek(0)
sys.stderr.seek(0)
if six.PY2:
stdout = sys.stdout.read()
stderr = sys.stderr.read()
else:
try:
stdout = sys.stdout.read()
stderr = sys.stderr.read()
except UnicodeDecodeError:
sys.stdout.seek(0)
sys.stderr.seek(0)
stdout = sys.stdout.buffer.read()
stderr = sys.stderr.buffer.read()
[six.ensure_text(string) for string in self.accumulated_stderr]
[six.ensure_text(string) for string in self.accumulated_stdout]
stdout = six.ensure_text(get_utf8able_str(stdout))
stderr = six.ensure_text(get_utf8able_str(stderr))
stdout += ''.join(self.accumulated_stdout)
stderr += ''.join(self.accumulated_stderr)
_AttemptToCloseSysFd(sys.stdout)
_AttemptToCloseSysFd(sys.stderr)
sys.stdout = self.stdout_save
sys.stderr = self.stderr_save
os.unlink(self.stdout_file)
os.unlink(self.stderr_file)
_id = six.ensure_text(self.id())
if self.is_debugging and stdout:
print_to_fd('==== stdout {} ====\n'.format(_id), file=sys.stderr)
print_to_fd(stdout, file=sys.stderr)
print_to_fd('==== end stdout ====\n', file=sys.stderr)
if self.is_debugging and stderr:
print_to_fd('==== stderr {} ====\n'.format(_id), file=sys.stderr)
print_to_fd(stderr, file=sys.stderr)
print_to_fd('==== end stderr ====\n', file=sys.stderr)
if self.is_debugging and log_output:
print_to_fd('==== log output {} ====\n'.format(_id),
file=sys.stderr)
print_to_fd(log_output, file=sys.stderr)
print_to_fd('==== end log output ====\n', file=sys.stderr)
def MaybeCheckForAndOfferSoftwareUpdate(self, command_name, debug):
"""Checks the last time we checked for an update and offers one if needed.
Offer is made if the time since the last update check is longer
than the configured threshold offers the user to update gsutil.
Args:
command_name: The name of the command being run.
debug: Debug level to pass in to boto connection (range 0..3).
Returns:
True if the user decides to update.
"""
# Don't try to interact with user if:
# - gsutil is not connected to a tty (e.g., if being run from cron);
# - user is running gsutil -q
# - user is running the config command (which could otherwise attempt to
# check for an update for a user running behind a proxy, who has not yet
# configured gsutil to go through the proxy; for such users we need the
# first connection attempt to be made by the gsutil config command).
# - user is running the version command (which gets run when using
# gsutil -D, which would prevent users with proxy config problems from
# sending us gsutil -D output).
# - user is running the update command (which could otherwise cause an
# additional note that an update is available when user is already trying
# to perform an update);
# - user specified gs_host (which could be a non-production different
# service instance, in which case credentials won't work for checking
# gsutil tarball).
# - user is using a Cloud SDK install (which should only be updated via
# gcloud components update)
logger = logging.getLogger()
if (not system_util.IsRunningInteractively()
or command_name in ('config', 'update', 'ver', 'version')
or not logger.isEnabledFor(logging.INFO)
or HAS_NON_DEFAULT_GS_HOST
or system_util.InvokedViaCloudSdk()):
return False
software_update_check_period = boto.config.getint(
'GSUtil', 'software_update_check_period', 30)
# Setting software_update_check_period to 0 means periodic software
# update checking is disabled.
if software_update_check_period == 0:
return False
last_checked_for_gsutil_update_timestamp_file = (
boto_util.GetLastCheckedForGsutilUpdateTimestampFile())
cur_ts = int(time.time())
if not os.path.isfile(last_checked_for_gsutil_update_timestamp_file):
# Set last_checked_ts from date of VERSION file, so if the user installed
# an old copy of gsutil it will get noticed (and an update offered) the
# first time they try to run it.
last_checked_ts = gslib.GetGsutilVersionModifiedTime()
with open(last_checked_for_gsutil_update_timestamp_file, 'w') as f:
f.write(str(last_checked_ts))
else:
try:
with open(last_checked_for_gsutil_update_timestamp_file,
'r') as f:
last_checked_ts = int(f.readline())
except (TypeError, ValueError):
return False
if (cur_ts - last_checked_ts >
software_update_check_period * SECONDS_PER_DAY):
# Create a credential-less gsutil API to check for the public
# update tarball.
gsutil_api = GcsJsonApi(self.bucket_storage_uri_class,
logger,
DiscardMessagesQueue(),
credentials=NoOpCredentials(),
debug=debug)
cur_ver = gslib.VERSION
try:
cur_ver = LookUpGsutilVersion(gsutil_api, GSUTIL_PUB_TARBALL)
except Exception:
return False
with open(last_checked_for_gsutil_update_timestamp_file, 'w') as f:
f.write(str(cur_ts))
(g, m) = CompareVersions(cur_ver, gslib.VERSION)
if m:
print_to_fd('\n'.join(
textwrap.wrap(
'A newer version of gsutil (%s) is available than the version you '
'are running (%s). NOTE: This is a major new version, so it is '
'strongly recommended that you review the release note details at '
'%s before updating to this version, especially if you use gsutil '
'in scripts.' %
(cur_ver, gslib.VERSION, RELEASE_NOTES_URL))))
if gslib.IS_PACKAGE_INSTALL:
return False
print_to_fd('\n')
answer = input('Would you like to update [y/N]? ')
return answer and answer.lower()[0] == 'y'
elif g:
print_to_fd('\n'.join(
textwrap.wrap(
'A newer version of gsutil (%s) is available than the version you '
'are running (%s). A detailed log of gsutil release changes is '
'available at %s if you would like to read them before updating.'
% (cur_ver, gslib.VERSION, RELEASE_NOTES_URL))))
if gslib.IS_PACKAGE_INSTALL:
return False
print_to_fd('\n')
answer = input('Would you like to update [Y/n]? ')
return not answer or answer.lower()[0] != 'n'
return False
from gslib.utils import system_util, text_util
# pylint: disable=g-import-not-at-top
# This module also imports boto, and will override the UserAgent global variable
# if imported above.
from gslib import metrics
# We parse the options and arguments here so we can pass the results to the user
# agent helper.
try:
opts, args = GetArgumentsAndOptions()
except CommandException as e:
reason = e.reason if e.informational else 'CommandException: %s' % e.reason
err = '%s\n' % reason
try:
text_util.print_to_fd(err, end='', file=sys.stderr)
except UnicodeDecodeError:
# Can happen when outputting invalid Unicode filenames.
sys.stderr.write(err)
if e:
metrics.LogFatalError(e)
sys.exit(1)
# This calculated user agent can be stored for use in StorageV1.
gslib.USER_AGENT = GetUserAgent(args, metrics.MetricsCollector.IsDisabled())
boto.UserAgent += gslib.USER_AGENT
# pylint: disable=g-bad-import-order
import httplib2
import oauth2client
from google_reauth import reauth_creds
def RunCommand(self):
"""Command entry point for the hash command."""
(calc_crc32c, calc_md5, format_func, cloud_format_func,
output_format) = (self._ParseOpts(self.sub_opts, self.logger))
matched_one = False
for url_str in self.args:
for file_ref in self.WildcardIterator(url_str).IterObjects(
bucket_listing_fields=[
'crc32c',
'customerEncryption',
'md5Hash',
'size',
]):
matched_one = True
url = StorageUrlFromString(url_str)
file_name = file_ref.storage_url.object_name
if StorageUrlFromString(url_str).IsFileUrl():
file_size = os.path.getsize(file_name)
self.gsutil_api.status_queue.put(
FileMessage(url,
None,
time.time(),
size=file_size,
finished=False,
message_type=FileMessage.FILE_HASH))
callback_processor = ProgressCallbackWithTimeout(
file_size,
FileProgressCallbackHandler(self.gsutil_api.status_queue,
src_url=StorageUrlFromString(url_str),
operation_name='Hashing').call)
hash_dict = self._GetHashClassesFromArgs(calc_crc32c, calc_md5)
with open(file_name, 'rb') as fp:
hashing_helper.CalculateHashesFromContents(
fp, hash_dict, callback_processor=callback_processor)
self.gsutil_api.status_queue.put(
FileMessage(url,
None,
time.time(),
size=file_size,
finished=True,
message_type=FileMessage.FILE_HASH))
else:
hash_dict = {}
obj_metadata = file_ref.root_object
file_size = obj_metadata.size
md5_present = obj_metadata.md5Hash is not None
crc32c_present = obj_metadata.crc32c is not None
if not md5_present and not crc32c_present:
logging.getLogger().warn('No hashes present for %s', url_str)
continue
if md5_present:
hash_dict['md5'] = obj_metadata.md5Hash
if crc32c_present:
hash_dict['crc32c'] = obj_metadata.crc32c
text_util.print_to_fd('Hashes [%s] for %s:' %
(output_format, file_name))
for name, digest in six.iteritems(hash_dict):
text_util.print_to_fd('\tHash (%s):\t\t%s' %
(name, (format_func(digest) if url.IsFileUrl()
else cloud_format_func(digest))))
if not matched_one:
raise CommandException('No files matched')
_PutToQueueWithTimeout(self.gsutil_api.status_queue,
FinalMessage(time.time()))
return 0
def _PrintPrefixLong(blr):
text_util.print_to_fd('%-33s%s' % ('', blr.url_string.decode(UTF8)))
def RunCommand(self):
"""Command entry point for the ls command."""
got_nomatch_errors = False
got_bucket_nomatch_errors = False
listing_style = ListingStyle.SHORT
get_bucket_info = False
self.recursion_requested = False
self.all_versions = False
self.include_etag = False
self.human_readable = False
self.list_subdir_contents = True
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-a':
self.all_versions = True
elif o == '-e':
self.include_etag = True
elif o == '-b':
get_bucket_info = True
elif o == '-h':
self.human_readable = True
elif o == '-l':
listing_style = ListingStyle.LONG
elif o == '-L':
listing_style = ListingStyle.LONG_LONG
elif o == '-p':
# Project IDs are sent as header values when using gs and s3 XML APIs.
InsistAscii(a, 'Invalid non-ASCII character found in project ID')
self.project_id = a
elif o == '-r' or o == '-R':
self.recursion_requested = True
elif o == '-d':
self.list_subdir_contents = False
if not self.args:
# default to listing all gs buckets
self.args = ['gs://']
total_objs = 0
total_bytes = 0
def MaybePrintBucketHeader(blr):
if len(self.args) > 1:
text_util.print_to_fd('%s:' % blr.url_string.decode(UTF8))
print_bucket_header = MaybePrintBucketHeader
for url_str in self.args:
storage_url = StorageUrlFromString(url_str)
if storage_url.IsFileUrl():
raise CommandException('Only cloud URLs are supported for %s' %
self.command_name)
bucket_fields = None
if (listing_style == ListingStyle.SHORT or
listing_style == ListingStyle.LONG):
bucket_fields = ['id']
elif listing_style == ListingStyle.LONG_LONG:
bucket_fields = [
'acl',
'billing',
'cors',
'defaultObjectAcl',
'encryption',
'iamConfiguration',
'labels',
'location',
'logging',
'lifecycle',
'metageneration',
'retentionPolicy',
'defaultEventBasedHold',
'storageClass',
'timeCreated',
'updated',
'versioning',
'website',
]
if storage_url.IsProvider():
# Provider URL: use bucket wildcard to list buckets.
for blr in self.WildcardIterator(
'%s://*' %
storage_url.scheme).IterBuckets(bucket_fields=bucket_fields):
self._PrintBucketInfo(blr, listing_style)
elif storage_url.IsBucket() and get_bucket_info:
# ls -b bucket listing request: List info about bucket(s).
total_buckets = 0
for blr in self.WildcardIterator(url_str).IterBuckets(
bucket_fields=bucket_fields):
if not ContainsWildcard(url_str) and not blr.root_object:
# Iterator does not make an HTTP call for non-wildcarded
# listings with fields=='id'. Ensure the bucket exists by calling
# GetBucket.
self.gsutil_api.GetBucket(blr.storage_url.bucket_name,
fields=['id'],
provider=storage_url.scheme)
self._PrintBucketInfo(blr, listing_style)
total_buckets += 1
if not ContainsWildcard(url_str) and not total_buckets:
got_bucket_nomatch_errors = True
else:
# URL names a bucket, object, or object subdir ->
# list matching object(s) / subdirs.
def _PrintPrefixLong(blr):
text_util.print_to_fd('%-33s%s' % ('', blr.url_string.decode(UTF8)))
if listing_style == ListingStyle.SHORT:
# ls helper by default readies us for a short listing.
listing_helper = LsHelper(
self.WildcardIterator,
self.logger,
all_versions=self.all_versions,
print_bucket_header_func=print_bucket_header,
should_recurse=self.recursion_requested,
list_subdir_contents=self.list_subdir_contents)
elif listing_style == ListingStyle.LONG:
bucket_listing_fields = [
'name',
'size',
'timeCreated',
'updated',
]
if self.all_versions:
bucket_listing_fields.extend([
'generation',
'metageneration',
])
if self.include_etag:
bucket_listing_fields.append('etag')
listing_helper = LsHelper(
self.WildcardIterator,
self.logger,
print_object_func=self._PrintLongListing,
print_dir_func=_PrintPrefixLong,
print_bucket_header_func=print_bucket_header,
all_versions=self.all_versions,
should_recurse=self.recursion_requested,
fields=bucket_listing_fields,
list_subdir_contents=self.list_subdir_contents)
elif listing_style == ListingStyle.LONG_LONG:
# List all fields
bucket_listing_fields = (UNENCRYPTED_FULL_LISTING_FIELDS +
ENCRYPTED_FIELDS)
listing_helper = LsHelper(
self.WildcardIterator,
self.logger,
print_object_func=PrintFullInfoAboutObject,
print_dir_func=_PrintPrefixLong,
print_bucket_header_func=print_bucket_header,
all_versions=self.all_versions,
should_recurse=self.recursion_requested,
fields=bucket_listing_fields,
list_subdir_contents=self.list_subdir_contents)
else:
raise CommandException('Unknown listing style: %s' % listing_style)
exp_dirs, exp_objs, exp_bytes = (
listing_helper.ExpandUrlAndPrint(storage_url))
if storage_url.IsObject() and exp_objs == 0 and exp_dirs == 0:
got_nomatch_errors = True
total_bytes += exp_bytes
total_objs += exp_objs
if total_objs and listing_style != ListingStyle.SHORT:
text_util.print_to_fd(
'TOTAL: %d objects, %d bytes (%s)' %
(total_objs, total_bytes, MakeHumanReadable(float(total_bytes))))
if got_nomatch_errors:
raise CommandException('One or more URLs matched no objects.')
if got_bucket_nomatch_errors:
raise NotFoundException('One or more bucket URLs matched no buckets.')
return 0
def MaybePrintBucketHeader(blr):
if len(self.args) > 1:
text_util.print_to_fd('%s:' % blr.url_string.decode(UTF8))
def PrintNewLine(): """Default function for printing new lines between directories.""" text_util.print_to_fd()
