def _RecursePrint(self, blr):
"""
Expands a bucket listing reference and recurses to its children, calling
_PrintInfoAboutBucketListingRef for each expanded object found.
Args:
blr: An instance of BucketListingRef.
Returns:
Tuple containing (number of object, total number of bytes)
"""
num_bytes = 0
num_objs = 0
if blr.HasKey():
blr_iterator = iter([blr])
elif blr.HasPrefix():
blr_iterator = self.WildcardIterator(
'%s/*' % blr.GetRStrippedUriString(), all_versions=self.all_versions)
elif blr.NamesBucket():
blr_iterator = self.WildcardIterator(
'%s*' % blr.GetUriString(), all_versions=self.all_versions)
else:
# This BLR didn't come from a bucket listing. This case happens for
# BLR's instantiated from a user-provided URI.
blr_iterator = PluralityCheckableIterator(
UriOnlyBlrExpansionIterator(
self, blr, all_versions=self.all_versions))
if blr_iterator.is_empty() and not ContainsWildcard(blr.GetUriString()):
raise CommandException('No such object %s' % blr.GetUriString())
for cur_blr in blr_iterator:
if self.exclude_patterns:
tomatch = cur_blr.GetUriString()
skip = False
for pattern in self.exclude_patterns:
if fnmatch.fnmatch(tomatch, pattern):
skip = True
break
if skip:
continue
if cur_blr.HasKey():
# Object listing.
no, nb = self._PrintInfoAboutBucketListingRef(cur_blr)
else:
# Subdir listing.
if cur_blr.GetUriString().endswith('//'):
# Expand gs://bucket// into gs://bucket//* so we don't infinite
# loop. This case happens when user has uploaded an object whose
# name begins with a /.
cur_blr = BucketListingRef(self.suri_builder.StorageUri(
'%s*' % cur_blr.GetUriString()), None, None, cur_blr.headers)
no, nb = self._RecursePrint(cur_blr)
num_bytes += nb
num_objs += no
if blr.HasPrefix() and not self.summary_only:
self._PrintSummaryLine(num_bytes, blr.GetUriString().encode('utf-8'))
return num_objs, num_bytes
def testPluralityCheckableIteratorWith3Elems(self): """Tests PluralityCheckableIterator with 3 elements.""" input_list = range(3) it = iter(input_list) pcit = PluralityCheckableIterator(it) self.assertFalse(pcit.is_empty()) self.assertTrue(pcit.has_plurality()) output_list = list(pcit) self.assertEqual(input_list, output_list)
def testPluralityCheckableIteratorWith3Elems(self):
"""Tests PluralityCheckableIterator with 3 elements."""
input_list = range(3)
it = iter(input_list)
pcit = PluralityCheckableIterator(it)
self.assertFalse(pcit.is_empty())
self.assertTrue(pcit.has_plurality())
output_list = list(pcit)
self.assertEqual(input_list, output_list)
def __iter__(self):
for blr in self.blr_iter:
uri = blr.GetUri()
if uri.names_object():
# URI could be a bucket subdir.
implicit_subdir_iterator = PluralityCheckableIterator(
self.name_expansion_instance._WildcardIterator(
self.name_expansion_instance.suri_builder.StorageUri(
'%s/%s' %
(uri.uri.rstrip('/'), self.name_expansion_instance.
_flatness_wildcard[self.flat]))))
if not implicit_subdir_iterator.is_empty():
for exp_blr in implicit_subdir_iterator:
yield (True, exp_blr)
else:
yield (False, blr)
else:
yield (False, blr)
def __iter__(self):
for blr in self.blr_iter:
uri = blr.GetUri()
if uri.names_object():
# URI could be a bucket subdir.
implicit_subdir_iterator = PluralityCheckableIterator(
self.name_expansion_instance._WildcardIterator(
self.name_expansion_instance.suri_builder.StorageUri(
'%s/%s' % (uri.uri.rstrip('/'),
self.name_expansion_instance._flatness_wildcard[
self.flat]))))
if not implicit_subdir_iterator.is_empty():
for exp_blr in implicit_subdir_iterator:
yield (True, exp_blr)
else:
yield (False, blr)
else:
yield (False, blr)
def NameExpansionIterator(command_name, proj_id_handler, headers, debug,
bucket_storage_uri_class, uri_strs,
recursion_requested,
have_existing_dst_container=None, flat=True):
"""
Static factory function for instantiating _NameExpansionIterator, which
wraps the resulting iterator in a PluralityCheckableIterator and checks
that it is non-empty.
Args are as documented in constructor for _NameExpansionIterator class.
"""
name_expansion_iterator = _NameExpansionIterator(
command_name, proj_id_handler, headers, debug, bucket_storage_uri_class,
uri_strs, recursion_requested, have_existing_dst_container, flat)
name_expansion_iterator = PluralityCheckableIterator(name_expansion_iterator)
if name_expansion_iterator.is_empty():
raise CommandException('No URIs matched')
return name_expansion_iterator
def NameExpansionIterator(command_name,
proj_id_handler,
headers,
debug,
bucket_storage_uri_class,
uri_strs,
recursion_requested,
have_existing_dst_container=None,
flat=True,
all_versions=False,
for_all_version_delete=False):
"""
Static factory function for instantiating _NameExpansionIterator, which
wraps the resulting iterator in a PluralityCheckableIterator and checks
that it is non-empty. Also, allows uri_strs can be either an array or an
iterator.
Args:
command_name: name of command being run.
proj_id_handler: ProjectIdHandler to use for current command.
headers: Dictionary containing optional HTTP headers to pass to boto.
debug: Debug level to pass in to boto connection (range 0..3).
bucket_storage_uri_class: Class to instantiate for cloud StorageUris.
Settable for testing/mocking.
uri_strs: PluralityCheckableIterator of URI strings needing expansion.
recursion_requested: True if -R specified on command-line.
have_existing_dst_container: Bool indicator whether this is a copy
request to an existing bucket, bucket subdir, or directory. Default
None value should be used in cases where this is not needed (commands
other than cp).
flat: Bool indicating whether bucket listings should be flattened, i.e.,
so the mapped-to results contain objects spanning subdirectories.
all_versions: Bool indicating whether to iterate over all object versions.
for_all_version_delete: Bool indicating whether this is for an all-version
delete.
Examples of ExpandWildcardsAndContainers with flat=True:
- Calling with one of the uri_strs being 'gs://bucket' will enumerate all
top-level objects, as will 'gs://bucket/' and 'gs://bucket/*'.
- 'gs://bucket/**' will enumerate all objects in the bucket.
- 'gs://bucket/abc' will enumerate all next-level objects under directory
abc (i.e., not including subdirectories of abc) if gs://bucket/abc/*
matches any objects; otherwise it will enumerate the single name
gs://bucket/abc
- 'gs://bucket/abc/**' will enumerate all objects under abc or any of its
subdirectories.
- 'file:///tmp' will enumerate all files under /tmp, as will
'file:///tmp/*'
- 'file:///tmp/**' will enumerate all files under /tmp or any of its
subdirectories.
Example if flat=False: calling with gs://bucket/abc/* lists matching objects
or subdirs, but not sub-subdirs or objects beneath subdirs.
Note: In step-by-step comments below we give examples assuming there's a
gs://bucket with object paths:
abcd/o1.txt
abcd/o2.txt
xyz/o1.txt
xyz/o2.txt
and a directory file://dir with file paths:
dir/a.txt
dir/b.txt
dir/c/
"""
uri_strs = PluralityCheckableIterator(uri_strs)
name_expansion_iterator = _NameExpansionIterator(
command_name,
proj_id_handler,
headers,
debug,
bucket_storage_uri_class,
uri_strs,
recursion_requested,
have_existing_dst_container,
flat,
all_versions=all_versions,
for_all_version_delete=for_all_version_delete)
name_expansion_iterator = PluralityCheckableIterator(
name_expansion_iterator)
if name_expansion_iterator.is_empty():
raise CommandException('No URIs matched')
return name_expansion_iterator
def __iter__(self):
for uri_str in self.uri_strs:
# Step 1: Expand any explicitly specified wildcards. The output from this
# step is an iterator of BucketListingRef.
# Starting with gs://buck*/abc* this step would expand to gs://bucket/abcd
if ContainsWildcard(uri_str):
post_step1_iter = self._WildcardIterator(uri_str)
else:
suri = self.suri_builder.StorageUri(uri_str)
post_step1_iter = iter([BucketListingRef(suri)])
post_step1_iter = PluralityCheckableIterator(post_step1_iter)
# Step 2: Expand bucket subdirs and versions. The output from this
# step is an iterator of (names_container, BucketListingRef).
# Starting with gs://bucket/abcd this step would expand to:
# iter([(True, abcd/o1.txt), (True, abcd/o2.txt)]).
if self.flat and self.recursion_requested:
post_step2_iter = _ImplicitBucketSubdirIterator(
self, post_step1_iter, self.flat)
elif self.all_versions:
post_step2_iter = _AllVersionIterator(self,
post_step1_iter,
headers=self.headers)
else:
post_step2_iter = _NonContainerTuplifyIterator(post_step1_iter)
post_step2_iter = PluralityCheckableIterator(post_step2_iter)
# Step 3. Expand directories and buckets. This step yields the iterated
# values. Starting with gs://bucket this step would expand to:
# [abcd/o1.txt, abcd/o2.txt, xyz/o1.txt, xyz/o2.txt]
# Starting with file://dir this step would expand to:
# [dir/a.txt, dir/b.txt, dir/c/]
exp_src_bucket_listing_refs = []
wc = self._flatness_wildcard[self.flat]
src_uri_expands_to_multi = (post_step1_iter.has_plurality()
or post_step2_iter.has_plurality())
is_multi_src_request = (self.uri_strs.has_plurality()
or src_uri_expands_to_multi)
if post_step2_iter.is_empty():
raise CommandException('No URIs matched: %s' % uri_str)
for (names_container, blr) in post_step2_iter:
if (not blr.GetUri().names_container()
and (self.flat or not blr.HasPrefix())):
yield NameExpansionResult(uri_str,
is_multi_src_request,
src_uri_expands_to_multi,
names_container,
blr.GetUriString(),
self.have_existing_dst_container,
is_latest=blr.IsLatest())
continue
if not self.recursion_requested:
if blr.GetUri().is_file_uri():
desc = 'directory'
else:
desc = 'bucket'
print 'Omitting %s "%s". (Did you mean to do %s -R?)' % (
desc, blr.GetUri(), self.command_name)
continue
if blr.GetUri().is_file_uri():
# Convert dir to implicit recursive wildcard.
uri_to_iterate = '%s/%s' % (blr.GetUriString(), wc)
else:
# Convert bucket to implicit recursive wildcard.
uri_to_iterate = blr.GetUri().clone_replace_name(wc)
wc_iter = PluralityCheckableIterator(
self._WildcardIterator(uri_to_iterate))
src_uri_expands_to_multi = (src_uri_expands_to_multi
or wc_iter.has_plurality())
is_multi_src_request = (self.uri_strs.has_plurality()
or src_uri_expands_to_multi)
for blr in wc_iter:
yield NameExpansionResult(uri_str,
is_multi_src_request,
src_uri_expands_to_multi,
True,
blr.GetUriString(),
self.have_existing_dst_container,
is_latest=blr.IsLatest())
def NameExpansionIterator(command_name, proj_id_handler, headers, debug,
logger, bucket_storage_uri_class, uri_strs,
recursion_requested,
have_existing_dst_container=None, flat=True,
all_versions=False,
for_all_version_delete=False,
cmd_supports_recursion=True):
"""
Static factory function for instantiating _NameExpansionIterator, which
wraps the resulting iterator in a PluralityCheckableIterator and checks
that it is non-empty. Also, allows uri_strs can be either an array or an
iterator.
Args:
command_name: name of command being run.
proj_id_handler: ProjectIdHandler to use for current command.
headers: Dictionary containing optional HTTP headers to pass to boto.
debug: Debug level to pass in to boto connection (range 0..3).
logger: logging.Logger object.
bucket_storage_uri_class: Class to instantiate for cloud StorageUris.
Settable for testing/mocking.
uri_strs: PluralityCheckableIterator of URI strings needing expansion.
recursion_requested: True if -R specified on command-line.
have_existing_dst_container: Bool indicator whether this is a copy
request to an existing bucket, bucket subdir, or directory. Default
None value should be used in cases where this is not needed (commands
other than cp).
flat: Bool indicating whether bucket listings should be flattened, i.e.,
so the mapped-to results contain objects spanning subdirectories.
all_versions: Bool indicating whether to iterate over all object versions.
for_all_version_delete: Bool indicating whether this is for an all-version
delete.
cmd_supports_recursion: Bool indicating whether this command supports a '-R'
flag. Useful for printing helpful error messages.
Examples of ExpandWildcardsAndContainers with flat=True:
- Calling with one of the uri_strs being 'gs://bucket' will enumerate all
top-level objects, as will 'gs://bucket/' and 'gs://bucket/*'.
- 'gs://bucket/**' will enumerate all objects in the bucket.
- 'gs://bucket/abc' will enumerate all next-level objects under directory
abc (i.e., not including subdirectories of abc) if gs://bucket/abc/*
matches any objects; otherwise it will enumerate the single name
gs://bucket/abc
- 'gs://bucket/abc/**' will enumerate all objects under abc or any of its
subdirectories.
- 'file:///tmp' will enumerate all files under /tmp, as will
'file:///tmp/*'
- 'file:///tmp/**' will enumerate all files under /tmp or any of its
subdirectories.
Example if flat=False: calling with gs://bucket/abc/* lists matching objects
or subdirs, but not sub-subdirs or objects beneath subdirs.
Note: In step-by-step comments below we give examples assuming there's a
gs://bucket with object paths:
abcd/o1.txt
abcd/o2.txt
xyz/o1.txt
xyz/o2.txt
and a directory file://dir with file paths:
dir/a.txt
dir/b.txt
dir/c/
"""
uri_strs = PluralityCheckableIterator(uri_strs)
name_expansion_iterator = _NameExpansionIterator(
command_name, proj_id_handler, headers, debug, logger,
bucket_storage_uri_class, uri_strs, recursion_requested,
have_existing_dst_container, flat, all_versions=all_versions,
for_all_version_delete=for_all_version_delete,
cmd_supports_recursion=cmd_supports_recursion)
name_expansion_iterator = PluralityCheckableIterator(name_expansion_iterator)
if name_expansion_iterator.is_empty():
raise CommandException('No URIs matched')
return name_expansion_iterator
def __iter__(self):
for uri_str in self.uri_strs:
# Step 1: Expand any explicitly specified wildcards. The output from this
# step is an iterator of BucketListingRef.
# Starting with gs://buck*/abc* this step would expand to gs://bucket/abcd
if ContainsWildcard(uri_str):
post_step1_iter = self._WildcardIterator(uri_str)
else:
suri = self.suri_builder.StorageUri(uri_str)
post_step1_iter = iter([BucketListingRef(suri)])
post_step1_iter = PluralityCheckableIterator(post_step1_iter)
# Step 2: Expand bucket subdirs and versions. The output from this
# step is an iterator of (names_container, BucketListingRef).
# Starting with gs://bucket/abcd this step would expand to:
# iter([(True, abcd/o1.txt), (True, abcd/o2.txt)]).
if self.flat and self.recursion_requested:
post_step2_iter = _ImplicitBucketSubdirIterator(self,
post_step1_iter, self.flat)
elif self.all_versions:
post_step2_iter = _AllVersionIterator(self, post_step1_iter,
headers=self.headers)
else:
post_step2_iter = _NonContainerTuplifyIterator(post_step1_iter)
post_step2_iter = PluralityCheckableIterator(post_step2_iter)
# Step 3. Expand directories and buckets. This step yields the iterated
# values. Starting with gs://bucket this step would expand to:
# [abcd/o1.txt, abcd/o2.txt, xyz/o1.txt, xyz/o2.txt]
# Starting with file://dir this step would expand to:
# [dir/a.txt, dir/b.txt, dir/c/]
exp_src_bucket_listing_refs = []
wc = self._flatness_wildcard[self.flat]
src_uri_expands_to_multi = (post_step1_iter.has_plurality()
or post_step2_iter.has_plurality())
is_multi_src_request = (self.uri_strs.has_plurality
or src_uri_expands_to_multi)
if post_step2_iter.is_empty():
raise CommandException('No URIs matched: %s' % uri_str)
for (names_container, blr) in post_step2_iter:
if (not blr.GetUri().names_container()
and (self.flat or not blr.HasPrefix())):
yield NameExpansionResult(uri_str, is_multi_src_request,
src_uri_expands_to_multi, names_container,
blr.GetUriString(),
self.have_existing_dst_container,
is_latest=blr.IsLatest())
continue
if not self.recursion_requested:
if blr.GetUri().is_file_uri():
desc = 'directory'
elif blr.GetUri().names_bucket():
desc = 'bucket'
else:
desc = 'bucket subdir'
if self.cmd_supports_recursion:
self.logger.info(
'Omitting %s "%s". (Did you mean to do %s -R?)',
desc, blr.GetUri(), self.command_name)
else:
self.logger.info('Omitting %s "%s".', desc, blr.GetUri())
continue
if blr.GetUri().is_file_uri():
# Convert dir to implicit recursive wildcard.
uri_to_iterate = '%s/%s' % (blr.GetUriString(), wc)
else:
# Convert bucket to implicit recursive wildcard.
uri_to_iterate = blr.GetUri().clone_replace_name(wc)
wc_iter = PluralityCheckableIterator(
self._WildcardIterator(uri_to_iterate))
src_uri_expands_to_multi = (src_uri_expands_to_multi
or wc_iter.has_plurality())
is_multi_src_request = (self.uri_strs.has_plurality
or src_uri_expands_to_multi)
for blr in wc_iter:
yield NameExpansionResult(uri_str, is_multi_src_request,
src_uri_expands_to_multi, True,
blr.GetUriString(),
self.have_existing_dst_container,
is_latest=blr.IsLatest())
def _ExpandUriAndPrintInfo(self, uri, listing_style, should_recurse=False):
"""
Expands wildcards and directories/buckets for uri as needed, and
calls _PrintInfoAboutBucketListingRef() on each.
Args:
uri: StorageUri being listed.
listing_style: ListingStyle enum describing type of output desired.
should_recurse: bool indicator of whether to expand recursively.
Returns:
Tuple (number of matching objects, number of bytes across these objects).
"""
# We do a two-level loop, with the outer loop iterating level-by-level from
# blrs_to_expand, and the inner loop iterating the matches at the current
# level, printing them, and adding any new subdirs that need expanding to
# blrs_to_expand (to be picked up in the next outer loop iteration).
blrs_to_expand = [BucketListingRef(uri)]
num_objs = 0
num_bytes = 0
expanding_top_level = True
printed_one = False
num_expanded_blrs = 0
while len(blrs_to_expand):
if printed_one:
print
blr = blrs_to_expand.pop(0)
if blr.HasKey():
blr_iterator = iter([blr])
elif blr.HasPrefix():
# Bucket subdir from a previous iteration. Print "header" line only if
# we're listing more than one subdir (or if it's a recursive listing),
# to be consistent with the way UNIX ls works.
if num_expanded_blrs > 1 or should_recurse:
print '%s:' % blr.GetUriString().encode('utf-8')
printed_one = True
blr_iterator = self.WildcardIterator('%s/*' %
blr.GetRStrippedUriString(),
all_versions=self.all_versions)
elif blr.NamesBucket():
blr_iterator = self.WildcardIterator('%s*' % blr.GetUriString(),
all_versions=self.all_versions)
else:
# This BLR didn't come from a bucket listing. This case happens for
# BLR's instantiated from a user-provided URI.
blr_iterator = PluralityCheckableIterator(
_UriOnlyBlrExpansionIterator(
self, blr, all_versions=self.all_versions))
if blr_iterator.is_empty() and not ContainsWildcard(uri):
raise CommandException('No such object %s' % uri)
for cur_blr in blr_iterator:
num_expanded_blrs = num_expanded_blrs + 1
if cur_blr.HasKey():
# Object listing.
(no, nb) = self._PrintInfoAboutBucketListingRef(
cur_blr, listing_style)
num_objs += no
num_bytes += nb
printed_one = True
else:
# Subdir listing. If we're at the top level of a bucket subdir
# listing don't print the list here (corresponding to how UNIX ls
# dir just prints its contents, not the name followed by its
# contents).
if (expanding_top_level and not uri.names_bucket()) or should_recurse:
if cur_blr.GetUriString().endswith('//'):
# Expand gs://bucket// into gs://bucket//* so we don't infinite
# loop. This case happens when user has uploaded an object whose
# name begins with a /.
cur_blr = BucketListingRef(self.suri_builder.StorageUri(
'%s*' % cur_blr.GetUriString()), None, None, cur_blr.headers)
blrs_to_expand.append(cur_blr)
# Don't include the subdir name in the output if we're doing a
# recursive listing, as it will be printed as 'subdir:' when we get
# to the prefix expansion, the next iteration of the main loop.
else:
if listing_style == ListingStyle.LONG:
print '%-33s%s' % (
'', cur_blr.GetUriString().encode('utf-8'))
else:
print cur_blr.GetUriString().encode('utf-8')
expanding_top_level = False
return (num_objs, num_bytes)
def _ExpandUriAndPrintInfo(self, uri, listing_style, should_recurse=False):
"""
Expands wildcards and directories/buckets for uri as needed, and
calls _PrintInfoAboutBucketListingRef() on each.
Args:
uri: StorageUri being listed.
listing_style: ListingStyle enum describing type of output desired.
should_recurse: bool indicator of whether to expand recursively.
Returns:
Tuple (number of matching objects, number of bytes across these objects).
"""
# We do a two-level loop, with the outer loop iterating level-by-level from
# blrs_to_expand, and the inner loop iterating the matches at the current
# level, printing them, and adding any new subdirs that need expanding to
# blrs_to_expand (to be picked up in the next outer loop iteration).
blrs_to_expand = [BucketListingRef(uri)]
num_objs = 0
num_bytes = 0
expanding_top_level = True
printed_one = False
num_expanded_blrs = 0
while len(blrs_to_expand):
if printed_one:
print
blr = blrs_to_expand.pop(0)
if blr.HasKey():
blr_iterator = iter([blr])
elif blr.HasPrefix():
# Bucket subdir from a previous iteration. Print "header" line only if
# we're listing more than one subdir (or if it's a recursive listing),
# to be consistent with the way UNIX ls works.
if num_expanded_blrs > 1 or should_recurse:
print '%s:' % blr.GetUriString().encode('utf-8')
printed_one = True
blr_iterator = self.WildcardIterator(
'%s/*' % blr.GetRStrippedUriString())
elif blr.NamesBucket():
blr_iterator = self.WildcardIterator('%s*' %
blr.GetUriString())
else:
# This BLR didn't come from a bucket listing. This case happens for
# BLR's instantiated from a user-provided URI.
blr_iterator = PluralityCheckableIterator(
_UriOnlyBlrExpansionIterator(self, blr))
if blr_iterator.is_empty() and not ContainsWildcard(uri):
raise CommandException('No such object %s' % uri)
for cur_blr in blr_iterator:
num_expanded_blrs = num_expanded_blrs + 1
if cur_blr.HasKey():
# Object listing.
(no, nb) = self._PrintInfoAboutBucketListingRef(
cur_blr, listing_style)
num_objs += no
num_bytes += nb
printed_one = True
else:
# Subdir listing. If we're at the top level of a bucket subdir
# listing don't print the list here (corresponding to how UNIX ls
# dir just prints its contents, not the name followed by its
# contents).
if (expanding_top_level
and not uri.names_bucket()) or should_recurse:
if cur_blr.GetUriString().endswith('//'):
# Expand gs://bucket// into gs://bucket//* so we don't infinite
# loop. This case happens when user has uploaded an object whose
# name begins with a /.
cur_blr = BucketListingRef(
self.suri_builder.StorageUri(
'%s*' % cur_blr.GetUriString()), None,
None, cur_blr.headers)
blrs_to_expand.append(cur_blr)
# Don't include the subdir name in the output if we're doing a
# recursive listing, as it will be printed as 'subdir:' when we get
# to the prefix expansion, the next iteration of the main loop.
else:
if listing_style == ListingStyle.LONG:
print '%-33s%s' % (
'', cur_blr.GetUriString().encode('utf-8'))
else:
print cur_blr.GetUriString().encode('utf-8')
expanding_top_level = False
return (num_objs, num_bytes)
def _RecursePrint(self, blr):
"""
Expands a bucket listing reference and recurses to its children, calling
_PrintInfoAboutBucketListingRef for each expanded object found.
Args:
blr: An instance of BucketListingRef.
Returns:
Tuple containing (number of object, total number of bytes)
"""
num_bytes = 0
num_objs = 0
if blr.HasKey():
blr_iterator = iter([blr])
elif blr.HasPrefix():
blr_iterator = self.WildcardIterator(
'%s/*' % blr.GetRStrippedUriString(),
all_versions=self.all_versions)
elif blr.NamesBucket():
blr_iterator = self.WildcardIterator(
'%s*' % blr.GetUriString(), all_versions=self.all_versions)
else:
# This BLR didn't come from a bucket listing. This case happens for
# BLR's instantiated from a user-provided URI.
blr_iterator = PluralityCheckableIterator(
UriOnlyBlrExpansionIterator(self,
blr,
all_versions=self.all_versions))
if blr_iterator.is_empty() and not ContainsWildcard(
blr.GetUriString()):
raise CommandException('No such object %s' %
blr.GetUriString())
for cur_blr in blr_iterator:
if self.exclude_patterns:
tomatch = cur_blr.GetUriString()
skip = False
for pattern in self.exclude_patterns:
if fnmatch.fnmatch(tomatch, pattern):
skip = True
break
if skip:
continue
if cur_blr.HasKey():
# Object listing.
no, nb = self._PrintInfoAboutBucketListingRef(cur_blr)
else:
# Subdir listing.
if cur_blr.GetUriString().endswith('//'):
# Expand gs://bucket// into gs://bucket//* so we don't infinite
# loop. This case happens when user has uploaded an object whose
# name begins with a /.
cur_blr = BucketListingRef(
self.suri_builder.StorageUri(
'%s*' % cur_blr.GetUriString()), None, None,
cur_blr.headers)
no, nb = self._RecursePrint(cur_blr)
num_bytes += nb
num_objs += no
if blr.HasPrefix() and not self.summary_only:
self._PrintSummaryLine(num_bytes,
blr.GetUriString().encode('utf-8'))
return num_objs, num_bytes
