def testPluralityCheckableIteratorWith2Exceptions(self):
"""Tests PluralityCheckableIterator with 2 elements that both raise."""
class IterTest(six.Iterator):
def __init__(self):
self.position = 0
def __iter__(self):
return self
def __next__(self):
if self.position < 2:
self.position += 1
raise CustomTestException('Test exception %s' %
self.position)
else:
raise StopIteration()
pcit = PluralityCheckableIterator(IterTest())
try:
pcit.PeekException()
self.fail('Expected exception 1 from PeekException')
except CustomTestException as e:
self.assertIn(str(e), 'Test exception 1')
try:
for _ in pcit:
pass
self.fail('Expected exception 1 from iterator')
except CustomTestException as e:
self.assertIn(str(e), 'Test exception 1')
try:
pcit.PeekException()
self.fail('Expected exception 2 from PeekException')
except CustomTestException as e:
self.assertIn(str(e), 'Test exception 2')
try:
for _ in pcit:
pass
self.fail('Expected exception 2 from iterator')
except CustomTestException as e:
self.assertIn(str(e), 'Test exception 2')
for _ in pcit:
self.fail('Expected StopIteration')
def testPluralityCheckableIteratorWith2Exceptions(self):
"""Tests PluralityCheckableIterator with 2 elements that both raise."""
class IterTest(object):
def __init__(self):
self.position = 0
def __iter__(self):
return self
def next(self):
if self.position < 2:
self.position += 1
raise CustomTestException('Test exception %s' %
self.position)
else:
raise StopIteration()
pcit = PluralityCheckableIterator(IterTest())
try:
pcit.PeekException()
self.fail('Expected exception 1 from PeekException')
except CustomTestException, e:
self.assertIn(e.message, 'Test exception 1')
def testPluralityCheckableIteratorReadsAheadAsNeeded(self):
"""Tests that the PCI does not unnecessarily read new elements."""
class IterTest(object):
def __init__(self):
self.position = 0
def __iter__(self):
return self
def next(self):
if self.position == 3:
raise StopIteration()
self.position += 1
# IsEmpty and PeekException should retrieve only 1 element from the
# underlying iterator.
pcit = PluralityCheckableIterator(IterTest())
pcit.IsEmpty()
pcit.PeekException()
self.assertEquals(pcit.orig_iterator.position, 1)
# HasPlurality requires populating 2 elements into the iterator.
pcit.HasPlurality()
self.assertEquals(pcit.orig_iterator.position, 2)
# next should yield already-populated elements without advancing the
# iterator.
pcit.next() # Yields element 1
self.assertEquals(pcit.orig_iterator.position, 2)
pcit.next() # Yields element 2
self.assertEquals(pcit.orig_iterator.position, 2)
pcit.next() # Yields element 3
self.assertEquals(pcit.orig_iterator.position, 3)
try:
pcit.next() # Underlying iterator is empty
self.fail('Expected StopIteration')
except StopIteration:
pass
def ExpandUrlAndPrint(self, url):
"""Iterates over the given URL and calls print functions.
Args:
url: StorageUrl to iterate over.
Returns:
(num_objects, num_bytes) total number of objects and bytes iterated.
"""
num_objects = 0
num_dirs = 0
num_bytes = 0
print_newline = False
if url.IsBucket() or self.should_recurse:
# IsBucket() implies a top-level listing.
if url.IsBucket():
self._print_bucket_header_func(url)
return self._RecurseExpandUrlAndPrint(url.url_string,
print_initial_newline=False)
else:
# User provided a prefix or object URL, but it's impossible to tell
# which until we do a listing and see what matches.
top_level_iterator = PluralityCheckableIterator(
self._iterator_func(
url.CreatePrefixUrl(wildcard_suffix=None),
all_versions=self.all_versions).IterAll(
expand_top_level_buckets=True,
bucket_listing_fields=self.bucket_listing_fields))
plurality = top_level_iterator.HasPlurality()
try:
top_level_iterator.PeekException()
except EncryptionException:
# Detailed listing on a single object can perform a GetObjectMetadata
# call, which raises if a matching encryption key isn't found.
# Re-iterate without requesting encrypted fields.
top_level_iterator = PluralityCheckableIterator(
self._iterator_func(
url.CreatePrefixUrl(wildcard_suffix=None),
all_versions=self.all_versions).
IterAll(
expand_top_level_buckets=True,
bucket_listing_fields=UNENCRYPTED_FULL_LISTING_FIELDS))
plurality = top_level_iterator.HasPlurality()
for blr in top_level_iterator:
if self._MatchesExcludedPattern(blr):
continue
if blr.IsObject():
nd = 0
no, nb = self._print_object_func(blr)
print_newline = True
elif blr.IsPrefix():
if print_newline:
self._print_newline_func()
else:
print_newline = True
if plurality and self.list_subdir_contents:
self._print_dir_header_func(blr)
elif plurality and not self.list_subdir_contents:
print_newline = False
expansion_url_str = StorageUrlFromString(
blr.url_string).CreatePrefixUrl(
wildcard_suffix='*' if self.
list_subdir_contents else None)
nd, no, nb = self._RecurseExpandUrlAndPrint(
expansion_url_str)
self._print_dir_summary_func(nb, blr)
else:
# We handle all buckets at the top level, so this should never happen.
raise CommandException(
'Sub-level iterator returned a CsBucketListingRef of type Bucket'
)
num_objects += no
num_dirs += nd
num_bytes += nb
return num_dirs, num_objects, num_bytes
