def test_recursive_bucket_rm_with_wildcarding(self):
"""Tests removing all objects and buckets matching a bucket wildcard."""
buri_base = 'gsutil-test-%s' % self.GetTestMethodName()
buri_base = buri_base[:MAX_BUCKET_LENGTH - 20]
buri_base = '%s-%s' % (buri_base, self.MakeRandomTestString())
buri_base = 'aaa-' + buri_base
buri_base = util.MakeBucketNameValid(buri_base)
buri1 = self.CreateBucket(bucket_name='%s-tbuck1' % buri_base)
buri2 = self.CreateBucket(bucket_name='%s-tbuck2' % buri_base)
buri3 = self.CreateBucket(bucket_name='%s-tb3' % buri_base)
ouri1 = self.CreateObject(bucket_uri=buri1, object_name='o1', contents=b'z')
ouri2 = self.CreateObject(bucket_uri=buri2, object_name='o2', contents=b'z')
self.CreateObject(bucket_uri=buri3, object_name='o3', contents=b'z')
if self.multiregional_buckets:
self.AssertNObjectsInBucket(buri1, 1)
self.AssertNObjectsInBucket(buri2, 1)
self.AssertNObjectsInBucket(buri3, 1)
self._RunRemoveCommandAndCheck(
['rm', '-r',
'%s://%s-tbu*' % (self.default_provider, buri_base)],
objects_to_remove=[
'%s#%s' % (suri(ouri1), urigen(ouri1)),
'%s#%s' % (suri(ouri2), urigen(ouri2))
],
buckets_to_remove=[suri(buri1), suri(buri2)])
self.AssertNObjectsInBucket(buri3, 1)
def test_remove_all_versions_recursive_on_subdir(self):
"""Test that 'rm -r' works on subdir."""
bucket_uri = self.CreateVersionedBucket()
k1_uri = bucket_uri.clone_replace_name('dir/foo')
k2_uri = bucket_uri.clone_replace_name('dir/foo2')
k1_uri.set_contents_from_string('bar')
k2_uri.set_contents_from_string('bar2')
k1g1 = urigen(k1_uri)
k2g1 = urigen(k2_uri)
k1_uri.set_contents_from_string('baz')
k2_uri.set_contents_from_string('baz2')
k1g2 = urigen(k1_uri)
k2g2 = urigen(k2_uri)
self.AssertNObjectsInBucket(bucket_uri, 4, versioned=True)
self._RunRemoveCommandAndCheck(
['rm', '-r', '%s' % suri(bucket_uri, 'dir')],
objects_to_remove=[
'%s#%s' % (suri(k1_uri), k1g1),
'%s#%s' % (suri(k1_uri), k1g2),
'%s#%s' % (suri(k2_uri), k2g1),
'%s#%s' % (suri(k2_uri), k2g2)
])
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
def test_remove_all_versions_recursive_on_subdir(self):
"""Test that 'rm -r' works on subdir."""
bucket_uri = self.CreateVersionedBucket()
k1_uri = self.StorageUriCloneReplaceName(bucket_uri, 'dir/foo')
k2_uri = self.StorageUriCloneReplaceName(bucket_uri, 'dir/foo2')
self.StorageUriSetContentsFromString(k1_uri, 'bar')
self.StorageUriSetContentsFromString(k2_uri, 'bar2')
k1g1 = urigen(k1_uri)
k2g1 = urigen(k2_uri)
self.StorageUriSetContentsFromString(k1_uri, 'baz')
self.StorageUriSetContentsFromString(k2_uri, 'baz2')
k1g2 = urigen(k1_uri)
k2g2 = urigen(k2_uri)
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 4, versioned=True)
self._RunRemoveCommandAndCheck(
['rm', '-r', '%s' % suri(bucket_uri, 'dir')],
objects_to_remove=[
'%s#%s' % (suri(k1_uri), k1g1),
'%s#%s' % (suri(k1_uri), k1g2),
'%s#%s' % (suri(k2_uri), k2g1),
'%s#%s' % (suri(k2_uri), k2g2)
])
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
def test_remove_all_versions_recursive_on_subdir(self):
"""Test that 'rm -r' works on subdir."""
bucket_uri = self.CreateVersionedBucket()
k1_uri = bucket_uri.clone_replace_name("dir/foo")
k2_uri = bucket_uri.clone_replace_name("dir/foo2")
k1_uri.set_contents_from_string("bar")
k2_uri.set_contents_from_string("bar2")
k1g1 = urigen(k1_uri)
k2g1 = urigen(k2_uri)
k1_uri.set_contents_from_string("baz")
k2_uri.set_contents_from_string("baz2")
k1g2 = urigen(k1_uri)
k2g2 = urigen(k2_uri)
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 4, versioned=True)
self._RunRemoveCommandAndCheck(
["rm", "-r", "%s" % suri(bucket_uri, "dir")],
objects_to_remove=[
"%s#%s" % (suri(k1_uri), k1g1),
"%s#%s" % (suri(k1_uri), k1g2),
"%s#%s" % (suri(k2_uri), k2g1),
"%s#%s" % (suri(k2_uri), k2g2),
],
)
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
def test_remove_all_versions_recursive_on_subdir(self):
"""Test that 'rm -r' works on subdir."""
bucket_uri = self.CreateVersionedBucket()
k1_uri = bucket_uri.clone_replace_name('dir/foo')
k2_uri = bucket_uri.clone_replace_name('dir/foo2')
k1_uri.set_contents_from_string('bar')
k2_uri.set_contents_from_string('bar2')
k1g1 = urigen(k1_uri)
k2g1 = urigen(k2_uri)
k1_uri.set_contents_from_string('baz')
k2_uri.set_contents_from_string('baz2')
k1g2 = urigen(k1_uri)
k2g2 = urigen(k2_uri)
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 4, versioned=True)
self._RunRemoveCommandAndCheck(
['rm', '-r', '%s' % suri(bucket_uri, 'dir')],
objects_to_remove=[
'%s#%s' % (suri(k1_uri), k1g1),
'%s#%s' % (suri(k1_uri), k1g2),
'%s#%s' % (suri(k2_uri), k2g1),
'%s#%s' % (suri(k2_uri), k2g2)
])
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
def test_remove_all_versions_recursive_on_bucket(self):
"""Test that 'rm -r' works on bucket."""
bucket_uri = self.CreateVersionedBucket()
k1_uri = bucket_uri.clone_replace_name('foo')
k2_uri = bucket_uri.clone_replace_name('foo2')
k1_uri.set_contents_from_string('bar')
k2_uri.set_contents_from_string('bar2')
k1g1 = urigen(k1_uri)
k2g1 = urigen(k2_uri)
k1_uri.set_contents_from_string('baz')
k2_uri.set_contents_from_string('baz2')
k1g2 = urigen(k1_uri)
k2g2 = urigen(k2_uri)
self.AssertNObjectsInBucket(bucket_uri, 4, versioned=True)
self._RunRemoveCommandAndCheck(
['rm', '-r', suri(bucket_uri)],
objects_to_remove=['%s#%s' % (suri(k1_uri), k1g1),
'%s#%s' % (suri(k1_uri), k1g2),
'%s#%s' % (suri(k2_uri), k2g1),
'%s#%s' % (suri(k2_uri), k2g2)],
buckets_to_remove=[suri(bucket_uri)])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
# Bucket should no longer exist.
stderr = self.RunGsUtil(['ls', '-a', suri(bucket_uri)],
return_stderr=True, expected_status=1)
self.assertIn('bucket does not exist', stderr)
_Check()
def test_all_versions_current(self):
"""Test that 'rm -a' for an object with a current version works."""
bucket_uri = self.CreateVersionedBucket()
key_uri = bucket_uri.clone_replace_name('foo')
key_uri.set_contents_from_string('bar')
g1 = urigen(key_uri)
key_uri.set_contents_from_string('baz')
g2 = urigen(key_uri)
def _Check1(stderr_lines):
stderr = self.RunGsUtil(['-m', 'rm', '-a', suri(key_uri)],
return_stderr=True)
stderr_lines.update(set(stderr.splitlines()))
stderr = '\n'.join(stderr_lines)
self.assertEqual(stderr.count('Removing %s://' % self.default_provider),
2)
self.assertIn('Removing %s#%s...' % (suri(key_uri), g1), stderr)
self.assertIn('Removing %s#%s...' % (suri(key_uri), g2), stderr)
all_stderr_lines = set()
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True)
@Retry(AssertionError, tries=3, timeout_secs=1)
# Use @Retry as hedge against bucket listing eventual consistency.
def _CheckWithRetries(stderr_lines):
_Check1(stderr_lines)
_CheckWithRetries(all_stderr_lines)
else:
_Check1(all_stderr_lines)
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
def test_recursive_bucket_rm_with_wildcarding(self):
"""Tests removing all objects and buckets matching a bucket wildcard."""
buri_base = 'gsutil-test-%s' % self.GetTestMethodName()
buri_base = buri_base[:MAX_BUCKET_LENGTH - 20]
buri_base = '%s-%s' % (buri_base, self.MakeRandomTestString())
buri1 = self.CreateBucket(bucket_name='%s-tbuck1' % buri_base)
buri2 = self.CreateBucket(bucket_name='%s-tbuck2' % buri_base)
buri3 = self.CreateBucket(bucket_name='%s-tb3' % buri_base)
ouri1 = self.CreateObject(bucket_uri=buri1,
object_name='o1',
contents='z')
ouri2 = self.CreateObject(bucket_uri=buri2,
object_name='o2',
contents='z')
self.CreateObject(bucket_uri=buri3, object_name='o3', contents='z')
self.AssertNObjectsInBucket(buri1, 1)
self.AssertNObjectsInBucket(buri2, 1)
self.AssertNObjectsInBucket(buri3, 1)
self._RunRemoveCommandAndCheck(
['rm', '-r',
'%s://%s-tbu*' % (self.default_provider, buri_base)],
objects_to_remove=[
'%s#%s' % (suri(ouri1), urigen(ouri1)),
'%s#%s' % (suri(ouri2), urigen(ouri2))
],
buckets_to_remove=[suri(buri1), suri(buri2)])
self.AssertNObjectsInBucket(buri3, 1)
def test_all_versions_current(self):
"""Test that 'rm -a' for an object with a current version works."""
bucket_uri = self.CreateVersionedBucket()
key_uri = bucket_uri.clone_replace_name('foo')
key_uri.set_contents_from_string('bar')
g1 = urigen(key_uri)
key_uri.set_contents_from_string('baz')
g2 = urigen(key_uri)
def _Check1(stderr_lines):
stderr = self.RunGsUtil(
['-m', 'rm', '-a', suri(key_uri)], return_stderr=True)
stderr_lines.update(set(stderr.splitlines()))
stderr = '\n'.join(stderr_lines)
self.assertEqual(
stderr.count('Removing %s://' % self.default_provider), 2)
self.assertIn('Removing %s#%s...' % (suri(key_uri), g1), stderr)
self.assertIn('Removing %s#%s...' % (suri(key_uri), g2), stderr)
all_stderr_lines = set()
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True)
@Retry(AssertionError, tries=3, timeout_secs=1)
# Use @Retry as hedge against bucket listing eventual consistency.
def _CheckWithRetries(stderr_lines):
_Check1(stderr_lines)
_CheckWithRetries(all_stderr_lines)
else:
_Check1(all_stderr_lines)
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
def test_stdin_args(self):
"""Tests rm with the -I option."""
buri1 = self.CreateVersionedBucket()
ouri1 = self.CreateObject(bucket_uri=buri1,
object_name='foo',
contents='foocontents')
self.CreateObject(bucket_uri=buri1,
object_name='bar',
contents='barcontents')
ouri3 = self.CreateObject(bucket_uri=buri1,
object_name='baz',
contents='bazcontents')
buri2 = self.CreateVersionedBucket()
ouri4 = self.CreateObject(bucket_uri=buri2,
object_name='moo',
contents='moocontents')
self.AssertNObjectsInBucket(buri1, 3, versioned=True)
self.AssertNObjectsInBucket(buri2, 1, versioned=True)
objects_to_remove = [
'%s#%s' % (suri(ouri1), urigen(ouri1)),
'%s#%s' % (suri(ouri3), urigen(ouri3)),
'%s#%s' % (suri(ouri4), urigen(ouri4))
]
stdin = '\n'.join(objects_to_remove)
self._RunRemoveCommandAndCheck(['rm', '-I'],
objects_to_remove=objects_to_remove,
stdin=stdin)
self.AssertNObjectsInBucket(buri1, 1, versioned=True)
self.AssertNObjectsInBucket(buri2, 0, versioned=True)
def test_remove_all_versions_recursive_on_bucket(self):
"""Test that 'rm -r' works on bucket."""
bucket_uri = self.CreateVersionedBucket()
k1_uri = bucket_uri.clone_replace_name('foo')
k2_uri = bucket_uri.clone_replace_name('foo2')
k1_uri.set_contents_from_string('bar')
k2_uri.set_contents_from_string('bar2')
k1g1 = urigen(k1_uri)
k2g1 = urigen(k2_uri)
k1_uri.set_contents_from_string('baz')
k2_uri.set_contents_from_string('baz2')
k1g2 = urigen(k1_uri)
k2g2 = urigen(k2_uri)
self.AssertNObjectsInBucket(bucket_uri, 4, versioned=True)
self._RunRemoveCommandAndCheck(
['rm', '-r', suri(bucket_uri)],
objects_to_remove=['%s#%s' % (suri(k1_uri), k1g1),
'%s#%s' % (suri(k1_uri), k1g2),
'%s#%s' % (suri(k2_uri), k2g1),
'%s#%s' % (suri(k2_uri), k2g2)],
buckets_to_remove=[suri(bucket_uri)])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
# Bucket should no longer exist.
stderr = self.RunGsUtil(['ls', '-a', suri(bucket_uri)],
return_stderr=True, expected_status=1)
self.assertIn('bucket does not exist', stderr)
_Check()
def test_stdin_args(self):
"""Tests rm with the -I option."""
buri1 = self.CreateVersionedBucket()
ouri1 = self.CreateObject(bucket_uri=buri1,
object_name='foo',
contents='foocontents')
self.CreateObject(bucket_uri=buri1, object_name='bar',
contents='barcontents')
ouri3 = self.CreateObject(bucket_uri=buri1,
object_name='baz',
contents='bazcontents')
buri2 = self.CreateVersionedBucket()
ouri4 = self.CreateObject(bucket_uri=buri2,
object_name='moo',
contents='moocontents')
self.AssertNObjectsInBucket(buri1, 3, versioned=True)
self.AssertNObjectsInBucket(buri2, 1, versioned=True)
objects_to_remove = ['%s#%s' % (suri(ouri1), urigen(ouri1)),
'%s#%s' % (suri(ouri3), urigen(ouri3)),
'%s#%s' % (suri(ouri4), urigen(ouri4))]
stdin = '\n'.join(objects_to_remove)
self._RunRemoveCommandAndCheck(['rm', '-I'],
objects_to_remove=objects_to_remove,
stdin=stdin)
self.AssertNObjectsInBucket(buri1, 1, versioned=True)
self.AssertNObjectsInBucket(buri2, 0, versioned=True)
def test_all_versions_no_current(self):
"""Test that 'rm -a' for an object without a current version works."""
bucket_uri = self.CreateVersionedBucket()
key_uri = bucket_uri.clone_replace_name("foo")
key_uri.set_contents_from_string("bar")
g1 = urigen(key_uri)
key_uri.set_contents_from_string("baz")
g2 = urigen(key_uri)
self._RunRemoveCommandAndCheck(
["-m", "rm", "-a", suri(key_uri)],
objects_to_remove=["%s#%s" % (suri(key_uri), g1), "%s#%s" % (suri(key_uri), g2)],
)
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
def test_all_versions_no_current(self):
"""Test that 'rm -a' for an object without a current version works."""
bucket_uri = self.CreateVersionedBucket()
key_uri = bucket_uri.clone_replace_name('foo')
key_uri.set_contents_from_string('bar')
g1 = urigen(key_uri)
key_uri.set_contents_from_string('baz')
g2 = urigen(key_uri)
self._RunRemoveCommandAndCheck(
['-m', 'rm', '-a', suri(key_uri)],
objects_to_remove=['%s#%s' % (suri(key_uri), g1),
'%s#%s' % (suri(key_uri), g2)])
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
def test_all_versions_no_current(self):
"""Test that 'rm -a' for an object without a current version works."""
bucket_uri = self.CreateVersionedBucket()
key_uri = bucket_uri.clone_replace_name('foo')
key_uri.set_contents_from_string('bar')
g1 = urigen(key_uri)
key_uri.set_contents_from_string('baz')
g2 = urigen(key_uri)
self._RunRemoveCommandAndCheck(
['-m', 'rm', '-a', suri(key_uri)],
objects_to_remove=['%s#%s' % (suri(key_uri), g1),
'%s#%s' % (suri(key_uri), g2)])
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
def test_all_versions_no_current(self):
"""Test that 'rm -a' for an object without a current version works."""
bucket_uri = self.CreateVersionedBucket()
key_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo')
self.StorageUriSetContentsFromString(key_uri, 'bar')
g1 = urigen(key_uri)
self.StorageUriSetContentsFromString(key_uri, 'baz')
g2 = urigen(key_uri)
self._RunRemoveCommandAndCheck(
['-m', 'rm', '-a', suri(key_uri)],
objects_to_remove=[
'%s#%s' % (suri(key_uri), g1),
'%s#%s' % (suri(key_uri), g2)
])
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
def test_slasher_horror_film(self):
"""Tests removing a bucket with objects that are filled with slashes."""
bucket_uri = self.CreateVersionedBucket()
ouri1 = self.CreateObject(bucket_uri=bucket_uri,
object_name='h/e/l//lo',
contents='Halloween')
ouri2 = self.CreateObject(bucket_uri=bucket_uri,
object_name='/h/e/l/l/o',
contents='A Nightmare on Elm Street')
ouri3 = self.CreateObject(bucket_uri=bucket_uri,
object_name='//h//e/l//l/o',
contents='Friday the 13th')
ouri4 = self.CreateObject(bucket_uri=bucket_uri,
object_name='//h//e//l//l//o',
contents='I Know What You Did Last Summer')
ouri5 = self.CreateObject(bucket_uri=bucket_uri,
object_name='/',
contents='Scream')
ouri6 = self.CreateObject(bucket_uri=bucket_uri,
object_name='//',
contents='Child\'s Play')
ouri7 = self.CreateObject(bucket_uri=bucket_uri,
object_name='///',
contents='The Prowler')
ouri8 = self.CreateObject(bucket_uri=bucket_uri,
object_name='////',
contents='Black Christmas')
ouri9 = self.CreateObject(
bucket_uri=bucket_uri,
object_name='everything/is/better/with/slashes///////',
contents='Maniac')
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 9, versioned=True)
# We add a slash to URIs with a trailing slash,
# because ObjectToURI (suri) removes one trailing slash.
objects_to_remove = [
'%s#%s' % (suri(ouri1), urigen(ouri1)),
'%s#%s' % (suri(ouri2), urigen(ouri2)),
'%s#%s' % (suri(ouri3), urigen(ouri3)),
'%s#%s' % (suri(ouri4), urigen(ouri4)),
'%s#%s' % (suri(ouri5) + '/', urigen(ouri5)),
'%s#%s' % (suri(ouri6) + '/', urigen(ouri6)),
'%s#%s' % (suri(ouri7) + '/', urigen(ouri7)),
'%s#%s' % (suri(ouri8) + '/', urigen(ouri8)),
'%s#%s' % (suri(ouri9) + '/', urigen(ouri9))
]
self._RunRemoveCommandAndCheck(
['-m', 'rm', '-r', suri(bucket_uri)],
objects_to_remove=objects_to_remove,
buckets_to_remove=[suri(bucket_uri)])
def test_slasher_horror_film(self):
"""Tests removing a bucket with objects that are filled with slashes."""
bucket_uri = self.CreateVersionedBucket()
ouri1 = self.CreateObject(bucket_uri=bucket_uri,
object_name='h/e/l//lo',
contents=b'Halloween')
ouri2 = self.CreateObject(bucket_uri=bucket_uri,
object_name='/h/e/l/l/o',
contents=b'A Nightmare on Elm Street')
ouri3 = self.CreateObject(bucket_uri=bucket_uri,
object_name='//h//e/l//l/o',
contents=b'Friday the 13th')
ouri4 = self.CreateObject(bucket_uri=bucket_uri,
object_name='//h//e//l//l//o',
contents=b'I Know What You Did Last Summer')
ouri5 = self.CreateObject(bucket_uri=bucket_uri,
object_name='/',
contents=b'Scream')
ouri6 = self.CreateObject(bucket_uri=bucket_uri,
object_name='//',
contents=b'Child\'s Play')
ouri7 = self.CreateObject(bucket_uri=bucket_uri,
object_name='///',
contents=b'The Prowler')
ouri8 = self.CreateObject(bucket_uri=bucket_uri,
object_name='////',
contents=b'Black Christmas')
ouri9 = self.CreateObject(
bucket_uri=bucket_uri,
object_name='everything/is/better/with/slashes///////',
contents=b'Maniac')
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 9, versioned=True)
# We add a slash to URIs with a trailing slash,
# because ObjectToURI (suri) removes one trailing slash.
objects_to_remove = [
'%s#%s' % (suri(ouri1), urigen(ouri1)),
'%s#%s' % (suri(ouri2), urigen(ouri2)),
'%s#%s' % (suri(ouri3), urigen(ouri3)),
'%s#%s' % (suri(ouri4), urigen(ouri4)),
'%s#%s' % (suri(ouri5) + '/', urigen(ouri5)),
'%s#%s' % (suri(ouri6) + '/', urigen(ouri6)),
'%s#%s' % (suri(ouri7) + '/', urigen(ouri7)),
'%s#%s' % (suri(ouri8) + '/', urigen(ouri8)),
'%s#%s' % (suri(ouri9) + '/', urigen(ouri9))
]
self._RunRemoveCommandAndCheck(
['-m', 'rm', '-r', suri(bucket_uri)],
objects_to_remove=objects_to_remove,
buckets_to_remove=[suri(bucket_uri)])
def test_versioned(self):
"""Tests listing all versions with the -a flag."""
bucket_uri = self.CreateVersionedBucket()
object_uri1 = self.CreateObject(
bucket_uri=bucket_uri, object_name='foo', contents='foo')
object_uri2 = self.CreateObject(
bucket_uri=bucket_uri, object_name='foo', contents='foo2',
gs_idempotent_generation=urigen(object_uri1))
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
stdout = self.RunGsUtil(['du', suri(bucket_uri)], return_stdout=True)
self.assertEqual(stdout, '%-10s %s\n' % (4, suri(object_uri2)))
_Check1()
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check2():
stdout = self.RunGsUtil(['du', '-a', suri(bucket_uri)],
return_stdout=True)
self.assertSetEqual(set(stdout.splitlines()), set([
'%-10s %s#%s' % (
3, suri(object_uri1), object_uri1.generation),
'%-10s %s#%s' % (
4, suri(object_uri2), object_uri2.generation),
]))
_Check2()
def test_folder_objects_deleted_with_wildcard(self):
"""Test for 'rm -r' of a folder with a dir_$folder$ marker."""
bucket_uri = self.CreateVersionedBucket()
key_uri = bucket_uri.clone_replace_name('abc/o1')
key_uri.set_contents_from_string('foobar')
folder_uri = bucket_uri.clone_replace_name('abc_$folder$')
folder_uri.set_contents_from_string('')
self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True)
self._RunRemoveCommandAndCheck(
['rm', '-r', '%s' % suri(bucket_uri, '**')],
objects_to_remove=['%s#%s' % (suri(key_uri), urigen(key_uri)),
'%s#%s' % (suri(folder_uri), urigen(folder_uri))])
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
# Bucket should not be deleted (Should not get ServiceException).
bucket_uri.get_location(validate=False)
def test_cat_version(self):
"""Tests cat command on versioned objects."""
bucket_uri = self.CreateVersionedBucket()
# Create 2 versions of an object.
uri1 = self.CreateObject(bucket_uri=bucket_uri, contents='data1',
gs_idempotent_generation=0)
uri2 = self.CreateObject(bucket_uri=bucket_uri,
object_name=uri1.object_name, contents='data2',
gs_idempotent_generation=urigen(uri1))
stdout = self.RunGsUtil(['cat', suri(uri1)], return_stdout=True)
# Last version written should be live.
self.assertEqual('data2', stdout)
# Using either version-specific URI should work.
stdout = self.RunGsUtil(['cat', uri1.version_specific_uri],
return_stdout=True)
self.assertEqual('data1', stdout)
stdout = self.RunGsUtil(['cat', uri2.version_specific_uri],
return_stdout=True)
self.assertEqual('data2', stdout)
if RUN_S3_TESTS:
# S3 GETs of invalid versions return 400s.
# Also, appending between 1 and 3 characters to the version_id can
# result in a success (200) response from the server.
stderr = self.RunGsUtil(['cat', uri2.version_specific_uri + '23456'],
return_stderr=True, expected_status=1)
self.assertIn('BadRequestException: 400', stderr)
else:
# Attempting to cat invalid version should result in an error.
stderr = self.RunGsUtil(['cat', uri2.version_specific_uri + '23'],
return_stderr=True, expected_status=1)
self.assertIn(NO_URLS_MATCHED_TARGET % uri2.version_specific_uri + '23',
stderr)
def test_folder_objects_deleted_with_wildcard(self):
"""Test for 'rm -r' of a folder with a dir_$folder$ marker."""
bucket_uri = self.CreateVersionedBucket()
key_uri = bucket_uri.clone_replace_name('abc/o1')
key_uri.set_contents_from_string('foobar')
folder_uri = bucket_uri.clone_replace_name('abc_$folder$')
folder_uri.set_contents_from_string('')
self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True)
self._RunRemoveCommandAndCheck(
['rm', '-r', '%s' % suri(bucket_uri, '**')],
objects_to_remove=['%s#%s' % (suri(key_uri), urigen(key_uri)),
'%s#%s' % (suri(folder_uri), urigen(folder_uri))])
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
# Bucket should not be deleted (Should not get ServiceException).
bucket_uri.get_location(validate=False)
def testMultiVersionSupport(self):
"""Tests changing ACLs on multiple object versions."""
bucket = self.CreateVersionedBucket()
object_name = self.MakeTempName('obj')
obj1_uri = self.CreateObject(bucket_uri=bucket,
object_name=object_name,
contents='One thing')
# Create another on the same URI, giving us a second version.
self.CreateObject(bucket_uri=bucket,
object_name=object_name,
contents='Another thing',
gs_idempotent_generation=urigen(obj1_uri))
lines = self.AssertNObjectsInBucket(bucket, 2, versioned=True)
obj_v1, obj_v2 = lines[0], lines[1]
test_regex = self._MakeScopeRegex('READER', 'group',
self.GROUP_TEST_ADDRESS)
json_text = self.RunGsUtil(self._get_acl_prefix + [obj_v1],
return_stdout=True)
self.assertNotRegexpMatches(json_text, test_regex)
self.RunGsUtil(self._ch_acl_prefix +
['-g', self.GROUP_TEST_ADDRESS + ':READ', obj_v1])
json_text = self.RunGsUtil(self._get_acl_prefix + [obj_v1],
return_stdout=True)
self.assertRegexpMatches(json_text, test_regex)
json_text = self.RunGsUtil(self._get_acl_prefix + [obj_v2],
return_stdout=True)
self.assertNotRegexpMatches(json_text, test_regex)
def testMultiVersionSupport(self):
"""Tests changing ACLs on multiple object versions."""
bucket = self.CreateVersionedBucket()
object_name = self.MakeTempName("obj")
obj1_uri = self.CreateObject(bucket_uri=bucket, object_name=object_name, contents="One thing")
# Create another on the same URI, giving us a second version.
self.CreateObject(
bucket_uri=bucket,
object_name=object_name,
contents="Another thing",
gs_idempotent_generation=urigen(obj1_uri),
)
lines = self.AssertNObjectsInBucket(bucket, 2, versioned=True)
obj_v1, obj_v2 = lines[0], lines[1]
test_regex = self._MakeScopeRegex("READER", "group", self.GROUP_TEST_ADDRESS)
json_text = self.RunGsUtil(self._get_acl_prefix + [obj_v1], return_stdout=True)
self.assertNotRegexpMatches(json_text, test_regex)
self.RunGsUtil(self._ch_acl_prefix + ["-g", self.GROUP_TEST_ADDRESS + ":READ", obj_v1])
json_text = self.RunGsUtil(self._get_acl_prefix + [obj_v1], return_stdout=True)
self.assertRegexpMatches(json_text, test_regex)
json_text = self.RunGsUtil(self._get_acl_prefix + [obj_v2], return_stdout=True)
self.assertNotRegexpMatches(json_text, test_regex)
def test_rm_object_with_slash(self):
"""Tests removing a bucket that has an object with a slash in it."""
bucket_uri = self.CreateVersionedBucket()
ouri1 = self.CreateObject(bucket_uri=bucket_uri, object_name="/dirwithslash/foo", contents="z")
ouri2 = self.CreateObject(bucket_uri=bucket_uri, object_name="dirnoslash/foo", contents="z")
ouri3 = self.CreateObject(bucket_uri=bucket_uri, object_name="dirnoslash/foo2", contents="z")
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 3, versioned=True)
self._RunRemoveCommandAndCheck(
["rm", "-r", suri(bucket_uri)],
objects_to_remove=[
"%s#%s" % (suri(ouri1), urigen(ouri1)),
"%s#%s" % (suri(ouri2), urigen(ouri2)),
"%s#%s" % (suri(ouri3), urigen(ouri3)),
],
buckets_to_remove=[suri(bucket_uri)],
)
def test_rm_object_with_slash(self):
"""Tests removing a bucket that has an object with a slash in it."""
bucket_uri = self.CreateVersionedBucket()
ouri1 = self.CreateObject(bucket_uri=bucket_uri,
object_name='/dirwithslash/foo', contents='z')
ouri2 = self.CreateObject(bucket_uri=bucket_uri,
object_name='dirnoslash/foo', contents='z')
ouri3 = self.CreateObject(bucket_uri=bucket_uri,
object_name='dirnoslash/foo2', contents='z')
self.AssertNObjectsInBucket(bucket_uri, 3, versioned=True)
self._RunRemoveCommandAndCheck(
['rm', '-r', suri(bucket_uri)],
objects_to_remove=['%s#%s' % (suri(ouri1), urigen(ouri1)),
'%s#%s' % (suri(ouri2), urigen(ouri2)),
'%s#%s' % (suri(ouri3), urigen(ouri3))],
buckets_to_remove=[suri(bucket_uri)])
def test_rm_object_with_slash(self):
"""Tests removing a bucket that has an object with a slash in it."""
bucket_uri = self.CreateVersionedBucket()
ouri1 = self.CreateObject(bucket_uri=bucket_uri,
object_name='/dirwithslash/foo', contents='z')
ouri2 = self.CreateObject(bucket_uri=bucket_uri,
object_name='dirnoslash/foo', contents='z')
ouri3 = self.CreateObject(bucket_uri=bucket_uri,
object_name='dirnoslash/foo2', contents='z')
self.AssertNObjectsInBucket(bucket_uri, 3, versioned=True)
self._RunRemoveCommandAndCheck(
['rm', '-r', suri(bucket_uri)],
objects_to_remove=['%s#%s' % (suri(ouri1), urigen(ouri1)),
'%s#%s' % (suri(ouri2), urigen(ouri2)),
'%s#%s' % (suri(ouri3), urigen(ouri3))],
buckets_to_remove=[suri(bucket_uri)])
def test_recursive_bucket_rm(self):
"""Test for 'rm -r' of a bucket."""
bucket_uri = self.CreateBucket()
object_uri = self.CreateObject(bucket_uri, contents=b'foo')
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 1)
self._RunRemoveCommandAndCheck(
['rm', '-r', suri(bucket_uri)],
objects_to_remove=[
'%s#%s' % (suri(object_uri), urigen(object_uri))
],
buckets_to_remove=[suri(bucket_uri)])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
# Bucket should be deleted.
stderr = self.RunGsUtil(
['ls', '-Lb', suri(bucket_uri)],
return_stderr=True,
expected_status=1,
force_gsutil=True)
self.assertIn('bucket does not exist', stderr)
_Check1()
# Now try same thing, but for a versioned bucket with multiple versions of
# an object present.
bucket_uri = self.CreateVersionedBucket()
self.CreateObject(bucket_uri, 'obj', contents=b'z')
self.CreateObject(bucket_uri, 'obj', contents=b'z')
final_uri = self.CreateObject(bucket_uri, 'obj', contents=b'z')
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 3, versioned=True)
self._RunRemoveCommandAndCheck(['rm', suri(bucket_uri, '**')],
objects_to_remove=['%s' % final_uri])
stderr = self.RunGsUtil(['rb', suri(bucket_uri)],
return_stderr=True,
expected_status=1,
force_gsutil=True)
self.assertIn('not empty', stderr)
# Now try with rm -r.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check2():
self.RunGsUtil(['rm', '-r', suri(bucket_uri)])
# Bucket should be deleted.
stderr = self.RunGsUtil(
['ls', '-Lb', suri(bucket_uri)],
return_stderr=True,
expected_status=1,
force_gsutil=True)
self.assertIn('bucket does not exist', stderr)
_Check2()
def test_folder_objects_deleted_with_wildcard(self):
"""Test for 'rm -r' of a folder with a dir_$folder$ marker."""
bucket_uri = self.CreateVersionedBucket()
key_uri = bucket_uri.clone_replace_name("abc/o1")
key_uri.set_contents_from_string("foobar")
folder_uri = bucket_uri.clone_replace_name("abc_$folder$")
folder_uri.set_contents_from_string("")
self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True)
self._RunRemoveCommandAndCheck(
["rm", "-r", "%s" % suri(bucket_uri, "**")],
objects_to_remove=[
"%s#%s" % (suri(key_uri), urigen(key_uri)),
"%s#%s" % (suri(folder_uri), urigen(folder_uri)),
],
)
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
# Bucket should not be deleted (Should not get ServiceException).
bucket_uri.get_location(validate=False)
def test_remove_all_versions_recursive_on_bucket(self):
"""Test that 'rm -r' works on bucket."""
bucket_uri = self.CreateVersionedBucket()
k1_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo')
k2_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo2')
self.StorageUriSetContentsFromString(k1_uri, 'bar')
self.StorageUriSetContentsFromString(k2_uri, 'bar2')
k1g1 = urigen(k1_uri)
k2g1 = urigen(k2_uri)
self.StorageUriSetContentsFromString(k1_uri, 'baz')
self.StorageUriSetContentsFromString(k2_uri, 'baz2')
k1g2 = urigen(k1_uri)
k2g2 = urigen(k2_uri)
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 4, versioned=True)
self._RunRemoveCommandAndCheck(
['rm', '-r', suri(bucket_uri)],
objects_to_remove=[
'%s#%s' % (suri(k1_uri), k1g1),
'%s#%s' % (suri(k1_uri), k1g2),
'%s#%s' % (suri(k2_uri), k2g1),
'%s#%s' % (suri(k2_uri), k2g2)
],
buckets_to_remove=[suri(bucket_uri)])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
# Bucket should no longer exist.
stderr = self.RunGsUtil(['ls', '-a', suri(bucket_uri)],
return_stderr=True,
expected_status=1)
if self._use_gcloud_storage:
if self._use_gcloud_storage:
# GCS and S3 responses.
self.assertTrue('not found: 404' in stderr
or 'NoSuchBucket' in stderr)
else:
self.assertIn('bucket does not exist', stderr)
_Check()
def test_folder_objects_deleted_with_wildcard(self):
"""Test for 'rm -r' of a folder with a dir_$folder$ marker."""
bucket_uri = self.CreateVersionedBucket()
key_uri = self.StorageUriCloneReplaceName(bucket_uri, 'abc/o1')
self.StorageUriSetContentsFromString(key_uri, 'foobar')
folder_uri = self.StorageUriCloneReplaceName(bucket_uri,
'abc_$folder$')
self.StorageUriSetContentsFromString(folder_uri, '')
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True)
self._RunRemoveCommandAndCheck(
['rm', '-r', '%s' % suri(bucket_uri, '**')],
objects_to_remove=[
'%s#%s' % (suri(key_uri), urigen(key_uri)),
'%s#%s' % (suri(folder_uri), urigen(folder_uri))
])
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
# Bucket should not be deleted (Should not get ServiceException).
bucket_uri.get_location(validate=False)
def test_stdin_args(self):
"""Tests rm with the -I option."""
buri1 = self.CreateVersionedBucket()
ouri1 = self.CreateObject(bucket_uri=buri1, object_name="foo", contents="foocontents")
self.CreateObject(bucket_uri=buri1, object_name="bar", contents="barcontents")
ouri3 = self.CreateObject(bucket_uri=buri1, object_name="baz", contents="bazcontents")
buri2 = self.CreateVersionedBucket()
ouri4 = self.CreateObject(bucket_uri=buri2, object_name="moo", contents="moocontents")
self.AssertNObjectsInBucket(buri1, 3, versioned=True)
self.AssertNObjectsInBucket(buri2, 1, versioned=True)
objects_to_remove = [
"%s#%s" % (suri(ouri1), urigen(ouri1)),
"%s#%s" % (suri(ouri3), urigen(ouri3)),
"%s#%s" % (suri(ouri4), urigen(ouri4)),
]
stdin = "\n".join(objects_to_remove)
self._RunRemoveCommandAndCheck(["rm", "-I"], objects_to_remove=objects_to_remove, stdin=stdin)
self.AssertNObjectsInBucket(buri1, 1, versioned=True)
self.AssertNObjectsInBucket(buri2, 0, versioned=True)
def _create_multiple_objects(self):
"""Creates two versioned objects and return references to all versions.
Returns:
A four-tuple (a, b, a*, b*) of storage_uri.BucketStorageUri instances.
"""
old_gsutil_object = self.CreateObject(
bucket_uri=self.versioned_bucket, contents='foo')
old_gsutil_object2 = self.CreateObject(
bucket_uri=self.versioned_bucket, contents='bar')
gsutil_object = self.CreateObject(
bucket_uri=self.versioned_bucket,
object_name=old_gsutil_object.object_name,
contents='new_foo', gs_idempotent_generation=urigen(old_gsutil_object))
gsutil_object2 = self.CreateObject(
bucket_uri=self.versioned_bucket,
object_name=old_gsutil_object2.object_name,
contents='new_bar', gs_idempotent_generation=urigen(old_gsutil_object2))
return (old_gsutil_object, old_gsutil_object2, gsutil_object,
gsutil_object2)
def _create_multiple_objects(self):
"""Creates two versioned objects and return references to all versions.
Returns:
A four-tuple (a, b, a*, b*) of storage_uri.BucketStorageUri instances.
"""
old_gsutil_object = self.CreateObject(
bucket_uri=self.versioned_bucket, contents='foo')
old_gsutil_object2 = self.CreateObject(
bucket_uri=self.versioned_bucket, contents='bar')
gsutil_object = self.CreateObject(
bucket_uri=self.versioned_bucket,
object_name=old_gsutil_object.object_name,
contents='new_foo', gs_idempotent_generation=urigen(old_gsutil_object))
gsutil_object2 = self.CreateObject(
bucket_uri=self.versioned_bucket,
object_name=old_gsutil_object2.object_name,
contents='new_bar', gs_idempotent_generation=urigen(old_gsutil_object2))
return (old_gsutil_object, old_gsutil_object2, gsutil_object,
gsutil_object2)
def test_rm_object_with_slashes(self):
"""Tests removing a bucket that has objects with slashes."""
bucket_uri = self.CreateVersionedBucket()
ouri1 = self.CreateObject(bucket_uri=bucket_uri,
object_name='h/e/l//lo',
contents=b'z')
ouri2 = self.CreateObject(bucket_uri=bucket_uri,
object_name='dirnoslash/foo',
contents=b'z')
ouri3 = self.CreateObject(bucket_uri=bucket_uri,
object_name='dirnoslash/foo2',
contents=b'z')
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 3, versioned=True)
self._RunRemoveCommandAndCheck(['rm', '-r', suri(bucket_uri)],
objects_to_remove=[
'%s#%s' % (suri(ouri1), urigen(ouri1)),
'%s#%s' % (suri(ouri2), urigen(ouri2)),
'%s#%s' % (suri(ouri3), urigen(ouri3))
],
buckets_to_remove=[suri(bucket_uri)])
def test_all_versions_current(self):
"""Test that 'rm -a' for an object with a current version works."""
bucket_uri = self.CreateVersionedBucket()
key_uri = bucket_uri.clone_replace_name("foo")
key_uri.set_contents_from_string("bar")
g1 = urigen(key_uri)
key_uri.set_contents_from_string("baz")
g2 = urigen(key_uri)
self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True)
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1(stderr_lines):
stderr = self.RunGsUtil(["-m", "rm", "-a", suri(key_uri)], return_stderr=True)
stderr_lines.update(set(stderr.splitlines()))
stderr = "\n".join(stderr_lines)
self.assertEqual(stderr.count("Removing %s://" % self.default_provider), 2)
self.assertIn("Removing %s#%s..." % (suri(key_uri), g1), stderr)
self.assertIn("Removing %s#%s..." % (suri(key_uri), g2), stderr)
all_stderr_lines = set()
_Check1(all_stderr_lines)
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
def test_recursive_bucket_rm(self):
"""Test for 'rm -r' of a bucket."""
bucket_uri = self.CreateBucket()
object_uri = self.CreateObject(bucket_uri, contents=b'foo')
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 1)
self._RunRemoveCommandAndCheck(
['rm', '-r', suri(bucket_uri)],
objects_to_remove=['%s#%s' % (suri(object_uri), urigen(object_uri))],
buckets_to_remove=[suri(bucket_uri)])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
# Bucket should be deleted.
stderr = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)],
return_stderr=True,
expected_status=1)
self.assertIn('bucket does not exist', stderr)
_Check1()
# Now try same thing, but for a versioned bucket with multiple versions of
# an object present.
bucket_uri = self.CreateVersionedBucket()
self.CreateObject(bucket_uri, 'obj', contents=b'z')
self.CreateObject(bucket_uri, 'obj', contents=b'z')
final_uri = self.CreateObject(bucket_uri, 'obj', contents=b'z')
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 3, versioned=True)
self._RunRemoveCommandAndCheck(['rm', suri(bucket_uri, '**')],
objects_to_remove=['%s' % final_uri])
stderr = self.RunGsUtil(['rb', suri(bucket_uri)],
return_stderr=True,
expected_status=1)
self.assertIn('Bucket is not empty', stderr)
# Now try with rm -r.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check2():
self.RunGsUtil(['rm', '-r', suri(bucket_uri)])
# Bucket should be deleted.
stderr = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)],
return_stderr=True,
expected_status=1)
self.assertIn('bucket does not exist', stderr)
_Check2()
def test_remove_all_versions_recursive_on_bucket(self):
"""Test that 'rm -r' works on bucket."""
bucket_uri = self.CreateVersionedBucket()
k1_uri = bucket_uri.clone_replace_name("foo")
k2_uri = bucket_uri.clone_replace_name("foo2")
k1_uri.set_contents_from_string("bar")
k2_uri.set_contents_from_string("bar2")
k1g1 = urigen(k1_uri)
k2g1 = urigen(k2_uri)
k1_uri.set_contents_from_string("baz")
k2_uri.set_contents_from_string("baz2")
k1g2 = urigen(k1_uri)
k2g2 = urigen(k2_uri)
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 4, versioned=True)
self._RunRemoveCommandAndCheck(
["rm", "-r", suri(bucket_uri)],
objects_to_remove=[
"%s#%s" % (suri(k1_uri), k1g1),
"%s#%s" % (suri(k1_uri), k1g2),
"%s#%s" % (suri(k2_uri), k2g1),
"%s#%s" % (suri(k2_uri), k2g2),
],
buckets_to_remove=[suri(bucket_uri)],
)
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
# Bucket should no longer exist.
stderr = self.RunGsUtil(["ls", "-a", suri(bucket_uri)], return_stderr=True, expected_status=1)
self.assertIn("bucket does not exist", stderr)
_Check()
def test_versioned_stat_output(self):
"""Tests stat output of an outdated object under version control."""
bucket_uri = self.CreateVersionedBucket()
old_object_uri = self.CreateObject(bucket_uri=bucket_uri, contents='z')
# Update object
self.CreateObject(bucket_uri=bucket_uri,
object_name=old_object_uri.object_name,
contents='z',
gs_idempotent_generation=urigen(old_object_uri))
stdout = self.RunGsUtil(['stat', old_object_uri.version_specific_uri],
return_stdout=True)
self.assertIn('Archived time', stdout)
def test_versioned_stat_output(self):
"""Tests stat output of an outdated object under version control."""
bucket_uri = self.CreateVersionedBucket()
old_object_uri = self.CreateObject(
bucket_uri=bucket_uri, contents='z')
# Update object
self.CreateObject(
bucket_uri=bucket_uri,
object_name=old_object_uri.object_name,
contents='z', gs_idempotent_generation=urigen(old_object_uri))
stdout = self.RunGsUtil(
['stat', old_object_uri.version_specific_uri], return_stdout=True)
self.assertIn('Archived time', stdout)
def test_slasher_horror_film(self):
"""Tests removing a bucket with objects that are filled with slashes."""
bucket_uri = self.CreateVersionedBucket()
ouri1 = self.CreateObject(bucket_uri=bucket_uri, object_name="h/e/l//lo", contents="Halloween")
ouri2 = self.CreateObject(bucket_uri=bucket_uri, object_name="/h/e/l/l/o", contents="A Nightmare on Elm Street")
ouri3 = self.CreateObject(bucket_uri=bucket_uri, object_name="//h//e/l//l/o", contents="Friday the 13th")
ouri4 = self.CreateObject(
bucket_uri=bucket_uri, object_name="//h//e//l//l//o", contents="I Know What You Did Last Summer"
)
ouri5 = self.CreateObject(bucket_uri=bucket_uri, object_name="/", contents="Scream")
ouri6 = self.CreateObject(bucket_uri=bucket_uri, object_name="//", contents="Child's Play")
ouri7 = self.CreateObject(bucket_uri=bucket_uri, object_name="///", contents="The Prowler")
ouri8 = self.CreateObject(bucket_uri=bucket_uri, object_name="////", contents="Black Christmas")
ouri9 = self.CreateObject(
bucket_uri=bucket_uri, object_name="everything/is/better/with/slashes///////", contents="Maniac"
)
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 9, versioned=True)
# We add a slash to URIs with a trailing slash,
# because ObjectToURI (suri) removes one trailing slash.
objects_to_remove = [
"%s#%s" % (suri(ouri1), urigen(ouri1)),
"%s#%s" % (suri(ouri2), urigen(ouri2)),
"%s#%s" % (suri(ouri3), urigen(ouri3)),
"%s#%s" % (suri(ouri4), urigen(ouri4)),
"%s#%s" % (suri(ouri5) + "/", urigen(ouri5)),
"%s#%s" % (suri(ouri6) + "/", urigen(ouri6)),
"%s#%s" % (suri(ouri7) + "/", urigen(ouri7)),
"%s#%s" % (suri(ouri8) + "/", urigen(ouri8)),
"%s#%s" % (suri(ouri9) + "/", urigen(ouri9)),
]
self._RunRemoveCommandAndCheck(
["-m", "rm", "-r", suri(bucket_uri)],
objects_to_remove=objects_to_remove,
buckets_to_remove=[suri(bucket_uri)],
)
def test_rm_object_with_prefix_slash(self):
"""Tests removing a bucket that has an object starting with slash.
The boto lib used for S3 does not handle objects starting with slashes
if we use V4 signature. Hence we are testing objects with prefix
slashes separately.
"""
bucket_uri = self.CreateVersionedBucket()
ouri1 = self.CreateObject(bucket_uri=bucket_uri,
object_name='/dirwithslash/foo',
contents=b'z')
if self.multiregional_buckets:
self.AssertNObjectsInBucket(bucket_uri, 1, versioned=True)
self._RunRemoveCommandAndCheck(
['rm', '-r', suri(bucket_uri)],
objects_to_remove=['%s#%s' % (suri(ouri1), urigen(ouri1))],
buckets_to_remove=[suri(bucket_uri)])
def test_recursive_bucket_rm(self):
"""Test for 'rm -r' of a bucket."""
bucket_uri = self.CreateBucket()
object_uri = self.CreateObject(bucket_uri, contents="foo")
self.AssertNObjectsInBucket(bucket_uri, 1)
self._RunRemoveCommandAndCheck(
["rm", "-r", suri(bucket_uri)],
objects_to_remove=["%s#%s" % (suri(object_uri), urigen(object_uri))],
buckets_to_remove=[suri(bucket_uri)],
)
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
# Bucket should be deleted.
stderr = self.RunGsUtil(["ls", "-Lb", suri(bucket_uri)], return_stderr=True, expected_status=1)
self.assertIn("bucket does not exist", stderr)
_Check1()
# Now try same thing, but for a versioned bucket with multiple versions of
# an object present.
bucket_uri = self.CreateVersionedBucket()
self.CreateObject(bucket_uri, "obj", "z")
self.CreateObject(bucket_uri, "obj", "z")
final_uri = self.CreateObject(bucket_uri, "obj", "z")
self.AssertNObjectsInBucket(bucket_uri, 3, versioned=True)
self._RunRemoveCommandAndCheck(["rm", suri(bucket_uri, "**")], objects_to_remove=["%s" % final_uri])
stderr = self.RunGsUtil(["rb", suri(bucket_uri)], return_stderr=True, expected_status=1)
self.assertIn("Bucket is not empty", stderr)
# Now try with rm -r.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check2():
self.RunGsUtil(["rm", "-r", suri(bucket_uri)])
# Bucket should be deleted.
stderr = self.RunGsUtil(["ls", "-Lb", suri(bucket_uri)], return_stderr=True, expected_status=1)
self.assertIn("bucket does not exist", stderr)
_Check2()
def test_recursive_bucket_rm_with_wildcarding(self):
"""Tests removing all objects and buckets matching a bucket wildcard."""
buri_base = "gsutil-test-%s" % self.GetTestMethodName()
buri_base = buri_base[: MAX_BUCKET_LENGTH - 20]
buri_base = "%s-%s" % (buri_base, self.MakeRandomTestString())
buri1 = self.CreateBucket(bucket_name="%s-tbuck1" % buri_base)
buri2 = self.CreateBucket(bucket_name="%s-tbuck2" % buri_base)
buri3 = self.CreateBucket(bucket_name="%s-tb3" % buri_base)
ouri1 = self.CreateObject(bucket_uri=buri1, object_name="o1", contents="z")
ouri2 = self.CreateObject(bucket_uri=buri2, object_name="o2", contents="z")
self.CreateObject(bucket_uri=buri3, object_name="o3", contents="z")
self.AssertNObjectsInBucket(buri1, 1)
self.AssertNObjectsInBucket(buri2, 1)
self.AssertNObjectsInBucket(buri3, 1)
self._RunRemoveCommandAndCheck(
["rm", "-r", "%s://%s-tbu*" % (self.default_provider, buri_base)],
objects_to_remove=["%s#%s" % (suri(ouri1), urigen(ouri1)), "%s#%s" % (suri(ouri2), urigen(ouri2))],
buckets_to_remove=[suri(buri1), suri(buri2)],
)
self.AssertNObjectsInBucket(buri3, 1)
def _test_rewrite_resume_or_restart(self,
initial_dec_key,
initial_enc_key,
new_dec_key=None,
new_enc_key=None):
"""Tests that the rewrite command restarts if the object's key changed.
Args:
initial_dec_key: Initial key the object is encrypted with, used as
decryption key in the first rewrite call.
initial_enc_key: Initial encryption key to rewrite the object with,
used as encryption key in the first rewrite call.
new_dec_key: Decryption key for the second rewrite call; if specified,
object will be overwritten with a new encryption key in between
the first and second rewrite calls, and this key will be used for
the second rewrite call.
new_enc_key: Encryption key for the second rewrite call; if specified,
this key will be used for the second rewrite call, otherwise the
initial key will be used.
Returns:
None
"""
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
bucket_uri = self.CreateBucket()
# maxBytesPerCall must be >= 1 MiB, so create an object > 2 MiB because we
# need 2 response from the service: 1 success, 1 failure prior to
# completion.
object_uri = self.CreateObject(bucket_uri=bucket_uri,
object_name='foo',
contents=(b'12' * ONE_MIB) + b'bar',
prefer_json_api=True,
encryption_key=initial_dec_key)
gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(),
DiscardMessagesQueue(), self.default_provider)
with SetBotoConfigForTest([('GSUtil', 'decryption_key1', initial_dec_key)]):
src_obj_metadata = gsutil_api.GetObjectMetadata(
object_uri.bucket_name,
object_uri.object_name,
provider=self.default_provider,
fields=['bucket', 'contentType', 'etag', 'name'])
dst_obj_metadata = src_obj_metadata
tracker_file_name = GetRewriteTrackerFilePath(src_obj_metadata.bucket,
src_obj_metadata.name,
dst_obj_metadata.bucket,
dst_obj_metadata.name,
self.test_api)
decryption_tuple = CryptoKeyWrapperFromKey(initial_dec_key)
decryption_tuple2 = CryptoKeyWrapperFromKey(new_dec_key or initial_dec_key)
encryption_tuple = CryptoKeyWrapperFromKey(initial_enc_key)
encryption_tuple2 = CryptoKeyWrapperFromKey(new_enc_key or initial_enc_key)
try:
try:
gsutil_api.CopyObject(src_obj_metadata,
dst_obj_metadata,
progress_callback=HaltingRewriteCallbackHandler(
ONE_MIB * 2).call,
max_bytes_per_call=ONE_MIB,
decryption_tuple=decryption_tuple,
encryption_tuple=encryption_tuple)
self.fail('Expected RewriteHaltException.')
except RewriteHaltException:
pass
# Tracker file should be left over.
self.assertTrue(os.path.exists(tracker_file_name))
if new_dec_key:
# Recreate the object with a different encryption key.
self.CreateObject(bucket_uri=bucket_uri,
object_name='foo',
contents=(b'12' * ONE_MIB) + b'bar',
prefer_json_api=True,
encryption_key=new_dec_key,
gs_idempotent_generation=urigen(object_uri))
with SetBotoConfigForTest([('GSUtil', 'decryption_key1', new_dec_key or
initial_dec_key)]):
original_md5 = gsutil_api.GetObjectMetadata(
src_obj_metadata.bucket,
src_obj_metadata.name,
fields=['customerEncryption', 'md5Hash']).md5Hash
if new_dec_key or new_enc_key:
# Keys changed, rewrite should be restarted.
progress_callback = EnsureRewriteRestartCallbackHandler(ONE_MIB).call
else:
# Keys are the same, rewrite should be resumed.
progress_callback = EnsureRewriteResumeCallbackHandler(ONE_MIB * 2).call
# Now resume. Callback ensures the appropriate resume/restart behavior.
gsutil_api.CopyObject(src_obj_metadata,
dst_obj_metadata,
progress_callback=progress_callback,
max_bytes_per_call=ONE_MIB,
decryption_tuple=decryption_tuple2,
encryption_tuple=encryption_tuple2)
# Copy completed; tracker file should be deleted.
self.assertFalse(os.path.exists(tracker_file_name))
final_enc_key = new_enc_key or initial_enc_key
with SetBotoConfigForTest([('GSUtil', 'encryption_key', final_enc_key)]):
self.assertEqual(
original_md5,
gsutil_api.GetObjectMetadata(
dst_obj_metadata.bucket,
dst_obj_metadata.name,
fields=['customerEncryption', 'md5Hash']).md5Hash,
'Error: Rewritten object\'s hash doesn\'t match source object.')
finally:
# Clean up if something went wrong.
DeleteTrackerFile(tracker_file_name)
def _test_rewrite_resume_or_restart(self,
initial_dec_key,
initial_enc_key,
new_dec_key=None,
new_enc_key=None):
"""Tests that the rewrite command restarts if the object's key changed.
Args:
initial_dec_key: Initial key the object is encrypted with, used as
decryption key in the first rewrite call.
initial_enc_key: Initial encryption key to rewrite the object with,
used as encryption key in the first rewrite call.
new_dec_key: Decryption key for the second rewrite call; if specified,
object will be overwritten with a new encryption key in between
the first and second rewrite calls, and this key will be used for
the second rewrite call.
new_enc_key: Encryption key for the second rewrite call; if specified,
this key will be used for the second rewrite call, otherwise the
initial key will be used.
Returns:
None
"""
if self.test_api == ApiSelector.XML:
return unittest.skip('Rewrite API is only supported in JSON.')
bucket_uri = self.CreateBucket()
# maxBytesPerCall must be >= 1 MiB, so create an object > 2 MiB because we
# need 2 response from the service: 1 success, 1 failure prior to
# completion.
object_uri = self.CreateObject(bucket_uri=bucket_uri,
object_name='foo',
contents=('12' * ONE_MIB) + 'bar',
prefer_json_api=True,
encryption_key=initial_dec_key)
gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(),
DiscardMessagesQueue(), self.default_provider)
with SetBotoConfigForTest([('GSUtil', 'decryption_key1',
initial_dec_key)]):
src_obj_metadata = gsutil_api.GetObjectMetadata(
object_uri.bucket_name,
object_uri.object_name,
provider=self.default_provider,
fields=['bucket', 'contentType', 'etag', 'name'])
dst_obj_metadata = src_obj_metadata
tracker_file_name = GetRewriteTrackerFilePath(src_obj_metadata.bucket,
src_obj_metadata.name,
dst_obj_metadata.bucket,
dst_obj_metadata.name,
self.test_api)
decryption_tuple = CryptoKeyWrapperFromKey(initial_dec_key)
decryption_tuple2 = CryptoKeyWrapperFromKey(new_dec_key
or initial_dec_key)
encryption_tuple = CryptoKeyWrapperFromKey(initial_enc_key)
encryption_tuple2 = CryptoKeyWrapperFromKey(new_enc_key
or initial_enc_key)
try:
try:
gsutil_api.CopyObject(
src_obj_metadata,
dst_obj_metadata,
progress_callback=HaltingRewriteCallbackHandler(ONE_MIB *
2).call,
max_bytes_per_call=ONE_MIB,
decryption_tuple=decryption_tuple,
encryption_tuple=encryption_tuple)
self.fail('Expected RewriteHaltException.')
except RewriteHaltException:
pass
# Tracker file should be left over.
self.assertTrue(os.path.exists(tracker_file_name))
if new_dec_key:
# Recreate the object with a different encryption key.
self.CreateObject(bucket_uri=bucket_uri,
object_name='foo',
contents=('12' * ONE_MIB) + 'bar',
prefer_json_api=True,
encryption_key=new_dec_key,
gs_idempotent_generation=urigen(object_uri))
with SetBotoConfigForTest([('GSUtil', 'decryption_key1',
new_dec_key or initial_dec_key)]):
original_md5 = gsutil_api.GetObjectMetadata(
src_obj_metadata.bucket,
src_obj_metadata.name,
fields=['customerEncryption', 'md5Hash']).md5Hash
if new_dec_key or new_enc_key:
# Keys changed, rewrite should be restarted.
progress_callback = EnsureRewriteRestartCallbackHandler(
ONE_MIB).call
else:
# Keys are the same, rewrite should be resumed.
progress_callback = EnsureRewriteResumeCallbackHandler(
ONE_MIB * 2).call
# Now resume. Callback ensures the appropriate resume/restart behavior.
gsutil_api.CopyObject(src_obj_metadata,
dst_obj_metadata,
progress_callback=progress_callback,
max_bytes_per_call=ONE_MIB,
decryption_tuple=decryption_tuple2,
encryption_tuple=encryption_tuple2)
# Copy completed; tracker file should be deleted.
self.assertFalse(os.path.exists(tracker_file_name))
final_enc_key = new_enc_key or initial_enc_key
with SetBotoConfigForTest([('GSUtil', 'encryption_key',
final_enc_key)]):
self.assertEqual(
original_md5,
gsutil_api.GetObjectMetadata(
dst_obj_metadata.bucket,
dst_obj_metadata.name,
fields=['customerEncryption', 'md5Hash']).md5Hash,
'Error: Rewritten object\'s hash doesn\'t match source object.'
)
finally:
# Clean up if something went wrong.
DeleteTrackerFile(tracker_file_name)