def testInitialAndSubsequentCh(self):
bucket_uri = self.CreateBucket()
ch_subargs = ['-l', '%s:%s' % (KEY1, VALUE1),
'-l', '%s:%s' % (KEY2, VALUE2)]
# Ensure 'ch' progress message shows in stderr.
stderr = self.RunGsUtil(
['label', 'ch'] + ch_subargs + [suri(bucket_uri)],
return_stderr=True)
self.assertEqual(
stderr.strip(),
LABEL_SETTING_OUTPUT % suri(bucket_uri))
# Check the bucket to ensure it's configured with the labels we just
# specified.
stdout = self.RunGsUtil(
['label', 'get', suri(bucket_uri)], return_stdout=True)
self.assertItemsEqual(json.loads(stdout), self._label_dict)
# Ensure a subsequent 'ch' command works correctly.
new_key = 'new-key'
new_value = 'new-value'
self.RunGsUtil([
'label', 'ch', '-l', '%s:%s' % (new_key, new_value), '-d', KEY2,
suri(bucket_uri)])
stdout = self.RunGsUtil(
['label', 'get', suri(bucket_uri)], return_stdout=True)
actual = json.loads(stdout)
expected = {KEY1: VALUE1, new_key: new_value}
self.assertItemsEqual(actual, expected)
def testCh(self):
bucket_uri = self.CreateBucket()
self.RunGsUtil(['label', 'ch',
'-l', '%s:%s' % (KEY1, VALUE1),
'-l', '%s:%s' % (KEY2, VALUE2),
suri(bucket_uri)])
# Verify that the bucket is configured with the labels we just set.
# Work around eventual consistency for S3 tagging.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
stdout = self.RunGsUtil(
['label', 'get', suri(bucket_uri)], return_stdout=True)
self.assertItemsEqual(
self._LabelDictFromXmlString(stdout),
self._LabelDictFromXmlString(self._label_xml))
_Check1()
# Remove KEY1, add a new key, and attempt to remove a nonexistent key
# with 'label ch'.
self.RunGsUtil(['label', 'ch',
'-d', KEY1,
'-l', 'new_key:new_value',
'-d', 'nonexistent-key',
suri(bucket_uri)])
expected_dict = {KEY2: VALUE2, 'new_key': 'new_value'}
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check2():
stdout = self.RunGsUtil(
['label', 'get', suri(bucket_uri)], return_stdout=True)
self.assertItemsEqual(
self._LabelDictFromXmlString(stdout),
expected_dict)
_Check2()
def test_stat_encrypted_object(self):
"""Tests stat command with an encrypted object."""
if self.test_api == ApiSelector.XML:
return unittest.skip("gsutil does not support encryption with the XML API")
bucket_uri = self.CreateBucket()
object_uri = self.CreateObject(
bucket_uri=bucket_uri,
object_name="foo",
contents=TEST_ENCRYPTION_CONTENT1,
encryption_key=TEST_ENCRYPTION_KEY1,
)
# Stat object with key should return unencrypted hashes.
with SetBotoConfigForTest([("GSUtil", "encryption_key", TEST_ENCRYPTION_KEY1)]):
stdout = self.RunGsUtil(["stat", suri(object_uri)], return_stdout=True)
self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout)
self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout)
self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout)
# Stat object without key should return encrypted hashes.
stdout = self.RunGsUtil(["stat", suri(object_uri)], return_stdout=True)
self.assertNotIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout)
self.assertNotIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout)
self.assertIn("encrypted", stdout)
self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout)
def test_set_empty_lifecycle1(self):
bucket_uri = self.CreateBucket()
fpath = self.CreateTempFile(contents=self.empty_doc1.encode('ascii'))
self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)])
stdout = self.RunGsUtil(
['lifecycle', 'get', suri(bucket_uri)], return_stdout=True)
self.assertIn(self.no_lifecycle_config, stdout)
def test_remove_all_versions_recursive_on_bucket(self):
"""Test that 'rm -ar' works on bucket."""
bucket_uri = self.CreateVersionedBucket()
k1_uri = bucket_uri.clone_replace_name('foo')
k2_uri = bucket_uri.clone_replace_name('foo2')
k1_uri.set_contents_from_string('bar')
k2_uri.set_contents_from_string('bar2')
k1g1 = k1_uri.generation
k2g1 = k2_uri.generation
k1_uri.set_contents_from_string('baz')
k2_uri.set_contents_from_string('baz2')
k1g2 = k1_uri.generation
k2g2 = k2_uri.generation
stderr = self.RunGsUtil(['rm', '-ar', suri(bucket_uri)],
return_stderr=True)
self.assertEqual(stderr.count('Removing gs://'), 4)
self.assertIn('Removing %s#%s...' % (suri(k1_uri), k1g1), stderr)
self.assertIn('Removing %s#%s...' % (suri(k1_uri), k1g2), stderr)
self.assertIn('Removing %s#%s...' % (suri(k2_uri), k2g1), stderr)
self.assertIn('Removing %s#%s...' % (suri(k2_uri), k2g2), stderr)
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, delay=1, backoff=1)
def _Check1():
stdout = self.RunGsUtil(['ls', '-a', suri(bucket_uri)],
return_stdout=True)
self.assertEqual(stdout, '')
_Check1()
def test_wildcard_prefix(self):
"""Tests that an object name with a wildcard does not infinite loop."""
bucket_uri = self.CreateBucket()
wildcard_folder_object = 'wildcard*/'
object_matching_folder = 'wildcard10/foo'
self.CreateObject(bucket_uri=bucket_uri,
object_name=wildcard_folder_object,
contents=b'foo')
self.CreateObject(bucket_uri=bucket_uri,
object_name=object_matching_folder,
contents=b'foo')
self.AssertNObjectsInBucket(bucket_uri, 2)
stderr = self.RunGsUtil(['ls', suri(bucket_uri, 'wildcard*')],
return_stderr=True,
expected_status=1)
self.assertIn(
'Cloud folder %s%s contains a wildcard' %
(suri(bucket_uri), '/wildcard*/'), stderr)
# Listing with a flat wildcard should still succeed.
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
stdout = self.RunGsUtil(['ls', '-l', suri(bucket_uri, '**')],
return_stdout=True)
self.assertNumLines(stdout, 3) # 2 object lines, one summary line.
_Check()
def test_versioning(self):
"""Tests listing a versioned bucket."""
bucket1_uri = self.CreateBucket(test_objects=1)
bucket2_uri = self.CreateVersionedBucket(test_objects=1)
self.AssertNObjectsInBucket(bucket1_uri, 1, versioned=True)
bucket_list = list(bucket1_uri.list_bucket())
objuri = [
bucket1_uri.clone_replace_key(key).versionless_uri
for key in bucket_list
][0]
self.RunGsUtil(['cp', objuri, suri(bucket2_uri)])
self.RunGsUtil(['cp', objuri, suri(bucket2_uri)])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check2():
stdout = self.RunGsUtil(['ls', '-a', suri(bucket2_uri)],
return_stdout=True)
self.assertNumLines(stdout, 3)
stdout = self.RunGsUtil(['ls', '-la', suri(bucket2_uri)],
return_stdout=True)
self.assertIn('%s#' % bucket2_uri.clone_replace_name(bucket_list[0].name),
stdout)
self.assertIn('metageneration=', stdout)
_Check2()
def _Check1():
stdout = self.RunGsUtil(['ls', '-d', '%s/dir*' % suri(bucket_uri)],
return_stdout=True)
self.assertEqual('%s/dir/\n%s/dir2/\n' %
(suri(bucket_uri), suri(bucket_uri)), stdout)
stdout = self.RunGsUtil(['ls', suri(k1_uri)], return_stdout=True)
self.assertEqual('%s\n' % suri(k1_uri), stdout)
def _Check():
stdout = self.RunGsUtil(['du', suri(obj_uri1), suri(obj_uri2)],
return_stdout=True)
self.assertSetEqual(set(stdout.splitlines()), set([
'%-10s %s' % (3, suri(obj_uri1)),
'%-10s %s' % (4, suri(obj_uri2)),
]))
def test_valid_lifecycle(self):
bucket_uri = self.CreateBucket()
fpath = self.CreateTempFile(contents=self.lifecycle_doc.encode('ascii'))
self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)])
stdout = self.RunGsUtil(
['lifecycle', 'get', suri(bucket_uri)], return_stdout=True)
self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
def test_stdin_args(self):
"""Tests rm with the -I option."""
buri1 = self.CreateVersionedBucket()
ouri1 = self.CreateObject(bucket_uri=buri1,
object_name='foo',
contents='foocontents')
self.CreateObject(bucket_uri=buri1, object_name='bar',
contents='barcontents')
ouri3 = self.CreateObject(bucket_uri=buri1,
object_name='baz',
contents='bazcontents')
buri2 = self.CreateVersionedBucket()
ouri4 = self.CreateObject(bucket_uri=buri2,
object_name='moo',
contents='moocontents')
self.AssertNObjectsInBucket(buri1, 3, versioned=True)
self.AssertNObjectsInBucket(buri2, 1, versioned=True)
objects_to_remove = ['%s#%s' % (suri(ouri1), urigen(ouri1)),
'%s#%s' % (suri(ouri3), urigen(ouri3)),
'%s#%s' % (suri(ouri4), urigen(ouri4))]
stdin = '\n'.join(objects_to_remove)
self._RunRemoveCommandAndCheck(['rm', '-I'],
objects_to_remove=objects_to_remove,
stdin=stdin)
self.AssertNObjectsInBucket(buri1, 1, versioned=True)
self.AssertNObjectsInBucket(buri2, 0, versioned=True)
def testMultithreadedAclChange(self, count=10):
"""Tests multi-threaded acl changing on several objects."""
objects = []
for i in range(count):
objects.append(self.CreateObject(
bucket_uri=self.sample_uri,
contents='something {0}'.format(i)))
self.AssertNObjectsInBucket(self.sample_uri, count)
test_regex = self._MakeScopeRegex(
'READER', 'group', self.GROUP_TEST_ADDRESS)
json_texts = []
for obj in objects:
json_texts.append(self.RunGsUtil(
self._get_acl_prefix + [suri(obj)], return_stdout=True))
for json_text in json_texts:
self.assertNotRegexpMatches(json_text, test_regex)
uris = [suri(obj) for obj in objects]
self.RunGsUtil(['-m', '-DD'] + self._ch_acl_prefix +
['-g', self.GROUP_TEST_ADDRESS+':READ'] + uris)
json_texts = []
for obj in objects:
json_texts.append(self.RunGsUtil(
self._get_acl_prefix + [suri(obj)], return_stdout=True))
for json_text in json_texts:
self.assertRegexpMatches(json_text, test_regex)
def _AddAcl():
self.RunGsUtil(
self._ch_acl_prefix +
['-R', '-g', self.GROUP_TEST_ADDRESS+':READ', suri(obj)[:-3]])
json_text = self.RunGsUtil(self._get_acl_prefix + [suri(obj)],
return_stdout=True)
self.assertRegexpMatches(json_text, test_regex)
def testProjectAclChangesOnBucket(self):
"""Tests project entity acl changes on a bucket."""
if self.test_api == ApiSelector.XML:
stderr = self.RunGsUtil(self._ch_acl_prefix +
['-p', self._project_test_acl +':w',
suri(self.sample_uri)],
expected_status=1,
return_stderr=True)
self.assertIn(('CommandException: XML API does not support project'
' scopes, cannot translate ACL.'), stderr)
else:
test_regex = self._MakeProjectScopeRegex(
'WRITER', self._project_team, self._project_number)
self.RunGsUtil(self._ch_acl_prefix +
['-p', self._project_test_acl +':w',
suri(self.sample_uri)])
json_text = self.RunGsUtil(
self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True)
self.assertRegexpMatches(json_text, test_regex)
self.RunGsUtil(self._ch_acl_prefix +
['-d', self._project_test_acl, suri(self.sample_uri)])
json_text2 = self.RunGsUtil(
self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True)
self.assertNotRegexpMatches(json_text2, test_regex)
def _Check1():
command = ['ls', '-a'] if versioned else ['ls']
b_uri = [suri(bucket_uri) + '/**'] if num_objects else [suri(bucket_uri)]
listing = self.RunGsUtil(command + b_uri, return_stdout=True).split('\n')
# num_objects + one trailing newline.
self.assertEquals(len(listing), num_objects + 1)
return listing
def test_set_valid_def_acl_bucket(self):
"""Ensures that valid default canned and XML ACLs works with get/set."""
bucket_uri = self.CreateBucket()
# Default ACL is project private.
obj_uri1 = suri(self.CreateObject(bucket_uri=bucket_uri, contents="foo"))
acl_string = self.RunGsUtil(self._get_acl_prefix + [obj_uri1], return_stdout=True)
# Change it to authenticated-read.
self.RunGsUtil(self._set_defacl_prefix + ["authenticated-read", suri(bucket_uri)])
# Default object ACL may take some time to propagate.
@Retry(AssertionError, tries=5, timeout_secs=1)
def _Check1():
obj_uri2 = suri(self.CreateObject(bucket_uri=bucket_uri, contents="foo2"))
acl_string2 = self.RunGsUtil(self._get_acl_prefix + [obj_uri2], return_stdout=True)
self.assertNotEqual(acl_string, acl_string2)
self.assertIn("allAuthenticatedUsers", acl_string2)
_Check1()
# Now change it back to the default via XML.
inpath = self.CreateTempFile(contents=acl_string)
self.RunGsUtil(self._set_defacl_prefix + [inpath, suri(bucket_uri)])
# Default object ACL may take some time to propagate.
@Retry(AssertionError, tries=5, timeout_secs=1)
def _Check2():
obj_uri3 = suri(self.CreateObject(bucket_uri=bucket_uri, contents="foo3"))
acl_string3 = self.RunGsUtil(self._get_acl_prefix + [obj_uri3], return_stdout=True)
self.assertEqual(acl_string, acl_string3)
_Check2()
def test_list_one_entry(self):
"""Tests notification config listing with one entry."""
if self.test_api == ApiSelector.XML:
return unittest.skip('Notifications only work with the JSON API.')
bucket_uri = self.CreateBucket()
bucket_name = bucket_uri.bucket_name
topic_name = self._RegisterDefaultTopicCreation(bucket_uri.bucket_name)
self.RunGsUtil([
'notification', 'create', '-f', 'json', '-e', 'OBJECT_FINALIZE', '-e',
'OBJECT_DELETE', '-m', 'someKey:someValue', '-p', 'somePrefix',
suri(bucket_uri)
],
return_stderr=True)
stdout = self.RunGsUtil(
['notification', 'list', suri(bucket_uri)], return_stdout=True)
self.assertEquals(
stdout, ('projects/_/buckets/{bucket_name}/notificationConfigs/1\n'
'\tCloud Pub/Sub topic: {topic_name}\n'
'\tCustom attributes:\n'
'\t\tsomeKey: someValue\n'
'\tFilters:\n'
'\t\tEvent Types: OBJECT_FINALIZE, OBJECT_DELETE\n'
'\t\tObject name prefix: \'somePrefix\'\n'.format(
bucket_name=bucket_name, topic_name=topic_name)))
def test_rb(self):
rp_bucket_uri = self.CreateBucket()
self._set_requester_pays(rp_bucket_uri)
self._run_requester_pays_test(['rb', suri(rp_bucket_uri)])
non_rp_bucket_uri = self.CreateBucket()
self._run_non_requester_pays_test(['rb', suri(non_rp_bucket_uri)])
def test_full(self):
bucket_uri = self.CreateBucket()
self.RunGsUtil(
self._set_web_cmd + ['-m', 'main', '-e', '404', suri(bucket_uri)])
stdout = self.RunGsUtil(
self._get_web_cmd + [suri(bucket_uri)], return_stdout=True)
self.assertEquals(json.loads(stdout), WEBCFG_FULL)
def test_setmeta_valid_with_multiple_colons_in_value(self):
obj_uri = self.CreateObject(contents='foo')
self.RunGsUtil([
'setmeta', '-h', 'x-%s-meta-foo:bar:baz' % self.provider_custom_meta,
suri(obj_uri)])
stdout = self.RunGsUtil(['stat', suri(obj_uri)], return_stdout=True)
self.assertRegexpMatches(stdout, r'foo:\s+bar:baz')
def test_stat_bucket_wildcard(self): bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents='z') stat_string = suri(bucket_uri)[:-1] + '?/foo' self.RunGsUtil(['stat', stat_string]) stat_string2 = suri(bucket_uri)[:-1] + '*/foo' self.RunGsUtil(['stat', stat_string2])
def setUp(self):
"""Creates 2 mock buckets, each containing 4 objects, including 1 nested."""
super(CloudWildcardIteratorTests, self).setUp()
self.immed_child_obj_names = ['abcd', 'abdd', 'ade$']
self.all_obj_names = ['abcd', 'abdd', 'ade$', 'nested1/nested2/xyz1',
'nested1/nested2/xyz2', 'nested1/nfile_abc']
self.base_bucket_uri = self.CreateBucket()
self.prefix_bucket_name = '%s_' % self.base_bucket_uri.bucket_name[:61]
self.base_uri_str = suri(self.base_bucket_uri)
self.base_uri_str = self.base_uri_str.replace(
self.base_bucket_uri.bucket_name, self.prefix_bucket_name)
self.test_bucket0_uri = self.CreateBucket(
bucket_name='%s0' % self.prefix_bucket_name)
self.test_bucket0_obj_uri_strs = set()
for obj_name in self.all_obj_names:
obj_uri = self.CreateObject(bucket_uri=self.test_bucket0_uri,
object_name=obj_name, contents='')
self.test_bucket0_obj_uri_strs.add(suri(obj_uri))
self.test_bucket1_uri = self.CreateBucket(
bucket_name='%s1' % self.prefix_bucket_name)
self.test_bucket1_obj_uri_strs = set()
for obj_name in self.all_obj_names:
obj_uri = self.CreateObject(bucket_uri=self.test_bucket1_uri,
object_name=obj_name, contents='')
self.test_bucket1_obj_uri_strs.add(suri(obj_uri))
def test_bucket_list_wildcard(self):
"""Tests listing multiple buckets with a wildcard."""
random_prefix = self.MakeRandomTestString()
bucket1_name = self.MakeTempName("bucket", prefix=random_prefix)
bucket2_name = self.MakeTempName("bucket", prefix=random_prefix)
bucket1_uri = self.CreateBucket(bucket_name=bucket1_name)
bucket2_uri = self.CreateBucket(bucket_name=bucket2_name)
# This just double checks that the common prefix of the two buckets is what
# we think it should be (based on implementation detail of CreateBucket).
# We want to be careful when setting a wildcard on buckets to make sure we
# don't step outside the test buckets to affect other buckets.
common_prefix = posixpath.commonprefix([suri(bucket1_uri), suri(bucket2_uri)])
self.assertTrue(
common_prefix.startswith(
"%s://%sgsutil-test-test_bucket_list_wildcard-bucket-" % (self.default_provider, random_prefix)
)
)
wildcard = "%s*" % common_prefix
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
stdout = self.RunGsUtil(["ls", "-b", wildcard], return_stdout=True)
expected = set([suri(bucket1_uri) + "/", suri(bucket2_uri) + "/"])
actual = set(stdout.split())
self.assertEqual(expected, actual)
_Check1()
def test_bucket_to_bucket_minus_d_empty_src(self):
"""Tests working with empty src bucket (iter runs out before dst iter)."""
bucket1_uri = self.CreateBucket()
bucket2_uri = self.CreateBucket()
self.CreateObject(bucket_uri=bucket2_uri, object_name='obj1',
contents='obj1')
self.CreateObject(bucket_uri=bucket2_uri, object_name='obj2',
contents='obj2')
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
self.RunGsUtil(['rsync', '-d', suri(bucket1_uri), suri(bucket2_uri)])
stderr = self.RunGsUtil(['ls', suri(bucket1_uri, '**')],
expected_status=1, return_stderr=True)
self.assertIn('One or more URLs matched no objects', stderr)
stderr = self.RunGsUtil(['ls', suri(bucket2_uri, '**')],
expected_status=1, return_stderr=True)
self.assertIn('One or more URLs matched no objects', stderr)
_Check()
# Check that re-running the same rsync command causes no more changes.
self.assertEquals(NO_CHANGES, self.RunGsUtil(
['rsync', '-d', suri(bucket1_uri), suri(bucket2_uri)],
return_stderr=True))
def test_recursive_bucket_rm(self):
"""Test for 'rm -r' of a bucket."""
bucket_uri = self.CreateBucket()
self.CreateObject(bucket_uri)
self.RunGsUtil(['rm', '-r', suri(bucket_uri)])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check1():
# Bucket should be deleted.
stderr = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)],
return_stderr=True, expected_status=1)
self.assertIn('bucket does not exist', stderr)
_Check1()
# Now try same thing, but for a versioned bucket with multiple versions of
# an object present.
bucket_uri = self.CreateVersionedBucket()
self.CreateObject(bucket_uri, 'obj', 'z')
self.CreateObject(bucket_uri, 'obj', 'z')
self.CreateObject(bucket_uri, 'obj', 'z')
self.AssertNObjectsInBucket(bucket_uri, 3, versioned=True)
self.RunGsUtil(['rm', suri(bucket_uri, '**')])
stderr = self.RunGsUtil(['rb', suri(bucket_uri)],
return_stderr=True, expected_status=1)
self.assertIn('Bucket is not empty', stderr)
# Now try with rm -r.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check2():
self.RunGsUtil(['rm', '-r', suri(bucket_uri)])
# Bucket should be deleted.
stderr = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)],
return_stderr=True, expected_status=1)
self.assertIn('bucket does not exist', stderr)
_Check2()
def test_one_object_with_L(self):
"""Tests listing one object with -L."""
obj_uri = self.CreateObject(contents='foo')
# Ensure that creation and update don't take place in the same second.
time.sleep(2)
# Check that the creation time, rather than the updated time, is displayed.
self.RunGsUtil(['setmeta', '-h', 'x-goog-meta-foo:bar', suri(obj_uri)])
find_time_created_re = re.compile(
r'^\s*Creation time:\s+(?P<time_created_val>.+)$', re.MULTILINE)
find_time_updated_re = re.compile(
r'^\s*Update time:\s+(?P<time_updated_val>.+)$', re.MULTILINE)
stdout = self.RunGsUtil(['ls', '-L', suri(obj_uri)], return_stdout=True)
time_created_match = re.search(find_time_created_re, stdout)
time_updated_match = re.search(find_time_updated_re, stdout)
time_created = time_created_match.group('time_created_val')
self.assertIsNotNone(time_created)
time_created = time.strptime(time_created, '%a, %d %b %Y %H:%M:%S %Z')
if self.test_api == ApiSelector.XML:
# XML API has no concept of updated time.
self.assertIsNone(time_updated_match)
elif self.test_api == ApiSelector.JSON:
time_updated = time_updated_match.group('time_updated_val')
self.assertIsNotNone(time_updated)
time_updated = time.strptime(time_updated, '%a, %d %b %Y %H:%M:%S %Z')
self.assertGreater(time_updated, time_created)
def testNoOpDirectoryIterator(self):
"""Tests that directory-only URI iterates just that one URI."""
results = list(
self._test_wildcard_iterator(suri(tempfile.tempdir)).IterAll(
expand_top_level_buckets=True))
self.assertEqual(1, len(results))
self.assertEqual(suri(tempfile.tempdir), str(results[0]))
def test_bucket_to_bucket_minus_d_with_composites(self):
"""Tests that rsync works with composite objects (which don't have MD5s)."""
bucket1_uri = self.CreateBucket()
bucket2_uri = self.CreateBucket()
self.CreateObject(bucket_uri=bucket1_uri, object_name='obj1',
contents='obj1')
self.CreateObject(bucket_uri=bucket1_uri, object_name='obj2',
contents='obj2')
self.RunGsUtil(
['compose', suri(bucket1_uri, 'obj1'), suri(bucket1_uri, 'obj2'),
suri(bucket1_uri, 'obj3')])
self.CreateObject(bucket_uri=bucket2_uri, object_name='obj2',
contents='OBJ2')
self.CreateObject(bucket_uri=bucket2_uri, object_name='obj4',
contents='obj4')
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
self.RunGsUtil(['rsync', '-d', suri(bucket1_uri), suri(bucket2_uri)])
listing1 = _TailSet(suri(bucket1_uri), self._FlatListBucket(bucket1_uri))
listing2 = _TailSet(suri(bucket2_uri), self._FlatListBucket(bucket2_uri))
# First bucket should have un-altered content.
self.assertEquals(listing1, set(['/obj1', '/obj2', '/obj3']))
# Second bucket should have content like first bucket but without the
# subdir objects synchronized.
self.assertEquals(listing2, set(['/obj1', '/obj2', '/obj3']))
_Check()
# Check that re-running the same rsync command causes no more changes.
self.assertEquals(NO_CHANGES, self.RunGsUtil(
['rsync', '-d', suri(bucket1_uri), suri(bucket2_uri)],
return_stderr=True))
def test_stat_bucket_wildcard(self):
bucket_uri = self.CreateBucket()
self.CreateObject(bucket_uri=bucket_uri, object_name="foo", contents="z")
stat_string = suri(bucket_uri)[:-1] + "?/foo"
self.RunGsUtil(["stat", stat_string])
stat_string2 = suri(bucket_uri)[:-1] + "*/foo"
self.RunGsUtil(["stat", stat_string2])
def test_rsync_minus_d_minus_p(self):
"""Tests that rsync -p preserves ACLs."""
bucket1_uri = self.CreateBucket()
bucket2_uri = self.CreateBucket()
self.CreateObject(bucket_uri=bucket1_uri, object_name='obj1',
contents='obj1')
# Set public-read (non-default) ACL so we can verify that rsync -p works.
self.RunGsUtil(['acl', 'set', 'public-read', suri(bucket1_uri, 'obj1')])
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
"""Tests rsync -p works as expected."""
self.RunGsUtil(['rsync', '-d', '-p', suri(bucket1_uri),
suri(bucket2_uri)])
listing1 = _TailSet(suri(bucket1_uri), self._FlatListBucket(bucket1_uri))
listing2 = _TailSet(suri(bucket2_uri), self._FlatListBucket(bucket2_uri))
self.assertEquals(listing1, set(['/obj1']))
self.assertEquals(listing2, set(['/obj1']))
acl1_json = self.RunGsUtil(['acl', 'get', suri(bucket1_uri, 'obj1')],
return_stdout=True)
acl2_json = self.RunGsUtil(['acl', 'get', suri(bucket2_uri, 'obj1')],
return_stdout=True)
self.assertEquals(acl1_json, acl2_json)
_Check()
# Check that re-running the same rsync command causes no more changes.
self.assertEquals(NO_CHANGES, self.RunGsUtil(
['rsync', '-d', '-p', suri(bucket1_uri), suri(bucket2_uri)],
return_stderr=True))
def FlatListBucket(self, bucket_url_string):
"""Perform a flat listing over bucket_url_string."""
return self.RunGsUtil(['ls', suri(bucket_url_string, '**')],
return_stdout=True)
def test_rb_bucket_not_empty(self):
bucket_uri = self.CreateBucket(test_objects=1)
stderr = self.RunGsUtil(['rb', suri(bucket_uri)],
expected_status=1,
return_stderr=True)
self.assertIn('BucketNotEmpty', stderr)
def _ListExpectDecrypted():
stdout = self.RunGsUtil(['ls', '-L', suri(object_uri)],
return_stdout=True)
self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout)
self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout)
self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout)
def _Check():
# Bucket should no longer exist.
stderr = self.RunGsUtil(['ls', '-a', suri(bucket_uri)],
return_stderr=True,
expected_status=1)
self.assertIn('bucket does not exist', stderr)
def _Check1():
stdout = self.RunGsUtil(['ls', '-lb', suri(bucket_uri)],
return_stdout=True)
self.assertIn(suri(bucket_uri), stdout)
self.assertNotIn('TOTAL:', stdout)
def _Check2():
obj_uri3 = suri(
self.CreateObject(bucket_uri=bucket_uri, contents='foo3'))
acl_string3 = self.RunGsUtil(self._get_acl_prefix + [obj_uri3],
return_stdout=True)
self.assertEqual(acl_string, acl_string3)
def _Check1():
stdout = self.RunGsUtil(['ls', suri(bucket_uri)], return_stdout=True)
self.assertEqual('%s\n' % obj_uri, stdout)
def test_list_missing_object(self):
"""Tests listing a non-existent object."""
bucket_uri = self.CreateBucket()
stderr = self.RunGsUtil(['ls', suri(bucket_uri, 'missing')],
return_stderr=True, expected_status=1)
self.assertIn('matched no objects', stderr)
def _Check1(): stdout = self.RunGsUtil(['ls', '-b', wildcard], return_stdout=True) expected = set([suri(bucket1_uri) + '/', suri(bucket2_uri) + '/']) actual = set(stdout.split()) self.assertEqual(expected, actual)
def _Check1(): stdout = self.RunGsUtil(['ls', suri(bucket_uri)], return_stdout=True) self.assertIn(os.path.basename(fpath1), stdout) self.assertIn(os.path.basename(fpath2), stdout) self.assertNumLines(stdout, 2)
def test_cp_key_to_local_stream(self): bucket_uri = self.CreateBucket() contents = 'foo' key_uri = self.CreateObject(bucket_uri=bucket_uri, contents=contents) stdout = self.RunGsUtil(['cp', suri(key_uri), '-'], return_stdout=True) self.assertIn(contents, stdout)
def test_update(self):
"""Tests that the update command works or raises proper exceptions."""
if system_util.InvokedViaCloudSdk():
stderr = self.RunGsUtil(['update'],
stdin='n',
return_stderr=True,
expected_status=1)
self.assertIn('update command is disabled for Cloud SDK', stderr)
return
if gslib.IS_PACKAGE_INSTALL:
# The update command is not present when installed via package manager.
stderr = self.RunGsUtil(['update'], return_stderr=True, expected_status=1)
self.assertIn('Invalid command', stderr)
return
# Create two temp directories, one of which we will run 'gsutil update' in
# to pull the changes from the other.
tmpdir_src = self.CreateTempDir()
tmpdir_dst = self.CreateTempDir()
# Copy gsutil to both source and destination directories.
gsutil_src = os.path.join(tmpdir_src, 'gsutil')
gsutil_dst = os.path.join(tmpdir_dst, 'gsutil')
# Path when executing from tmpdir (Windows doesn't support in-place rename)
gsutil_relative_dst = os.path.join('gsutil', 'gsutil')
ignore_callable = shutil.ignore_patterns(
'.git*',
'*.pyc',
'*.pyo',
'__pycache__',
)
shutil.copytree(GSUTIL_DIR, gsutil_src, ignore=ignore_callable)
# Copy specific files rather than all of GSUTIL_DIR so we don't pick up temp
# working files left in top-level directory by gsutil developers (like tags,
# .git*, .pyc files, etc.)
os.makedirs(gsutil_dst)
for comp in os.listdir(GSUTIL_DIR):
if ('.git' not in comp and
'__pycache__' not in comp and
not comp.endswith('.pyc') and
not comp.endswith('.pyo')): # yapf: disable
cp_src_path = os.path.join(GSUTIL_DIR, comp)
cp_dst_path = os.path.join(gsutil_dst, comp)
if os.path.isdir(cp_src_path):
shutil.copytree(cp_src_path, cp_dst_path, ignore=ignore_callable)
else:
shutil.copyfile(cp_src_path, cp_dst_path)
# Create a fake version number in the source so we can verify it in the
# destination.
expected_version = '17.25'
src_version_file = os.path.join(gsutil_src, 'VERSION')
self.assertTrue(os.path.exists(src_version_file))
with open(src_version_file, 'w') as f:
f.write(expected_version)
# Create a tarball out of the source directory and copy it to a bucket.
src_tarball = os.path.join(tmpdir_src, 'gsutil.test.tar.gz')
normpath = os.path.normpath
try:
# We monkey patch os.path.normpath here because the tarfile module
# normalizes the ./gsutil path, but the update command expects the tar
# file to be prefixed with . This preserves the ./gsutil path.
os.path.normpath = lambda fname: fname
tar = tarfile.open(src_tarball, 'w:gz')
tar.add(gsutil_src, arcname='./gsutil')
tar.close()
finally:
os.path.normpath = normpath
prefix = [sys.executable] if sys.executable else []
# Run with an invalid gs:// URI.
p = subprocess.Popen(prefix + ['gsutil', 'update', 'gs://pub'],
cwd=gsutil_dst,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(_, stderr) = p.communicate()
p.stdout.close()
p.stderr.close()
self.assertEqual(p.returncode, 1)
self.assertIn(b'update command only works with tar.gz', stderr)
# Run with non-existent gs:// URI.
p = subprocess.Popen(prefix +
['gsutil', 'update', 'gs://pub/Jdjh38)(;.tar.gz'],
cwd=gsutil_dst,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(_, stderr) = p.communicate()
p.stdout.close()
p.stderr.close()
self.assertEqual(p.returncode, 1)
self.assertIn(b'NotFoundException', stderr)
# Run with file:// URI wihout -f option.
p = subprocess.Popen(
prefix + ['gsutil', 'update', suri(src_tarball)],
cwd=gsutil_dst,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(_, stderr) = p.communicate()
p.stdout.close()
p.stderr.close()
self.assertEqual(p.returncode, 1)
self.assertIn(b'command does not support', stderr)
# Run with a file present that was not distributed with gsutil.
with open(os.path.join(gsutil_dst, 'userdata.txt'), 'w') as fp:
fp.write('important data\n')
p = subprocess.Popen(
prefix +
['gsutil', 'update', '-f', suri(src_tarball)],
cwd=gsutil_dst,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE)
(_, stderr) = p.communicate()
p.stdout.close()
p.stderr.close()
# Clean up before next test, and before assertions so failure doesn't leave
# this file around.
os.unlink(os.path.join(gsutil_dst, 'userdata.txt'))
self.assertEqual(p.returncode, 1)
# Additional check for Windows since it has \r\n and string may have just \n
os_ls = os.linesep.encode(UTF8)
if os_ls in stderr:
stderr = stderr.replace(os_ls, b' ')
elif b'\n' in stderr:
stderr = stderr.replace(b'\n', b' ')
self.assertIn(
b'The update command cannot run with user data in the gsutil directory',
stderr)
# Determine whether we'll need to decline the analytics prompt.
analytics_prompt = not (os.path.exists(_UUID_FILE_PATH) or
boto.config.get_value('GSUtil',
'disable_analytics_prompt'))
update_input = b'n\r\ny\r\n' if analytics_prompt else b'y\r\n'
# Now do the real update, which should succeed.
p = subprocess.Popen(
prefix + [gsutil_relative_dst, 'update', '-f',
suri(src_tarball)],
cwd=tmpdir_dst,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE)
(_, stderr) = p.communicate(input=update_input)
p.stdout.close()
p.stderr.close()
self.assertEqual(
p.returncode,
0,
msg=('Non-zero return code (%d) from gsutil update. stderr = \n%s' %
(p.returncode, stderr.decode(UTF8))))
# Verify that version file was updated.
dst_version_file = os.path.join(tmpdir_dst, 'gsutil', 'VERSION')
with open(dst_version_file, 'r') as f:
self.assertEqual(f.read(), expected_version)
# If the analytics prompt was given, that means we disabled analytics. We
# should reset to the default by deleting the UUID file.
if analytics_prompt:
os.unlink(_UUID_FILE_PATH)
def testAclBucketGetSet(self):
bucket_uri = self.CreateBucket()
stdout = self.RunGsUtil(self._get_acl_prefix + [suri(bucket_uri)],
return_stdout=True)
set_contents = self.CreateTempFile(contents=stdout)
self.RunGsUtil(self._set_acl_prefix + [set_contents, suri(bucket_uri)])
def test_moving(self):
"""Tests moving two buckets, one with 2 objects and one with 0 objects."""
bucket1_uri = self.CreateBucket(test_objects=2)
self.AssertNObjectsInBucket(bucket1_uri, 2)
bucket2_uri = self.CreateBucket()
self.AssertNObjectsInBucket(bucket2_uri, 0)
# Move two objects from bucket1 to bucket2.
objs = [
bucket1_uri.clone_replace_key(key).versionless_uri
for key in bucket1_uri.list_bucket()
]
cmd = (['-m', 'mv'] + objs + [suri(bucket2_uri)])
stderr = self.RunGsUtil(cmd, return_stderr=True)
# Rewrite API may output an additional 'Copying' progress notification.
self.assertGreaterEqual(
stderr.count('Copying'), 2,
'stderr did not contain 2 "Copying" lines:\n%s' % stderr)
self.assertLessEqual(
stderr.count('Copying'), 4,
'stderr did not contain <= 4 "Copying" lines:\n%s' % stderr)
self.assertEqual(
stderr.count('Copying') % 2, 0,
'stderr did not contain even number of "Copying" lines:\n%s' %
stderr)
self.assertEqual(
stderr.count('Removing'), 2,
'stderr did not contain 2 "Removing" lines:\n%s' % stderr)
self.AssertNObjectsInBucket(bucket1_uri, 0)
self.AssertNObjectsInBucket(bucket2_uri, 2)
# Remove one of the objects.
objs = [
bucket2_uri.clone_replace_key(key).versionless_uri
for key in bucket2_uri.list_bucket()
]
obj1 = objs[0]
self.RunGsUtil(['rm', obj1])
self.AssertNObjectsInBucket(bucket1_uri, 0)
self.AssertNObjectsInBucket(bucket2_uri, 1)
# Move the 1 remaining object back.
objs = [
suri(bucket2_uri.clone_replace_key(key))
for key in bucket2_uri.list_bucket()
]
cmd = (['-m', 'mv'] + objs + [suri(bucket1_uri)])
stderr = self.RunGsUtil(cmd, return_stderr=True)
# Rewrite API may output an additional 'Copying' progress notification.
self.assertGreaterEqual(
stderr.count('Copying'), 1,
'stderr did not contain >= 1 "Copying" lines:\n%s' % stderr)
self.assertLessEqual(
stderr.count('Copying'), 2,
'stderr did not contain <= 2 "Copying" lines:\n%s' % stderr)
self.assertEqual(stderr.count('Removing'), 1)
self.AssertNObjectsInBucket(bucket1_uri, 1)
self.AssertNObjectsInBucket(bucket2_uri, 0)
def _ListObjectChangeNotifications():
stderr = self.RunGsUtil(
['notification', 'list', '-o',
suri(bucket_uri)],
return_stderr=True)
return stderr
def test_slasher_horror_film(self):
"""Tests removing a bucket with objects that are filled with slashes."""
bucket_uri = self.CreateVersionedBucket()
ouri1 = self.CreateObject(bucket_uri=bucket_uri,
object_name='h/e/l//lo',
contents='Halloween')
ouri2 = self.CreateObject(bucket_uri=bucket_uri,
object_name='/h/e/l/l/o',
contents='A Nightmare on Elm Street')
ouri3 = self.CreateObject(bucket_uri=bucket_uri,
object_name='//h//e/l//l/o',
contents='Friday the 13th')
ouri4 = self.CreateObject(bucket_uri=bucket_uri,
object_name='//h//e//l//l//o',
contents='I Know What You Did Last Summer')
ouri5 = self.CreateObject(bucket_uri=bucket_uri,
object_name='/',
contents='Scream')
ouri6 = self.CreateObject(bucket_uri=bucket_uri,
object_name='//',
contents='Child\'s Play')
ouri7 = self.CreateObject(bucket_uri=bucket_uri,
object_name='///',
contents='The Prowler')
ouri8 = self.CreateObject(bucket_uri=bucket_uri,
object_name='////',
contents='Black Christmas')
ouri9 = self.CreateObject(
bucket_uri=bucket_uri,
object_name='everything/is/better/with/slashes///////',
contents='Maniac')
self.AssertNObjectsInBucket(bucket_uri, 9, versioned=True)
# We add a slash to URIs with a trailing slash,
# because ObjectToURI (suri) removes one trailing slash.
objects_to_remove = [
'%s#%s' % (suri(ouri1), urigen(ouri1)),
'%s#%s' % (suri(ouri2), urigen(ouri2)),
'%s#%s' % (suri(ouri3), urigen(ouri3)),
'%s#%s' % (suri(ouri4), urigen(ouri4)),
'%s#%s' % (suri(ouri5) + '/', urigen(ouri5)),
'%s#%s' % (suri(ouri6) + '/', urigen(ouri6)),
'%s#%s' % (suri(ouri7) + '/', urigen(ouri7)),
'%s#%s' % (suri(ouri8) + '/', urigen(ouri8)),
'%s#%s' % (suri(ouri9) + '/', urigen(ouri9))
]
self._RunRemoveCommandAndCheck(
['-m', 'rm', '-r', suri(bucket_uri)],
objects_to_remove=objects_to_remove,
buckets_to_remove=[suri(bucket_uri)])
def test_input_output(self):
outpath = self.CreateTempFile()
bucket_uri = self.CreateBucket()
self.RunGsUtil(['perfdiag', '-o', outpath, '-n', '1', '-t', 'lat',
suri(bucket_uri)])
self.RunGsUtil(['perfdiag', '-i', outpath])
def _Check():
stdout = self.RunGsUtil(['du', suri(bucket_uri)],
return_stdout=True)
self.assertEqual(stdout, '%-11s %s\n' % (3, suri(obj_uri)))
def test_bad_lifecycle3(self):
bucket_uri = self.CreateBucket()
fpath = self.CreateTempFile(contents=self.bad_doc3)
self.RunGsUtil(['lifecycle', 'set', fpath,
suri(bucket_uri)],
expected_status=1)
def _RemoveAndCheck():
self.RunGsUtil(
['rm', '-r', '%s' % suri(bucket_uri, 'abc')],
expected_status=None)
self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
def test_list_acl(self):
"""Tests that long listing includes an ACL."""
key_uri = self.CreateObject(contents='foo')
stdout = self.RunGsUtil(['ls', '-L', suri(key_uri)], return_stdout=True)
self.assertIn('ACL:', stdout)
self.assertNotIn('ACCESS DENIED', stdout)
def _Check():
stdout = self.RunGsUtil(['ls', '-l', suri(bucket_uri, '**')],
return_stdout=True)
self.assertNumLines(stdout, 3) # 2 object lines, one summary line.
def _Check1():
stdout = self.RunGsUtil(['stat', stat_string], return_stdout=True)
self.assertIn(suri(object1_uri), stdout)
self.assertIn(suri(object2_uri), stdout)
def test_move_dir_to_bucket(self): """Tests moving a local directory to a bucket.""" bucket_uri = self.CreateBucket() dir_to_move = self.CreateTempDir(test_files=2) self.RunGsUtil(['mv', dir_to_move, suri(bucket_uri)]) self.AssertNObjectsInBucket(bucket_uri, 2)
def test_s3_fails_for_get(self):
bucket_uri = self.CreateBucket()
stderr = self.RunGsUtil(['rpo', 'get', suri(bucket_uri)],
return_stderr=True,
expected_status=1)
self.assertIn('command can only be used for GCS buckets', stderr)
def _Check5():
stdout = self.RunGsUtil(['ls', '-alh', suri(bucket_uri)],
return_stdout=True)
self.assertIn('2 KiB', stdout)
def _Check2():
stdout = self.RunGsUtil(['ls', '-L', suri(bucket_uri)],
return_stdout=True)
self.assertIn('2048', stdout)
def test_default_lifecycle(self):
bucket_uri = self.CreateBucket()
stdout = self.RunGsUtil(
['lifecycle', 'get', suri(bucket_uri)], return_stdout=True)
self.assertEqual(stdout, self.empty_doc1)
def _Check1():
stdout = self.RunGsUtil(['ls', '%s/dir' % suri(bucket_uri)],
return_stdout=True)
self.assertEqual('%s\n' % suri(k2_uri), stdout)
stdout = self.RunGsUtil(['ls', suri(k1_uri)], return_stdout=True)
self.assertEqual('%s\n' % suri(k1_uri), stdout)
def test_set_default(self):
bucket_uri = self.CreateBucket(location='nam4')
self.RunGsUtil(['rpo', 'set', 'ASYNC_TURBO', suri(bucket_uri)])
self.VerifyCommandGet(bucket_uri, 'rpo', 'ASYNC_TURBO')
self.RunGsUtil(['rpo', 'set', 'DEFAULT', suri(bucket_uri)])
self._verify_get_returns_default_or_none(bucket_uri)
