def testInitialAndSubsequentCh(self): bucket_uri = self.CreateBucket() ch_subargs = ['-l', '%s:%s' % (KEY1, VALUE1), '-l', '%s:%s' % (KEY2, VALUE2)] # Ensure 'ch' progress message shows in stderr. stderr = self.RunGsUtil( ['label', 'ch'] + ch_subargs + [suri(bucket_uri)], return_stderr=True) self.assertEqual( stderr.strip(), LABEL_SETTING_OUTPUT % suri(bucket_uri)) # Check the bucket to ensure it's configured with the labels we just # specified. stdout = self.RunGsUtil( ['label', 'get', suri(bucket_uri)], return_stdout=True) self.assertItemsEqual(json.loads(stdout), self._label_dict) # Ensure a subsequent 'ch' command works correctly. new_key = 'new-key' new_value = 'new-value' self.RunGsUtil([ 'label', 'ch', '-l', '%s:%s' % (new_key, new_value), '-d', KEY2, suri(bucket_uri)]) stdout = self.RunGsUtil( ['label', 'get', suri(bucket_uri)], return_stdout=True) actual = json.loads(stdout) expected = {KEY1: VALUE1, new_key: new_value} self.assertItemsEqual(actual, expected)
def testCh(self): bucket_uri = self.CreateBucket() self.RunGsUtil(['label', 'ch', '-l', '%s:%s' % (KEY1, VALUE1), '-l', '%s:%s' % (KEY2, VALUE2), suri(bucket_uri)]) # Verify that the bucket is configured with the labels we just set. # Work around eventual consistency for S3 tagging. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil( ['label', 'get', suri(bucket_uri)], return_stdout=True) self.assertItemsEqual( self._LabelDictFromXmlString(stdout), self._LabelDictFromXmlString(self._label_xml)) _Check1() # Remove KEY1, add a new key, and attempt to remove a nonexistent key # with 'label ch'. self.RunGsUtil(['label', 'ch', '-d', KEY1, '-l', 'new_key:new_value', '-d', 'nonexistent-key', suri(bucket_uri)]) expected_dict = {KEY2: VALUE2, 'new_key': 'new_value'} @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): stdout = self.RunGsUtil( ['label', 'get', suri(bucket_uri)], return_stdout=True) self.assertItemsEqual( self._LabelDictFromXmlString(stdout), expected_dict) _Check2()
def test_stat_encrypted_object(self): """Tests stat command with an encrypted object.""" if self.test_api == ApiSelector.XML: return unittest.skip("gsutil does not support encryption with the XML API") bucket_uri = self.CreateBucket() object_uri = self.CreateObject( bucket_uri=bucket_uri, object_name="foo", contents=TEST_ENCRYPTION_CONTENT1, encryption_key=TEST_ENCRYPTION_KEY1, ) # Stat object with key should return unencrypted hashes. with SetBotoConfigForTest([("GSUtil", "encryption_key", TEST_ENCRYPTION_KEY1)]): stdout = self.RunGsUtil(["stat", suri(object_uri)], return_stdout=True) self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout) self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout) self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout) # Stat object without key should return encrypted hashes. stdout = self.RunGsUtil(["stat", suri(object_uri)], return_stdout=True) self.assertNotIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout) self.assertNotIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout) self.assertIn("encrypted", stdout) self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout)
def test_set_empty_lifecycle1(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=self.empty_doc1.encode('ascii')) self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)]) stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket_uri)], return_stdout=True) self.assertIn(self.no_lifecycle_config, stdout)
def test_remove_all_versions_recursive_on_bucket(self): """Test that 'rm -ar' works on bucket.""" bucket_uri = self.CreateVersionedBucket() k1_uri = bucket_uri.clone_replace_name('foo') k2_uri = bucket_uri.clone_replace_name('foo2') k1_uri.set_contents_from_string('bar') k2_uri.set_contents_from_string('bar2') k1g1 = k1_uri.generation k2g1 = k2_uri.generation k1_uri.set_contents_from_string('baz') k2_uri.set_contents_from_string('baz2') k1g2 = k1_uri.generation k2g2 = k2_uri.generation stderr = self.RunGsUtil(['rm', '-ar', suri(bucket_uri)], return_stderr=True) self.assertEqual(stderr.count('Removing gs://'), 4) self.assertIn('Removing %s#%s...' % (suri(k1_uri), k1g1), stderr) self.assertIn('Removing %s#%s...' % (suri(k1_uri), k1g2), stderr) self.assertIn('Removing %s#%s...' % (suri(k2_uri), k2g1), stderr) self.assertIn('Removing %s#%s...' % (suri(k2_uri), k2g2), stderr) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, delay=1, backoff=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-a', suri(bucket_uri)], return_stdout=True) self.assertEqual(stdout, '') _Check1()
def test_wildcard_prefix(self): """Tests that an object name with a wildcard does not infinite loop.""" bucket_uri = self.CreateBucket() wildcard_folder_object = 'wildcard*/' object_matching_folder = 'wildcard10/foo' self.CreateObject(bucket_uri=bucket_uri, object_name=wildcard_folder_object, contents=b'foo') self.CreateObject(bucket_uri=bucket_uri, object_name=object_matching_folder, contents=b'foo') self.AssertNObjectsInBucket(bucket_uri, 2) stderr = self.RunGsUtil(['ls', suri(bucket_uri, 'wildcard*')], return_stderr=True, expected_status=1) self.assertIn( 'Cloud folder %s%s contains a wildcard' % (suri(bucket_uri), '/wildcard*/'), stderr) # Listing with a flat wildcard should still succeed. # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): stdout = self.RunGsUtil(['ls', '-l', suri(bucket_uri, '**')], return_stdout=True) self.assertNumLines(stdout, 3) # 2 object lines, one summary line. _Check()
def test_versioning(self): """Tests listing a versioned bucket.""" bucket1_uri = self.CreateBucket(test_objects=1) bucket2_uri = self.CreateVersionedBucket(test_objects=1) self.AssertNObjectsInBucket(bucket1_uri, 1, versioned=True) bucket_list = list(bucket1_uri.list_bucket()) objuri = [ bucket1_uri.clone_replace_key(key).versionless_uri for key in bucket_list ][0] self.RunGsUtil(['cp', objuri, suri(bucket2_uri)]) self.RunGsUtil(['cp', objuri, suri(bucket2_uri)]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): stdout = self.RunGsUtil(['ls', '-a', suri(bucket2_uri)], return_stdout=True) self.assertNumLines(stdout, 3) stdout = self.RunGsUtil(['ls', '-la', suri(bucket2_uri)], return_stdout=True) self.assertIn('%s#' % bucket2_uri.clone_replace_name(bucket_list[0].name), stdout) self.assertIn('metageneration=', stdout) _Check2()
def _Check1(): stdout = self.RunGsUtil(['ls', '-d', '%s/dir*' % suri(bucket_uri)], return_stdout=True) self.assertEqual('%s/dir/\n%s/dir2/\n' % (suri(bucket_uri), suri(bucket_uri)), stdout) stdout = self.RunGsUtil(['ls', suri(k1_uri)], return_stdout=True) self.assertEqual('%s\n' % suri(k1_uri), stdout)
def _Check(): stdout = self.RunGsUtil(['du', suri(obj_uri1), suri(obj_uri2)], return_stdout=True) self.assertSetEqual(set(stdout.splitlines()), set([ '%-10s %s' % (3, suri(obj_uri1)), '%-10s %s' % (4, suri(obj_uri2)), ]))
def test_valid_lifecycle(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=self.lifecycle_doc.encode('ascii')) self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)]) stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
def test_stdin_args(self): """Tests rm with the -I option.""" buri1 = self.CreateVersionedBucket() ouri1 = self.CreateObject(bucket_uri=buri1, object_name='foo', contents='foocontents') self.CreateObject(bucket_uri=buri1, object_name='bar', contents='barcontents') ouri3 = self.CreateObject(bucket_uri=buri1, object_name='baz', contents='bazcontents') buri2 = self.CreateVersionedBucket() ouri4 = self.CreateObject(bucket_uri=buri2, object_name='moo', contents='moocontents') self.AssertNObjectsInBucket(buri1, 3, versioned=True) self.AssertNObjectsInBucket(buri2, 1, versioned=True) objects_to_remove = ['%s#%s' % (suri(ouri1), urigen(ouri1)), '%s#%s' % (suri(ouri3), urigen(ouri3)), '%s#%s' % (suri(ouri4), urigen(ouri4))] stdin = '\n'.join(objects_to_remove) self._RunRemoveCommandAndCheck(['rm', '-I'], objects_to_remove=objects_to_remove, stdin=stdin) self.AssertNObjectsInBucket(buri1, 1, versioned=True) self.AssertNObjectsInBucket(buri2, 0, versioned=True)
def testMultithreadedAclChange(self, count=10): """Tests multi-threaded acl changing on several objects.""" objects = [] for i in range(count): objects.append(self.CreateObject( bucket_uri=self.sample_uri, contents='something {0}'.format(i))) self.AssertNObjectsInBucket(self.sample_uri, count) test_regex = self._MakeScopeRegex( 'READER', 'group', self.GROUP_TEST_ADDRESS) json_texts = [] for obj in objects: json_texts.append(self.RunGsUtil( self._get_acl_prefix + [suri(obj)], return_stdout=True)) for json_text in json_texts: self.assertNotRegexpMatches(json_text, test_regex) uris = [suri(obj) for obj in objects] self.RunGsUtil(['-m', '-DD'] + self._ch_acl_prefix + ['-g', self.GROUP_TEST_ADDRESS+':READ'] + uris) json_texts = [] for obj in objects: json_texts.append(self.RunGsUtil( self._get_acl_prefix + [suri(obj)], return_stdout=True)) for json_text in json_texts: self.assertRegexpMatches(json_text, test_regex)
def _AddAcl(): self.RunGsUtil( self._ch_acl_prefix + ['-R', '-g', self.GROUP_TEST_ADDRESS+':READ', suri(obj)[:-3]]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(obj)], return_stdout=True) self.assertRegexpMatches(json_text, test_regex)
def testProjectAclChangesOnBucket(self): """Tests project entity acl changes on a bucket.""" if self.test_api == ApiSelector.XML: stderr = self.RunGsUtil(self._ch_acl_prefix + ['-p', self._project_test_acl +':w', suri(self.sample_uri)], expected_status=1, return_stderr=True) self.assertIn(('CommandException: XML API does not support project' ' scopes, cannot translate ACL.'), stderr) else: test_regex = self._MakeProjectScopeRegex( 'WRITER', self._project_team, self._project_number) self.RunGsUtil(self._ch_acl_prefix + ['-p', self._project_test_acl +':w', suri(self.sample_uri)]) json_text = self.RunGsUtil( self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertRegexpMatches(json_text, test_regex) self.RunGsUtil(self._ch_acl_prefix + ['-d', self._project_test_acl, suri(self.sample_uri)]) json_text2 = self.RunGsUtil( self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegexpMatches(json_text2, test_regex)
def _Check1(): command = ['ls', '-a'] if versioned else ['ls'] b_uri = [suri(bucket_uri) + '/**'] if num_objects else [suri(bucket_uri)] listing = self.RunGsUtil(command + b_uri, return_stdout=True).split('\n') # num_objects + one trailing newline. self.assertEquals(len(listing), num_objects + 1) return listing
def test_set_valid_def_acl_bucket(self): """Ensures that valid default canned and XML ACLs works with get/set.""" bucket_uri = self.CreateBucket() # Default ACL is project private. obj_uri1 = suri(self.CreateObject(bucket_uri=bucket_uri, contents="foo")) acl_string = self.RunGsUtil(self._get_acl_prefix + [obj_uri1], return_stdout=True) # Change it to authenticated-read. self.RunGsUtil(self._set_defacl_prefix + ["authenticated-read", suri(bucket_uri)]) # Default object ACL may take some time to propagate. @Retry(AssertionError, tries=5, timeout_secs=1) def _Check1(): obj_uri2 = suri(self.CreateObject(bucket_uri=bucket_uri, contents="foo2")) acl_string2 = self.RunGsUtil(self._get_acl_prefix + [obj_uri2], return_stdout=True) self.assertNotEqual(acl_string, acl_string2) self.assertIn("allAuthenticatedUsers", acl_string2) _Check1() # Now change it back to the default via XML. inpath = self.CreateTempFile(contents=acl_string) self.RunGsUtil(self._set_defacl_prefix + [inpath, suri(bucket_uri)]) # Default object ACL may take some time to propagate. @Retry(AssertionError, tries=5, timeout_secs=1) def _Check2(): obj_uri3 = suri(self.CreateObject(bucket_uri=bucket_uri, contents="foo3")) acl_string3 = self.RunGsUtil(self._get_acl_prefix + [obj_uri3], return_stdout=True) self.assertEqual(acl_string, acl_string3) _Check2()
def test_list_one_entry(self): """Tests notification config listing with one entry.""" if self.test_api == ApiSelector.XML: return unittest.skip('Notifications only work with the JSON API.') bucket_uri = self.CreateBucket() bucket_name = bucket_uri.bucket_name topic_name = self._RegisterDefaultTopicCreation(bucket_uri.bucket_name) self.RunGsUtil([ 'notification', 'create', '-f', 'json', '-e', 'OBJECT_FINALIZE', '-e', 'OBJECT_DELETE', '-m', 'someKey:someValue', '-p', 'somePrefix', suri(bucket_uri) ], return_stderr=True) stdout = self.RunGsUtil( ['notification', 'list', suri(bucket_uri)], return_stdout=True) self.assertEquals( stdout, ('projects/_/buckets/{bucket_name}/notificationConfigs/1\n' '\tCloud Pub/Sub topic: {topic_name}\n' '\tCustom attributes:\n' '\t\tsomeKey: someValue\n' '\tFilters:\n' '\t\tEvent Types: OBJECT_FINALIZE, OBJECT_DELETE\n' '\t\tObject name prefix: \'somePrefix\'\n'.format( bucket_name=bucket_name, topic_name=topic_name)))
def test_rb(self): rp_bucket_uri = self.CreateBucket() self._set_requester_pays(rp_bucket_uri) self._run_requester_pays_test(['rb', suri(rp_bucket_uri)]) non_rp_bucket_uri = self.CreateBucket() self._run_non_requester_pays_test(['rb', suri(non_rp_bucket_uri)])
def test_full(self): bucket_uri = self.CreateBucket() self.RunGsUtil( self._set_web_cmd + ['-m', 'main', '-e', '404', suri(bucket_uri)]) stdout = self.RunGsUtil( self._get_web_cmd + [suri(bucket_uri)], return_stdout=True) self.assertEquals(json.loads(stdout), WEBCFG_FULL)
def test_setmeta_valid_with_multiple_colons_in_value(self): obj_uri = self.CreateObject(contents='foo') self.RunGsUtil([ 'setmeta', '-h', 'x-%s-meta-foo:bar:baz' % self.provider_custom_meta, suri(obj_uri)]) stdout = self.RunGsUtil(['stat', suri(obj_uri)], return_stdout=True) self.assertRegexpMatches(stdout, r'foo:\s+bar:baz')
def test_stat_bucket_wildcard(self): bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents='z') stat_string = suri(bucket_uri)[:-1] + '?/foo' self.RunGsUtil(['stat', stat_string]) stat_string2 = suri(bucket_uri)[:-1] + '*/foo' self.RunGsUtil(['stat', stat_string2])
def setUp(self): """Creates 2 mock buckets, each containing 4 objects, including 1 nested.""" super(CloudWildcardIteratorTests, self).setUp() self.immed_child_obj_names = ['abcd', 'abdd', 'ade$'] self.all_obj_names = ['abcd', 'abdd', 'ade$', 'nested1/nested2/xyz1', 'nested1/nested2/xyz2', 'nested1/nfile_abc'] self.base_bucket_uri = self.CreateBucket() self.prefix_bucket_name = '%s_' % self.base_bucket_uri.bucket_name[:61] self.base_uri_str = suri(self.base_bucket_uri) self.base_uri_str = self.base_uri_str.replace( self.base_bucket_uri.bucket_name, self.prefix_bucket_name) self.test_bucket0_uri = self.CreateBucket( bucket_name='%s0' % self.prefix_bucket_name) self.test_bucket0_obj_uri_strs = set() for obj_name in self.all_obj_names: obj_uri = self.CreateObject(bucket_uri=self.test_bucket0_uri, object_name=obj_name, contents='') self.test_bucket0_obj_uri_strs.add(suri(obj_uri)) self.test_bucket1_uri = self.CreateBucket( bucket_name='%s1' % self.prefix_bucket_name) self.test_bucket1_obj_uri_strs = set() for obj_name in self.all_obj_names: obj_uri = self.CreateObject(bucket_uri=self.test_bucket1_uri, object_name=obj_name, contents='') self.test_bucket1_obj_uri_strs.add(suri(obj_uri))
def test_bucket_list_wildcard(self): """Tests listing multiple buckets with a wildcard.""" random_prefix = self.MakeRandomTestString() bucket1_name = self.MakeTempName("bucket", prefix=random_prefix) bucket2_name = self.MakeTempName("bucket", prefix=random_prefix) bucket1_uri = self.CreateBucket(bucket_name=bucket1_name) bucket2_uri = self.CreateBucket(bucket_name=bucket2_name) # This just double checks that the common prefix of the two buckets is what # we think it should be (based on implementation detail of CreateBucket). # We want to be careful when setting a wildcard on buckets to make sure we # don't step outside the test buckets to affect other buckets. common_prefix = posixpath.commonprefix([suri(bucket1_uri), suri(bucket2_uri)]) self.assertTrue( common_prefix.startswith( "%s://%sgsutil-test-test_bucket_list_wildcard-bucket-" % (self.default_provider, random_prefix) ) ) wildcard = "%s*" % common_prefix # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(["ls", "-b", wildcard], return_stdout=True) expected = set([suri(bucket1_uri) + "/", suri(bucket2_uri) + "/"]) actual = set(stdout.split()) self.assertEqual(expected, actual) _Check1()
def test_bucket_to_bucket_minus_d_empty_src(self): """Tests working with empty src bucket (iter runs out before dst iter).""" bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket2_uri, object_name='obj1', contents='obj1') self.CreateObject(bucket_uri=bucket2_uri, object_name='obj2', contents='obj2') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): self.RunGsUtil(['rsync', '-d', suri(bucket1_uri), suri(bucket2_uri)]) stderr = self.RunGsUtil(['ls', suri(bucket1_uri, '**')], expected_status=1, return_stderr=True) self.assertIn('One or more URLs matched no objects', stderr) stderr = self.RunGsUtil(['ls', suri(bucket2_uri, '**')], expected_status=1, return_stderr=True) self.assertIn('One or more URLs matched no objects', stderr) _Check() # Check that re-running the same rsync command causes no more changes. self.assertEquals(NO_CHANGES, self.RunGsUtil( ['rsync', '-d', suri(bucket1_uri), suri(bucket2_uri)], return_stderr=True))
def test_recursive_bucket_rm(self): """Test for 'rm -r' of a bucket.""" bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri) self.RunGsUtil(['rm', '-r', suri(bucket_uri)]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): # Bucket should be deleted. stderr = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stderr=True, expected_status=1) self.assertIn('bucket does not exist', stderr) _Check1() # Now try same thing, but for a versioned bucket with multiple versions of # an object present. bucket_uri = self.CreateVersionedBucket() self.CreateObject(bucket_uri, 'obj', 'z') self.CreateObject(bucket_uri, 'obj', 'z') self.CreateObject(bucket_uri, 'obj', 'z') self.AssertNObjectsInBucket(bucket_uri, 3, versioned=True) self.RunGsUtil(['rm', suri(bucket_uri, '**')]) stderr = self.RunGsUtil(['rb', suri(bucket_uri)], return_stderr=True, expected_status=1) self.assertIn('Bucket is not empty', stderr) # Now try with rm -r. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): self.RunGsUtil(['rm', '-r', suri(bucket_uri)]) # Bucket should be deleted. stderr = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stderr=True, expected_status=1) self.assertIn('bucket does not exist', stderr) _Check2()
def test_one_object_with_L(self): """Tests listing one object with -L.""" obj_uri = self.CreateObject(contents='foo') # Ensure that creation and update don't take place in the same second. time.sleep(2) # Check that the creation time, rather than the updated time, is displayed. self.RunGsUtil(['setmeta', '-h', 'x-goog-meta-foo:bar', suri(obj_uri)]) find_time_created_re = re.compile( r'^\s*Creation time:\s+(?P<time_created_val>.+)$', re.MULTILINE) find_time_updated_re = re.compile( r'^\s*Update time:\s+(?P<time_updated_val>.+)$', re.MULTILINE) stdout = self.RunGsUtil(['ls', '-L', suri(obj_uri)], return_stdout=True) time_created_match = re.search(find_time_created_re, stdout) time_updated_match = re.search(find_time_updated_re, stdout) time_created = time_created_match.group('time_created_val') self.assertIsNotNone(time_created) time_created = time.strptime(time_created, '%a, %d %b %Y %H:%M:%S %Z') if self.test_api == ApiSelector.XML: # XML API has no concept of updated time. self.assertIsNone(time_updated_match) elif self.test_api == ApiSelector.JSON: time_updated = time_updated_match.group('time_updated_val') self.assertIsNotNone(time_updated) time_updated = time.strptime(time_updated, '%a, %d %b %Y %H:%M:%S %Z') self.assertGreater(time_updated, time_created)
def testNoOpDirectoryIterator(self): """Tests that directory-only URI iterates just that one URI.""" results = list( self._test_wildcard_iterator(suri(tempfile.tempdir)).IterAll( expand_top_level_buckets=True)) self.assertEqual(1, len(results)) self.assertEqual(suri(tempfile.tempdir), str(results[0]))
def test_bucket_to_bucket_minus_d_with_composites(self): """Tests that rsync works with composite objects (which don't have MD5s).""" bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket1_uri, object_name='obj1', contents='obj1') self.CreateObject(bucket_uri=bucket1_uri, object_name='obj2', contents='obj2') self.RunGsUtil( ['compose', suri(bucket1_uri, 'obj1'), suri(bucket1_uri, 'obj2'), suri(bucket1_uri, 'obj3')]) self.CreateObject(bucket_uri=bucket2_uri, object_name='obj2', contents='OBJ2') self.CreateObject(bucket_uri=bucket2_uri, object_name='obj4', contents='obj4') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): self.RunGsUtil(['rsync', '-d', suri(bucket1_uri), suri(bucket2_uri)]) listing1 = _TailSet(suri(bucket1_uri), self._FlatListBucket(bucket1_uri)) listing2 = _TailSet(suri(bucket2_uri), self._FlatListBucket(bucket2_uri)) # First bucket should have un-altered content. self.assertEquals(listing1, set(['/obj1', '/obj2', '/obj3'])) # Second bucket should have content like first bucket but without the # subdir objects synchronized. self.assertEquals(listing2, set(['/obj1', '/obj2', '/obj3'])) _Check() # Check that re-running the same rsync command causes no more changes. self.assertEquals(NO_CHANGES, self.RunGsUtil( ['rsync', '-d', suri(bucket1_uri), suri(bucket2_uri)], return_stderr=True))
def test_stat_bucket_wildcard(self): bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, object_name="foo", contents="z") stat_string = suri(bucket_uri)[:-1] + "?/foo" self.RunGsUtil(["stat", stat_string]) stat_string2 = suri(bucket_uri)[:-1] + "*/foo" self.RunGsUtil(["stat", stat_string2])
def test_rsync_minus_d_minus_p(self): """Tests that rsync -p preserves ACLs.""" bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket1_uri, object_name='obj1', contents='obj1') # Set public-read (non-default) ACL so we can verify that rsync -p works. self.RunGsUtil(['acl', 'set', 'public-read', suri(bucket1_uri, 'obj1')]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): """Tests rsync -p works as expected.""" self.RunGsUtil(['rsync', '-d', '-p', suri(bucket1_uri), suri(bucket2_uri)]) listing1 = _TailSet(suri(bucket1_uri), self._FlatListBucket(bucket1_uri)) listing2 = _TailSet(suri(bucket2_uri), self._FlatListBucket(bucket2_uri)) self.assertEquals(listing1, set(['/obj1'])) self.assertEquals(listing2, set(['/obj1'])) acl1_json = self.RunGsUtil(['acl', 'get', suri(bucket1_uri, 'obj1')], return_stdout=True) acl2_json = self.RunGsUtil(['acl', 'get', suri(bucket2_uri, 'obj1')], return_stdout=True) self.assertEquals(acl1_json, acl2_json) _Check() # Check that re-running the same rsync command causes no more changes. self.assertEquals(NO_CHANGES, self.RunGsUtil( ['rsync', '-d', '-p', suri(bucket1_uri), suri(bucket2_uri)], return_stderr=True))
def FlatListBucket(self, bucket_url_string): """Perform a flat listing over bucket_url_string.""" return self.RunGsUtil(['ls', suri(bucket_url_string, '**')], return_stdout=True)
def test_rb_bucket_not_empty(self): bucket_uri = self.CreateBucket(test_objects=1) stderr = self.RunGsUtil(['rb', suri(bucket_uri)], expected_status=1, return_stderr=True) self.assertIn('BucketNotEmpty', stderr)
def _ListExpectDecrypted(): stdout = self.RunGsUtil(['ls', '-L', suri(object_uri)], return_stdout=True) self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout) self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout) self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64, stdout)
def _Check(): # Bucket should no longer exist. stderr = self.RunGsUtil(['ls', '-a', suri(bucket_uri)], return_stderr=True, expected_status=1) self.assertIn('bucket does not exist', stderr)
def _Check1(): stdout = self.RunGsUtil(['ls', '-lb', suri(bucket_uri)], return_stdout=True) self.assertIn(suri(bucket_uri), stdout) self.assertNotIn('TOTAL:', stdout)
def _Check2(): obj_uri3 = suri( self.CreateObject(bucket_uri=bucket_uri, contents='foo3')) acl_string3 = self.RunGsUtil(self._get_acl_prefix + [obj_uri3], return_stdout=True) self.assertEqual(acl_string, acl_string3)
def _Check1(): stdout = self.RunGsUtil(['ls', suri(bucket_uri)], return_stdout=True) self.assertEqual('%s\n' % obj_uri, stdout)
def test_list_missing_object(self): """Tests listing a non-existent object.""" bucket_uri = self.CreateBucket() stderr = self.RunGsUtil(['ls', suri(bucket_uri, 'missing')], return_stderr=True, expected_status=1) self.assertIn('matched no objects', stderr)
def _Check1(): stdout = self.RunGsUtil(['ls', '-b', wildcard], return_stdout=True) expected = set([suri(bucket1_uri) + '/', suri(bucket2_uri) + '/']) actual = set(stdout.split()) self.assertEqual(expected, actual)
def _Check1(): stdout = self.RunGsUtil(['ls', suri(bucket_uri)], return_stdout=True) self.assertIn(os.path.basename(fpath1), stdout) self.assertIn(os.path.basename(fpath2), stdout) self.assertNumLines(stdout, 2)
def test_cp_key_to_local_stream(self): bucket_uri = self.CreateBucket() contents = 'foo' key_uri = self.CreateObject(bucket_uri=bucket_uri, contents=contents) stdout = self.RunGsUtil(['cp', suri(key_uri), '-'], return_stdout=True) self.assertIn(contents, stdout)
def test_update(self): """Tests that the update command works or raises proper exceptions.""" if system_util.InvokedViaCloudSdk(): stderr = self.RunGsUtil(['update'], stdin='n', return_stderr=True, expected_status=1) self.assertIn('update command is disabled for Cloud SDK', stderr) return if gslib.IS_PACKAGE_INSTALL: # The update command is not present when installed via package manager. stderr = self.RunGsUtil(['update'], return_stderr=True, expected_status=1) self.assertIn('Invalid command', stderr) return # Create two temp directories, one of which we will run 'gsutil update' in # to pull the changes from the other. tmpdir_src = self.CreateTempDir() tmpdir_dst = self.CreateTempDir() # Copy gsutil to both source and destination directories. gsutil_src = os.path.join(tmpdir_src, 'gsutil') gsutil_dst = os.path.join(tmpdir_dst, 'gsutil') # Path when executing from tmpdir (Windows doesn't support in-place rename) gsutil_relative_dst = os.path.join('gsutil', 'gsutil') ignore_callable = shutil.ignore_patterns( '.git*', '*.pyc', '*.pyo', '__pycache__', ) shutil.copytree(GSUTIL_DIR, gsutil_src, ignore=ignore_callable) # Copy specific files rather than all of GSUTIL_DIR so we don't pick up temp # working files left in top-level directory by gsutil developers (like tags, # .git*, .pyc files, etc.) os.makedirs(gsutil_dst) for comp in os.listdir(GSUTIL_DIR): if ('.git' not in comp and '__pycache__' not in comp and not comp.endswith('.pyc') and not comp.endswith('.pyo')): # yapf: disable cp_src_path = os.path.join(GSUTIL_DIR, comp) cp_dst_path = os.path.join(gsutil_dst, comp) if os.path.isdir(cp_src_path): shutil.copytree(cp_src_path, cp_dst_path, ignore=ignore_callable) else: shutil.copyfile(cp_src_path, cp_dst_path) # Create a fake version number in the source so we can verify it in the # destination. expected_version = '17.25' src_version_file = os.path.join(gsutil_src, 'VERSION') self.assertTrue(os.path.exists(src_version_file)) with open(src_version_file, 'w') as f: f.write(expected_version) # Create a tarball out of the source directory and copy it to a bucket. src_tarball = os.path.join(tmpdir_src, 'gsutil.test.tar.gz') normpath = os.path.normpath try: # We monkey patch os.path.normpath here because the tarfile module # normalizes the ./gsutil path, but the update command expects the tar # file to be prefixed with . This preserves the ./gsutil path. os.path.normpath = lambda fname: fname tar = tarfile.open(src_tarball, 'w:gz') tar.add(gsutil_src, arcname='./gsutil') tar.close() finally: os.path.normpath = normpath prefix = [sys.executable] if sys.executable else [] # Run with an invalid gs:// URI. p = subprocess.Popen(prefix + ['gsutil', 'update', 'gs://pub'], cwd=gsutil_dst, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (_, stderr) = p.communicate() p.stdout.close() p.stderr.close() self.assertEqual(p.returncode, 1) self.assertIn(b'update command only works with tar.gz', stderr) # Run with non-existent gs:// URI. p = subprocess.Popen(prefix + ['gsutil', 'update', 'gs://pub/Jdjh38)(;.tar.gz'], cwd=gsutil_dst, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (_, stderr) = p.communicate() p.stdout.close() p.stderr.close() self.assertEqual(p.returncode, 1) self.assertIn(b'NotFoundException', stderr) # Run with file:// URI wihout -f option. p = subprocess.Popen( prefix + ['gsutil', 'update', suri(src_tarball)], cwd=gsutil_dst, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (_, stderr) = p.communicate() p.stdout.close() p.stderr.close() self.assertEqual(p.returncode, 1) self.assertIn(b'command does not support', stderr) # Run with a file present that was not distributed with gsutil. with open(os.path.join(gsutil_dst, 'userdata.txt'), 'w') as fp: fp.write('important data\n') p = subprocess.Popen( prefix + ['gsutil', 'update', '-f', suri(src_tarball)], cwd=gsutil_dst, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) (_, stderr) = p.communicate() p.stdout.close() p.stderr.close() # Clean up before next test, and before assertions so failure doesn't leave # this file around. os.unlink(os.path.join(gsutil_dst, 'userdata.txt')) self.assertEqual(p.returncode, 1) # Additional check for Windows since it has \r\n and string may have just \n os_ls = os.linesep.encode(UTF8) if os_ls in stderr: stderr = stderr.replace(os_ls, b' ') elif b'\n' in stderr: stderr = stderr.replace(b'\n', b' ') self.assertIn( b'The update command cannot run with user data in the gsutil directory', stderr) # Determine whether we'll need to decline the analytics prompt. analytics_prompt = not (os.path.exists(_UUID_FILE_PATH) or boto.config.get_value('GSUtil', 'disable_analytics_prompt')) update_input = b'n\r\ny\r\n' if analytics_prompt else b'y\r\n' # Now do the real update, which should succeed. p = subprocess.Popen( prefix + [gsutil_relative_dst, 'update', '-f', suri(src_tarball)], cwd=tmpdir_dst, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) (_, stderr) = p.communicate(input=update_input) p.stdout.close() p.stderr.close() self.assertEqual( p.returncode, 0, msg=('Non-zero return code (%d) from gsutil update. stderr = \n%s' % (p.returncode, stderr.decode(UTF8)))) # Verify that version file was updated. dst_version_file = os.path.join(tmpdir_dst, 'gsutil', 'VERSION') with open(dst_version_file, 'r') as f: self.assertEqual(f.read(), expected_version) # If the analytics prompt was given, that means we disabled analytics. We # should reset to the default by deleting the UUID file. if analytics_prompt: os.unlink(_UUID_FILE_PATH)
def testAclBucketGetSet(self): bucket_uri = self.CreateBucket() stdout = self.RunGsUtil(self._get_acl_prefix + [suri(bucket_uri)], return_stdout=True) set_contents = self.CreateTempFile(contents=stdout) self.RunGsUtil(self._set_acl_prefix + [set_contents, suri(bucket_uri)])
def test_moving(self): """Tests moving two buckets, one with 2 objects and one with 0 objects.""" bucket1_uri = self.CreateBucket(test_objects=2) self.AssertNObjectsInBucket(bucket1_uri, 2) bucket2_uri = self.CreateBucket() self.AssertNObjectsInBucket(bucket2_uri, 0) # Move two objects from bucket1 to bucket2. objs = [ bucket1_uri.clone_replace_key(key).versionless_uri for key in bucket1_uri.list_bucket() ] cmd = (['-m', 'mv'] + objs + [suri(bucket2_uri)]) stderr = self.RunGsUtil(cmd, return_stderr=True) # Rewrite API may output an additional 'Copying' progress notification. self.assertGreaterEqual( stderr.count('Copying'), 2, 'stderr did not contain 2 "Copying" lines:\n%s' % stderr) self.assertLessEqual( stderr.count('Copying'), 4, 'stderr did not contain <= 4 "Copying" lines:\n%s' % stderr) self.assertEqual( stderr.count('Copying') % 2, 0, 'stderr did not contain even number of "Copying" lines:\n%s' % stderr) self.assertEqual( stderr.count('Removing'), 2, 'stderr did not contain 2 "Removing" lines:\n%s' % stderr) self.AssertNObjectsInBucket(bucket1_uri, 0) self.AssertNObjectsInBucket(bucket2_uri, 2) # Remove one of the objects. objs = [ bucket2_uri.clone_replace_key(key).versionless_uri for key in bucket2_uri.list_bucket() ] obj1 = objs[0] self.RunGsUtil(['rm', obj1]) self.AssertNObjectsInBucket(bucket1_uri, 0) self.AssertNObjectsInBucket(bucket2_uri, 1) # Move the 1 remaining object back. objs = [ suri(bucket2_uri.clone_replace_key(key)) for key in bucket2_uri.list_bucket() ] cmd = (['-m', 'mv'] + objs + [suri(bucket1_uri)]) stderr = self.RunGsUtil(cmd, return_stderr=True) # Rewrite API may output an additional 'Copying' progress notification. self.assertGreaterEqual( stderr.count('Copying'), 1, 'stderr did not contain >= 1 "Copying" lines:\n%s' % stderr) self.assertLessEqual( stderr.count('Copying'), 2, 'stderr did not contain <= 2 "Copying" lines:\n%s' % stderr) self.assertEqual(stderr.count('Removing'), 1) self.AssertNObjectsInBucket(bucket1_uri, 1) self.AssertNObjectsInBucket(bucket2_uri, 0)
def _ListObjectChangeNotifications(): stderr = self.RunGsUtil( ['notification', 'list', '-o', suri(bucket_uri)], return_stderr=True) return stderr
def test_slasher_horror_film(self): """Tests removing a bucket with objects that are filled with slashes.""" bucket_uri = self.CreateVersionedBucket() ouri1 = self.CreateObject(bucket_uri=bucket_uri, object_name='h/e/l//lo', contents='Halloween') ouri2 = self.CreateObject(bucket_uri=bucket_uri, object_name='/h/e/l/l/o', contents='A Nightmare on Elm Street') ouri3 = self.CreateObject(bucket_uri=bucket_uri, object_name='//h//e/l//l/o', contents='Friday the 13th') ouri4 = self.CreateObject(bucket_uri=bucket_uri, object_name='//h//e//l//l//o', contents='I Know What You Did Last Summer') ouri5 = self.CreateObject(bucket_uri=bucket_uri, object_name='/', contents='Scream') ouri6 = self.CreateObject(bucket_uri=bucket_uri, object_name='//', contents='Child\'s Play') ouri7 = self.CreateObject(bucket_uri=bucket_uri, object_name='///', contents='The Prowler') ouri8 = self.CreateObject(bucket_uri=bucket_uri, object_name='////', contents='Black Christmas') ouri9 = self.CreateObject( bucket_uri=bucket_uri, object_name='everything/is/better/with/slashes///////', contents='Maniac') self.AssertNObjectsInBucket(bucket_uri, 9, versioned=True) # We add a slash to URIs with a trailing slash, # because ObjectToURI (suri) removes one trailing slash. objects_to_remove = [ '%s#%s' % (suri(ouri1), urigen(ouri1)), '%s#%s' % (suri(ouri2), urigen(ouri2)), '%s#%s' % (suri(ouri3), urigen(ouri3)), '%s#%s' % (suri(ouri4), urigen(ouri4)), '%s#%s' % (suri(ouri5) + '/', urigen(ouri5)), '%s#%s' % (suri(ouri6) + '/', urigen(ouri6)), '%s#%s' % (suri(ouri7) + '/', urigen(ouri7)), '%s#%s' % (suri(ouri8) + '/', urigen(ouri8)), '%s#%s' % (suri(ouri9) + '/', urigen(ouri9)) ] self._RunRemoveCommandAndCheck( ['-m', 'rm', '-r', suri(bucket_uri)], objects_to_remove=objects_to_remove, buckets_to_remove=[suri(bucket_uri)])
def test_input_output(self): outpath = self.CreateTempFile() bucket_uri = self.CreateBucket() self.RunGsUtil(['perfdiag', '-o', outpath, '-n', '1', '-t', 'lat', suri(bucket_uri)]) self.RunGsUtil(['perfdiag', '-i', outpath])
def _Check(): stdout = self.RunGsUtil(['du', suri(bucket_uri)], return_stdout=True) self.assertEqual(stdout, '%-11s %s\n' % (3, suri(obj_uri)))
def test_bad_lifecycle3(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=self.bad_doc3) self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)], expected_status=1)
def _RemoveAndCheck(): self.RunGsUtil( ['rm', '-r', '%s' % suri(bucket_uri, 'abc')], expected_status=None) self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True)
def test_list_acl(self): """Tests that long listing includes an ACL.""" key_uri = self.CreateObject(contents='foo') stdout = self.RunGsUtil(['ls', '-L', suri(key_uri)], return_stdout=True) self.assertIn('ACL:', stdout) self.assertNotIn('ACCESS DENIED', stdout)
def _Check(): stdout = self.RunGsUtil(['ls', '-l', suri(bucket_uri, '**')], return_stdout=True) self.assertNumLines(stdout, 3) # 2 object lines, one summary line.
def _Check1(): stdout = self.RunGsUtil(['stat', stat_string], return_stdout=True) self.assertIn(suri(object1_uri), stdout) self.assertIn(suri(object2_uri), stdout)
def test_move_dir_to_bucket(self): """Tests moving a local directory to a bucket.""" bucket_uri = self.CreateBucket() dir_to_move = self.CreateTempDir(test_files=2) self.RunGsUtil(['mv', dir_to_move, suri(bucket_uri)]) self.AssertNObjectsInBucket(bucket_uri, 2)
def test_s3_fails_for_get(self): bucket_uri = self.CreateBucket() stderr = self.RunGsUtil(['rpo', 'get', suri(bucket_uri)], return_stderr=True, expected_status=1) self.assertIn('command can only be used for GCS buckets', stderr)
def _Check5(): stdout = self.RunGsUtil(['ls', '-alh', suri(bucket_uri)], return_stdout=True) self.assertIn('2 KiB', stdout)
def _Check2(): stdout = self.RunGsUtil(['ls', '-L', suri(bucket_uri)], return_stdout=True) self.assertIn('2048', stdout)
def test_default_lifecycle(self): bucket_uri = self.CreateBucket() stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket_uri)], return_stdout=True) self.assertEqual(stdout, self.empty_doc1)
def _Check1(): stdout = self.RunGsUtil(['ls', '%s/dir' % suri(bucket_uri)], return_stdout=True) self.assertEqual('%s\n' % suri(k2_uri), stdout) stdout = self.RunGsUtil(['ls', suri(k1_uri)], return_stdout=True) self.assertEqual('%s\n' % suri(k1_uri), stdout)
def test_set_default(self): bucket_uri = self.CreateBucket(location='nam4') self.RunGsUtil(['rpo', 'set', 'ASYNC_TURBO', suri(bucket_uri)]) self.VerifyCommandGet(bucket_uri, 'rpo', 'ASYNC_TURBO') self.RunGsUtil(['rpo', 'set', 'DEFAULT', suri(bucket_uri)]) self._verify_get_returns_default_or_none(bucket_uri)