def list_objects(self, s3_path, dir_op): """ This function yields the appropriate object or objects under a common prefix depending if the operation is on objects under a common prefix. It yields the file's source path, size, and last update. """ # Short circuit path: if we are not recursing into the s3 # bucket and a specific path was given, we can just yield # that path and not have to call any operation in s3. bucket, prefix = find_bucket_key(s3_path) if not dir_op and prefix: yield self._list_single_object(s3_path) else: lister = BucketLister(self._client) for key in lister.list_objects(bucket=bucket, prefix=prefix, page_size=self.page_size): source_path, response_data = key if response_data['Size'] == 0 and source_path.endswith('/'): if self.operation_name == 'delete': # This is to filter out manually created folders # in S3. They have a size zero and would be # undesirably downloaded. Local directories # are automatically created when they do not # exist locally. But user should be able to # delete them. yield source_path, response_data elif not dir_op and s3_path != source_path: pass else: yield source_path, response_data
def list_objects(self, s3_path, dir_op): """ This function yields the appropriate object or objects under a common prefix depending if the operation is on objects under a common prefix. It yields the file's source path, size, and last update. """ # Short circuit path: if we are not recursing into the s3 # bucket and a specific path was given, we can just yield # that path and not have to call any operation in s3. bucket, prefix = find_bucket_key(s3_path) if not dir_op and prefix: yield self._list_single_object(s3_path) else: operation = self._service.get_operation('ListObjects') lister = BucketLister(operation, self._endpoint) for key in lister.list_objects(bucket=bucket, prefix=prefix, page_size=self.page_size): source_path, size, last_update = key if size == 0 and source_path.endswith('/'): if self.operation_name == 'delete': # This is to filter out manually created folders # in S3. They have a size zero and would be # undesirably downloaded. Local directories # are automatically created when they do not # exist locally. But user should be able to # delete them. yield source_path, size, last_update elif not dir_op and s3_path != source_path: pass else: yield source_path, size, last_update
def test_list_objects(self): now = mock.sentinel.now self.client.get_paginator.return_value.paginate = self.fake_paginate individual_response_elements = [{ 'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'a', 'Size': 1 }, { 'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'b', 'Size': 2 }, { 'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'c', 'Size': 3 }] self.responses = [{ 'Contents': individual_response_elements[0:2] }, { 'Contents': [individual_response_elements[2]] }] lister = BucketLister(self.client, self.date_parser) objects = list(lister.list_objects(bucket='foo')) self.assertEqual(objects, [('foo/a', individual_response_elements[0]), ('foo/b', individual_response_elements[1]), ('foo/c', individual_response_elements[2])]) for individual_response in individual_response_elements: self.assertEqual(individual_response['LastModified'], now)
def test_list_objects(self): now = mock.sentinel.now self.operation.paginate = self.fake_paginate self.responses = [ (None, { 'Contents': [ { 'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'a', 'Size': 1 }, { 'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'b', 'Size': 2 }, ] }), (None, { 'Contents': [ { 'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'c', 'Size': 3 }, ] }), ] lister = BucketLister(self.operation, self.endpoint, self.date_parser) objects = list(lister.list_objects(bucket='foo')) self.assertEqual(objects, [('foo/a', 1, now), ('foo/b', 2, now), ('foo/c', 3, now)])
def test_urlencoded_with_unicode_keys(self): now = mock.sentinel.now self.operation.paginate.return_value = [ (None, {'Contents': [ {'LastModified': '2014-02-27T04:20:38.000Z', 'Key': '%E2%9C%93', 'Size': 1}]}), ] lister = BucketLister(self.operation, self.endpoint, self.date_parser) objects = list(lister.list_objects(bucket='foo')) # And note how it's been converted to '\r'. self.assertEqual(objects, [(u'foo/\u2713', 1, now)])
def test_urlencoded_with_unicode_keys(self): now = mock.sentinel.now self.operation.paginate = self.fake_paginate self.responses = [ (None, {'Contents': [ {'LastModified': '2014-02-27T04:20:38.000Z', 'Key': '%E2%9C%93', 'Size': 1}]}), ] lister = BucketLister(self.operation, self.endpoint, self.date_parser) objects = list(lister.list_objects(bucket='foo')) # And note how it's been converted to '\r'. self.assertEqual(objects, [(u'foo/\u2713', 1, now)])
def test_urlencoded_with_unicode_keys(self): now = mock.sentinel.now self.client.get_paginator.return_value.paginate = self.fake_paginate self.responses = [ {'Contents': [ {'LastModified': '2014-02-27T04:20:38.000Z', 'Key': '%E2%9C%93', 'Size': 1}]}, ] lister = BucketLister(self.client, self.date_parser) objects = list(lister.list_objects(bucket='foo')) # And note how it's been converted to '\r'. self.assertEqual(objects, [(u'foo/\u2713', 1, now)])
def test_urlencoded_with_unicode_keys(self): now = mock.sentinel.now self.client.get_paginator.return_value.paginate = self.fake_paginate individual_response_element = { 'LastModified': '2014-02-27T04:20:38.000Z', 'Key': '%E2%9C%93', 'Size': 1} self.responses = [ {'Contents': [individual_response_element]} ] lister = BucketLister(self.client, self.date_parser) objects = list(lister.list_objects(bucket='foo')) # And note how it's been converted to '\r'. self.assertEqual( objects, [(u'foo/\u2713', individual_response_element)]) self.assertEqual(individual_response_element['LastModified'], now)
def test_urlencoded_keys(self): # In order to workaround control chars being in key names, # we force the urlencoding of the key names and we decode # them before yielding them. For example, note the %0D # in bar.txt: now = mock.sentinel.now self.operation.paginate.return_value = [ (None, {'Contents': [ {'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'bar%0D.txt', 'Size': 1}]}), ] lister = BucketLister(self.operation, self.endpoint, self.date_parser) objects = list(lister.list_objects(bucket='foo')) # And note how it's been converted to '\r'. self.assertEqual(objects, [('foo/bar\r.txt', 1, now)])
def test_list_objects_passes_in_extra_args(self): self.client.get_paginator.return_value.paginate.return_value = [{ 'Contents': [{ 'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'mykey', 'Size': 3 }] }] lister = BucketLister(self.client, self.date_parser) list( lister.list_objects(bucket='mybucket', extra_args={'RequestPayer': 'requester'})) self.client.get_paginator.return_value.paginate.assert_called_with( Bucket='mybucket', PaginationConfig={'PageSize': None}, RequestPayer='requester')
def test_urlencoded_keys(self): # In order to workaround control chars being in key names, # we force the urlencoding of the key names and we decode # them before yielding them. For example, note the %0D # in bar.txt: now = mock.sentinel.now self.operation.paginate = self.fake_paginate self.responses = [ (None, {'Contents': [ {'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'bar%0D.txt', 'Size': 1}]}), ] lister = BucketLister(self.operation, self.endpoint, self.date_parser) objects = list(lister.list_objects(bucket='foo')) # And note how it's been converted to '\r'. self.assertEqual(objects, [('foo/bar\r.txt', 1, now)])
def test_list_objects(self): now = mock.sentinel.now self.operation.paginate.return_value = [ (None, {'Contents': [ {'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'a', 'Size': 1}, {'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'b', 'Size': 2},]}), (None, {'Contents': [ {'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'c', 'Size': 3}, ]}), ] lister = BucketLister(self.operation, self.endpoint, self.date_parser) objects = list(lister.list_objects(bucket='foo')) self.assertEqual(objects, [('foo/a', 1, now), ('foo/b', 2, now), ('foo/c', 3, now)])
def test_list_objects_passes_in_extra_args(self): self.client.get_paginator.return_value.paginate.return_value = [ {'Contents': [ {'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'mykey', 'Size': 3} ]} ] lister = BucketLister(self.client, self.date_parser) list( lister.list_objects( bucket='mybucket', extra_args={'RequestPayer': 'requester'} ) ) self.client.get_paginator.return_value.paginate.assert_called_with( Bucket='mybucket', PaginationConfig={'PageSize': None}, RequestPayer='requester' )
def test_list_objects(self): now = mock.sentinel.now self.client.get_paginator.return_value.paginate = self.fake_paginate self.responses = [ {'Contents': [ {'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'a', 'Size': 1}, {'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'b', 'Size': 2}]}, {'Contents': [ {'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'c', 'Size': 3}]} ] lister = BucketLister(self.client, self.date_parser) objects = list(lister.list_objects(bucket='foo')) self.assertEqual(objects, [('foo/a', 1, now), ('foo/b', 2, now), ('foo/c', 3, now)])
def test_urlencoded_keys(self): # In order to workaround control chars being in key names, # we force the urlencoding of the key names and we decode # them before yielding them. For example, note the %0D # in bar.txt: now = mock.sentinel.now self.client.get_paginator.return_value.paginate = self.fake_paginate individual_response_element = { 'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'bar%0D.txt', 'Size': 1} self.responses = [ {'Contents': [individual_response_element]} ] lister = BucketLister(self.client, self.date_parser) objects = list(lister.list_objects(bucket='foo')) # And note how it's been converted to '\r'. self.assertEqual( objects, [('foo/bar\r.txt', individual_response_element)]) self.assertEqual(individual_response_element['LastModified'], now)
def test_list_objects(self): now = mock.sentinel.now self.client.get_paginator.return_value.paginate = self.fake_paginate individual_response_elements = [ {'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'a', 'Size': 1}, {'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'b', 'Size': 2}, {'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'c', 'Size': 3} ] self.responses = [ {'Contents': individual_response_elements[0:2]}, {'Contents': [individual_response_elements[2]]} ] lister = BucketLister(self.client, self.date_parser) objects = list(lister.list_objects(bucket='foo')) self.assertEqual(objects, [('foo/a', individual_response_elements[0]), ('foo/b', individual_response_elements[1]), ('foo/c', individual_response_elements[2])]) for individual_response in individual_response_elements: self.assertEqual(individual_response['LastModified'], now)
def test_list_objects(self): now = mock.sentinel.now self.client.get_paginator.return_value.paginate = self.fake_paginate individual_response_elements = [ {"LastModified": "2014-02-27T04:20:38.000Z", "Key": "a", "Size": 1}, {"LastModified": "2014-02-27T04:20:38.000Z", "Key": "b", "Size": 2}, {"LastModified": "2014-02-27T04:20:38.000Z", "Key": "c", "Size": 3}, ] self.responses = [ {"Contents": individual_response_elements[0:2]}, {"Contents": [individual_response_elements[2]]}, ] lister = BucketLister(self.client, self.date_parser) objects = list(lister.list_objects(bucket="foo")) self.assertEqual( objects, [ ("foo/a", individual_response_elements[0]), ("foo/b", individual_response_elements[1]), ("foo/c", individual_response_elements[2]), ], ) for individual_response in individual_response_elements: self.assertEqual(individual_response["LastModified"], now)
def test_list_objects(self): now = mock.sentinel.now self.client.get_paginator.return_value.paginate = self.fake_paginate self.responses = [{ 'Contents': [{ 'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'a', 'Size': 1 }, { 'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'b', 'Size': 2 }] }, { 'Contents': [{ 'LastModified': '2014-02-27T04:20:38.000Z', 'Key': 'c', 'Size': 3 }] }] lister = BucketLister(self.client, self.date_parser) objects = list(lister.list_objects(bucket='foo')) self.assertEqual(objects, [('foo/a', 1, now), ('foo/b', 2, now), ('foo/c', 3, now)])