def list_objects(self, s3_path, dir_op):
"""
This function yields the appropriate object or objects under a
common prefix depending if the operation is on objects under a
common prefix. It yields the file's source path, size, and last
update.
"""
# Short circuit path: if we are not recursing into the s3
# bucket and a specific path was given, we can just yield
# that path and not have to call any operation in s3.
bucket, prefix = find_bucket_key(s3_path)
if not dir_op and prefix:
yield self._list_single_object(s3_path)
else:
lister = BucketLister(self._client)
for key in lister.list_objects(bucket=bucket, prefix=prefix,
page_size=self.page_size):
source_path, response_data = key
if response_data['Size'] == 0 and source_path.endswith('/'):
if self.operation_name == 'delete':
# This is to filter out manually created folders
# in S3. They have a size zero and would be
# undesirably downloaded. Local directories
# are automatically created when they do not
# exist locally. But user should be able to
# delete them.
yield source_path, response_data
elif not dir_op and s3_path != source_path:
pass
else:
yield source_path, response_data
def list_objects(self, s3_path, dir_op):
"""
This function yields the appropriate object or objects under a
common prefix depending if the operation is on objects under a
common prefix. It yields the file's source path, size, and last
update.
"""
# Short circuit path: if we are not recursing into the s3
# bucket and a specific path was given, we can just yield
# that path and not have to call any operation in s3.
bucket, prefix = find_bucket_key(s3_path)
if not dir_op and prefix:
yield self._list_single_object(s3_path)
else:
operation = self._service.get_operation('ListObjects')
lister = BucketLister(operation, self._endpoint)
for key in lister.list_objects(bucket=bucket,
prefix=prefix,
page_size=self.page_size):
source_path, size, last_update = key
if size == 0 and source_path.endswith('/'):
if self.operation_name == 'delete':
# This is to filter out manually created folders
# in S3. They have a size zero and would be
# undesirably downloaded. Local directories
# are automatically created when they do not
# exist locally. But user should be able to
# delete them.
yield source_path, size, last_update
elif not dir_op and s3_path != source_path:
pass
else:
yield source_path, size, last_update
def test_list_objects(self):
now = mock.sentinel.now
self.client.get_paginator.return_value.paginate = self.fake_paginate
individual_response_elements = [{
'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'a',
'Size': 1
}, {
'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'b',
'Size': 2
}, {
'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'c',
'Size': 3
}]
self.responses = [{
'Contents': individual_response_elements[0:2]
}, {
'Contents': [individual_response_elements[2]]
}]
lister = BucketLister(self.client, self.date_parser)
objects = list(lister.list_objects(bucket='foo'))
self.assertEqual(objects, [('foo/a', individual_response_elements[0]),
('foo/b', individual_response_elements[1]),
('foo/c', individual_response_elements[2])])
for individual_response in individual_response_elements:
self.assertEqual(individual_response['LastModified'], now)
def test_list_objects(self):
now = mock.sentinel.now
self.operation.paginate = self.fake_paginate
self.responses = [
(None, {
'Contents': [
{
'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'a',
'Size': 1
},
{
'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'b',
'Size': 2
},
]
}),
(None, {
'Contents': [
{
'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'c',
'Size': 3
},
]
}),
]
lister = BucketLister(self.operation, self.endpoint, self.date_parser)
objects = list(lister.list_objects(bucket='foo'))
self.assertEqual(objects, [('foo/a', 1, now), ('foo/b', 2, now),
('foo/c', 3, now)])
def test_urlencoded_with_unicode_keys(self):
now = mock.sentinel.now
self.operation.paginate.return_value = [
(None, {'Contents': [
{'LastModified': '2014-02-27T04:20:38.000Z',
'Key': '%E2%9C%93', 'Size': 1}]}),
]
lister = BucketLister(self.operation, self.endpoint, self.date_parser)
objects = list(lister.list_objects(bucket='foo'))
# And note how it's been converted to '\r'.
self.assertEqual(objects, [(u'foo/\u2713', 1, now)])
def test_urlencoded_with_unicode_keys(self):
now = mock.sentinel.now
self.operation.paginate = self.fake_paginate
self.responses = [
(None, {'Contents': [
{'LastModified': '2014-02-27T04:20:38.000Z',
'Key': '%E2%9C%93', 'Size': 1}]}),
]
lister = BucketLister(self.operation, self.endpoint, self.date_parser)
objects = list(lister.list_objects(bucket='foo'))
# And note how it's been converted to '\r'.
self.assertEqual(objects, [(u'foo/\u2713', 1, now)])
def test_urlencoded_with_unicode_keys(self):
now = mock.sentinel.now
self.client.get_paginator.return_value.paginate = self.fake_paginate
self.responses = [
{'Contents': [
{'LastModified': '2014-02-27T04:20:38.000Z',
'Key': '%E2%9C%93', 'Size': 1}]},
]
lister = BucketLister(self.client, self.date_parser)
objects = list(lister.list_objects(bucket='foo'))
# And note how it's been converted to '\r'.
self.assertEqual(objects, [(u'foo/\u2713', 1, now)])
def test_urlencoded_with_unicode_keys(self):
now = mock.sentinel.now
self.client.get_paginator.return_value.paginate = self.fake_paginate
individual_response_element = {
'LastModified': '2014-02-27T04:20:38.000Z',
'Key': '%E2%9C%93', 'Size': 1}
self.responses = [
{'Contents': [individual_response_element]}
]
lister = BucketLister(self.client, self.date_parser)
objects = list(lister.list_objects(bucket='foo'))
# And note how it's been converted to '\r'.
self.assertEqual(
objects, [(u'foo/\u2713', individual_response_element)])
self.assertEqual(individual_response_element['LastModified'], now)
def test_urlencoded_keys(self):
# In order to workaround control chars being in key names,
# we force the urlencoding of the key names and we decode
# them before yielding them. For example, note the %0D
# in bar.txt:
now = mock.sentinel.now
self.operation.paginate.return_value = [
(None, {'Contents': [
{'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'bar%0D.txt', 'Size': 1}]}),
]
lister = BucketLister(self.operation, self.endpoint, self.date_parser)
objects = list(lister.list_objects(bucket='foo'))
# And note how it's been converted to '\r'.
self.assertEqual(objects, [('foo/bar\r.txt', 1, now)])
def test_list_objects_passes_in_extra_args(self):
self.client.get_paginator.return_value.paginate.return_value = [{
'Contents': [{
'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'mykey',
'Size': 3
}]
}]
lister = BucketLister(self.client, self.date_parser)
list(
lister.list_objects(bucket='mybucket',
extra_args={'RequestPayer': 'requester'}))
self.client.get_paginator.return_value.paginate.assert_called_with(
Bucket='mybucket',
PaginationConfig={'PageSize': None},
RequestPayer='requester')
def test_urlencoded_keys(self):
# In order to workaround control chars being in key names,
# we force the urlencoding of the key names and we decode
# them before yielding them. For example, note the %0D
# in bar.txt:
now = mock.sentinel.now
self.operation.paginate = self.fake_paginate
self.responses = [
(None, {'Contents': [
{'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'bar%0D.txt', 'Size': 1}]}),
]
lister = BucketLister(self.operation, self.endpoint, self.date_parser)
objects = list(lister.list_objects(bucket='foo'))
# And note how it's been converted to '\r'.
self.assertEqual(objects, [('foo/bar\r.txt', 1, now)])
def test_list_objects(self):
now = mock.sentinel.now
self.operation.paginate.return_value = [
(None, {'Contents': [
{'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'a', 'Size': 1},
{'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'b', 'Size': 2},]}),
(None, {'Contents': [
{'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'c', 'Size': 3},
]}),
]
lister = BucketLister(self.operation, self.endpoint, self.date_parser)
objects = list(lister.list_objects(bucket='foo'))
self.assertEqual(objects, [('foo/a', 1, now), ('foo/b', 2, now),
('foo/c', 3, now)])
def test_list_objects_passes_in_extra_args(self):
self.client.get_paginator.return_value.paginate.return_value = [
{'Contents': [
{'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'mykey', 'Size': 3}
]}
]
lister = BucketLister(self.client, self.date_parser)
list(
lister.list_objects(
bucket='mybucket', extra_args={'RequestPayer': 'requester'}
)
)
self.client.get_paginator.return_value.paginate.assert_called_with(
Bucket='mybucket', PaginationConfig={'PageSize': None},
RequestPayer='requester'
)
def test_list_objects(self):
now = mock.sentinel.now
self.client.get_paginator.return_value.paginate = self.fake_paginate
self.responses = [
{'Contents': [
{'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'a', 'Size': 1},
{'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'b', 'Size': 2}]},
{'Contents': [
{'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'c', 'Size': 3}]}
]
lister = BucketLister(self.client, self.date_parser)
objects = list(lister.list_objects(bucket='foo'))
self.assertEqual(objects, [('foo/a', 1, now), ('foo/b', 2, now),
('foo/c', 3, now)])
def test_urlencoded_keys(self):
# In order to workaround control chars being in key names,
# we force the urlencoding of the key names and we decode
# them before yielding them. For example, note the %0D
# in bar.txt:
now = mock.sentinel.now
self.client.get_paginator.return_value.paginate = self.fake_paginate
individual_response_element = {
'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'bar%0D.txt', 'Size': 1}
self.responses = [
{'Contents': [individual_response_element]}
]
lister = BucketLister(self.client, self.date_parser)
objects = list(lister.list_objects(bucket='foo'))
# And note how it's been converted to '\r'.
self.assertEqual(
objects, [('foo/bar\r.txt', individual_response_element)])
self.assertEqual(individual_response_element['LastModified'], now)
def test_urlencoded_keys(self):
# In order to workaround control chars being in key names,
# we force the urlencoding of the key names and we decode
# them before yielding them. For example, note the %0D
# in bar.txt:
now = mock.sentinel.now
self.client.get_paginator.return_value.paginate = self.fake_paginate
individual_response_element = {
'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'bar%0D.txt', 'Size': 1}
self.responses = [
{'Contents': [individual_response_element]}
]
lister = BucketLister(self.client, self.date_parser)
objects = list(lister.list_objects(bucket='foo'))
# And note how it's been converted to '\r'.
self.assertEqual(
objects, [('foo/bar\r.txt', individual_response_element)])
self.assertEqual(individual_response_element['LastModified'], now)
def test_list_objects(self):
now = mock.sentinel.now
self.client.get_paginator.return_value.paginate = self.fake_paginate
individual_response_elements = [
{'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'a', 'Size': 1},
{'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'b', 'Size': 2},
{'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'c', 'Size': 3}
]
self.responses = [
{'Contents': individual_response_elements[0:2]},
{'Contents': [individual_response_elements[2]]}
]
lister = BucketLister(self.client, self.date_parser)
objects = list(lister.list_objects(bucket='foo'))
self.assertEqual(objects,
[('foo/a', individual_response_elements[0]),
('foo/b', individual_response_elements[1]),
('foo/c', individual_response_elements[2])])
for individual_response in individual_response_elements:
self.assertEqual(individual_response['LastModified'], now)
def test_list_objects(self):
now = mock.sentinel.now
self.client.get_paginator.return_value.paginate = self.fake_paginate
individual_response_elements = [
{"LastModified": "2014-02-27T04:20:38.000Z", "Key": "a", "Size": 1},
{"LastModified": "2014-02-27T04:20:38.000Z", "Key": "b", "Size": 2},
{"LastModified": "2014-02-27T04:20:38.000Z", "Key": "c", "Size": 3},
]
self.responses = [
{"Contents": individual_response_elements[0:2]},
{"Contents": [individual_response_elements[2]]},
]
lister = BucketLister(self.client, self.date_parser)
objects = list(lister.list_objects(bucket="foo"))
self.assertEqual(
objects,
[
("foo/a", individual_response_elements[0]),
("foo/b", individual_response_elements[1]),
("foo/c", individual_response_elements[2]),
],
)
for individual_response in individual_response_elements:
self.assertEqual(individual_response["LastModified"], now)
def test_list_objects(self):
now = mock.sentinel.now
self.client.get_paginator.return_value.paginate = self.fake_paginate
self.responses = [{
'Contents': [{
'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'a',
'Size': 1
}, {
'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'b',
'Size': 2
}]
}, {
'Contents': [{
'LastModified': '2014-02-27T04:20:38.000Z',
'Key': 'c',
'Size': 3
}]
}]
lister = BucketLister(self.client, self.date_parser)
objects = list(lister.list_objects(bucket='foo'))
self.assertEqual(objects, [('foo/a', 1, now), ('foo/b', 2, now),
('foo/c', 3, now)])