def test_chunks(self):
with self.assertRaises(ValueError):
list(helpers.chunks([1, 2, 3], 0))
with self.assertRaises(ValueError):
list(helpers.chunks([1, 2, 3], -3))
self.assertEqual(list(helpers.chunks([], 5)), [])
self.assertEqual(list(helpers.chunks([1], 1)), [[1]])
self.assertEqual(list(helpers.chunks([1, 2, 3], 2)), [[1, 2], [3]])
def test_chunks(self):
with pytest.raises(ValueError):
list(helpers.chunks([1, 2, 3], 0))
with pytest.raises(ValueError):
list(helpers.chunks([1, 2, 3], -3))
assert list(helpers.chunks([], 5)) == []
assert list(helpers.chunks([1], 1)) == [[1]]
assert list(helpers.chunks([1, 2, 3], 2)) == [[1, 2], [3]]
def test_chunks(self):
with self.assertRaises(ValueError):
[i for i in helpers.chunks([1, 2, 3], 0)]
with self.assertRaises(ValueError):
[i for i in helpers.chunks([1, 2, 3], -3)]
self.assertEqual([i for i in helpers.chunks([], 5)], [])
self.assertEqual([i for i in helpers.chunks([1], 1)], [[1]])
self.assertEqual([i for i in helpers.chunks([1, 2, 3], 2)],
[[1, 2], [3]])
def test_chunks(self):
with self.assertRaises(ValueError):
[i for i in helpers.chunks([1, 2, 3], 0)]
with self.assertRaises(ValueError):
[i for i in helpers.chunks([1, 2, 3], -3)]
self.assertEqual([i for i in helpers.chunks([], 5)], [])
self.assertEqual([i for i in helpers.chunks([1], 1)], [[1]])
self.assertEqual([i for i in helpers.chunks([1, 2, 3], 2)],
[[1, 2], [3]])
def delete_objects(self, bucket: str, keys: Union[str, list]) -> None:
"""
Delete keys from the bucket.
:param bucket: Name of the bucket in which you are going to delete object(s)
:type bucket: str
:param keys: The key(s) to delete from S3 bucket.
When ``keys`` is a string, it's supposed to be the key name of
the single object to delete.
When ``keys`` is a list, it's supposed to be the list of the
keys to delete.
:type keys: str or list
"""
if isinstance(keys, str):
keys = [keys]
s3 = self.get_conn()
# We can only send a maximum of 1000 keys per request.
# For details see:
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.delete_objects
for chunk in chunks(keys, chunk_size=1000):
response = s3.delete_objects(Bucket=bucket, Delete={"Objects": [{"Key": k} for k in chunk]})
deleted_keys = [x['Key'] for x in response.get("Deleted", [])]
self.log.info("Deleted: %s", deleted_keys)
if "Errors" in response:
errors_keys = [x['Key'] for x in response.get("Errors", [])]
raise AirflowException(f"Errors when deleting: {errors_keys}")
