def test_chunks(self): with self.assertRaises(ValueError): list(helpers.chunks([1, 2, 3], 0)) with self.assertRaises(ValueError): list(helpers.chunks([1, 2, 3], -3)) self.assertEqual(list(helpers.chunks([], 5)), []) self.assertEqual(list(helpers.chunks([1], 1)), [[1]]) self.assertEqual(list(helpers.chunks([1, 2, 3], 2)), [[1, 2], [3]])
def test_chunks(self): with pytest.raises(ValueError): list(helpers.chunks([1, 2, 3], 0)) with pytest.raises(ValueError): list(helpers.chunks([1, 2, 3], -3)) assert list(helpers.chunks([], 5)) == [] assert list(helpers.chunks([1], 1)) == [[1]] assert list(helpers.chunks([1, 2, 3], 2)) == [[1, 2], [3]]
def test_chunks(self): with self.assertRaises(ValueError): [i for i in helpers.chunks([1, 2, 3], 0)] with self.assertRaises(ValueError): [i for i in helpers.chunks([1, 2, 3], -3)] self.assertEqual([i for i in helpers.chunks([], 5)], []) self.assertEqual([i for i in helpers.chunks([1], 1)], [[1]]) self.assertEqual([i for i in helpers.chunks([1, 2, 3], 2)], [[1, 2], [3]])
def test_chunks(self): with self.assertRaises(ValueError): [i for i in helpers.chunks([1, 2, 3], 0)] with self.assertRaises(ValueError): [i for i in helpers.chunks([1, 2, 3], -3)] self.assertEqual([i for i in helpers.chunks([], 5)], []) self.assertEqual([i for i in helpers.chunks([1], 1)], [[1]]) self.assertEqual([i for i in helpers.chunks([1, 2, 3], 2)], [[1, 2], [3]])
def delete_objects(self, bucket: str, keys: Union[str, list]) -> None: """ Delete keys from the bucket. :param bucket: Name of the bucket in which you are going to delete object(s) :type bucket: str :param keys: The key(s) to delete from S3 bucket. When ``keys`` is a string, it's supposed to be the key name of the single object to delete. When ``keys`` is a list, it's supposed to be the list of the keys to delete. :type keys: str or list """ if isinstance(keys, str): keys = [keys] s3 = self.get_conn() # We can only send a maximum of 1000 keys per request. # For details see: # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.delete_objects for chunk in chunks(keys, chunk_size=1000): response = s3.delete_objects(Bucket=bucket, Delete={"Objects": [{"Key": k} for k in chunk]}) deleted_keys = [x['Key'] for x in response.get("Deleted", [])] self.log.info("Deleted: %s", deleted_keys) if "Errors" in response: errors_keys = [x['Key'] for x in response.get("Errors", [])] raise AirflowException(f"Errors when deleting: {errors_keys}")