def test_firehose_s3(self): s3_resource = aws_stack.connect_to_resource("s3") firehose = aws_stack.connect_to_service("firehose") s3_prefix = "/testdata" test_data = '{"test": "firehose_data_%s"}' % short_uid() # create Firehose stream stream = firehose.create_delivery_stream( DeliveryStreamName=TEST_FIREHOSE_NAME, S3DestinationConfiguration={ "RoleARN": aws_stack.iam_resource_arn("firehose"), "BucketARN": aws_stack.s3_bucket_arn(TEST_BUCKET_NAME), "Prefix": s3_prefix, }, Tags=TEST_TAGS, ) self.assertTrue(stream) self.assertIn(TEST_FIREHOSE_NAME, firehose.list_delivery_streams()["DeliveryStreamNames"]) tags = firehose.list_tags_for_delivery_stream(DeliveryStreamName=TEST_FIREHOSE_NAME) self.assertEqual(TEST_TAGS, tags["Tags"]) # create target S3 bucket s3_resource.create_bucket(Bucket=TEST_BUCKET_NAME) # put records firehose.put_record( DeliveryStreamName=TEST_FIREHOSE_NAME, Record={"Data": to_bytes(test_data)} ) # check records in target bucket all_objects = testutil.list_all_s3_objects() testutil.assert_objects(json.loads(to_str(test_data)), all_objects) # check file layout in target bucket all_objects = testutil.map_all_s3_objects(buckets=[TEST_BUCKET_NAME]) for key in all_objects.keys(): self.assertRegex(key, r".*/\d{4}/\d{2}/\d{2}/\d{2}/.*\-\d{4}\-\d{2}\-\d{2}\-\d{2}.*")
def test_firehose_s3(self): s3_resource = aws_stack.connect_to_resource('s3') firehose = aws_stack.connect_to_service('firehose') s3_prefix = '/testdata' test_data = '{"test": "firehose_data_%s"}' % short_uid() # create Firehose stream stream = firehose.create_delivery_stream( DeliveryStreamName=TEST_FIREHOSE_NAME, S3DestinationConfiguration={ 'RoleARN': aws_stack.iam_resource_arn('firehose'), 'BucketARN': aws_stack.s3_bucket_arn(TEST_BUCKET_NAME), 'Prefix': s3_prefix }, Tags=TEST_TAGS ) self.assertTrue(stream) self.assertIn(TEST_FIREHOSE_NAME, firehose.list_delivery_streams()['DeliveryStreamNames']) tags = firehose.list_tags_for_delivery_stream(DeliveryStreamName=TEST_FIREHOSE_NAME) self.assertEquals(TEST_TAGS, tags['Tags']) # create target S3 bucket s3_resource.create_bucket(Bucket=TEST_BUCKET_NAME) # put records firehose.put_record( DeliveryStreamName=TEST_FIREHOSE_NAME, Record={ 'Data': to_bytes(test_data) } ) # check records in target bucket all_objects = testutil.list_all_s3_objects() testutil.assert_objects(json.loads(to_str(test_data)), all_objects) # check file layout in target bucket all_objects = testutil.map_all_s3_objects(buckets=[TEST_BUCKET_NAME]) for key in all_objects.keys(): self.assertRegexpMatches(key, r'.*/\d{4}/\d{2}/\d{2}/\d{2}/.*\-\d{4}\-\d{2}\-\d{2}\-\d{2}.*')