def test_firehose_s3(self): s3_resource = aws_stack.connect_to_resource("s3") firehose = aws_stack.connect_to_service("firehose") s3_prefix = "/testdata" test_data = '{"test": "firehose_data_%s"}' % short_uid() # create Firehose stream stream = firehose.create_delivery_stream( DeliveryStreamName=TEST_FIREHOSE_NAME, S3DestinationConfiguration={ "RoleARN": aws_stack.iam_resource_arn("firehose"), "BucketARN": aws_stack.s3_bucket_arn(TEST_BUCKET_NAME), "Prefix": s3_prefix, }, Tags=TEST_TAGS, ) self.assertTrue(stream) self.assertIn(TEST_FIREHOSE_NAME, firehose.list_delivery_streams()["DeliveryStreamNames"]) tags = firehose.list_tags_for_delivery_stream(DeliveryStreamName=TEST_FIREHOSE_NAME) self.assertEqual(TEST_TAGS, tags["Tags"]) # create target S3 bucket s3_resource.create_bucket(Bucket=TEST_BUCKET_NAME) # put records firehose.put_record( DeliveryStreamName=TEST_FIREHOSE_NAME, Record={"Data": to_bytes(test_data)} ) # check records in target bucket all_objects = testutil.list_all_s3_objects() testutil.assert_objects(json.loads(to_str(test_data)), all_objects) # check file layout in target bucket all_objects = testutil.map_all_s3_objects(buckets=[TEST_BUCKET_NAME]) for key in all_objects.keys(): self.assertRegex(key, r".*/\d{4}/\d{2}/\d{2}/\d{2}/.*\-\d{4}\-\d{2}\-\d{2}\-\d{2}.*")
def test_firehose_s3(): s3_resource = aws_stack.connect_to_resource('s3') firehose = aws_stack.connect_to_service('firehose') s3_prefix = '/testdata' test_data = '{"test": "firehose_data_%s"}' % short_uid() # create Firehose stream stream = firehose.create_delivery_stream( DeliveryStreamName=TEST_FIREHOSE_NAME, S3DestinationConfiguration={ 'RoleARN': aws_stack.iam_resource_arn('firehose'), 'BucketARN': aws_stack.s3_bucket_arn(TEST_BUCKET_NAME), 'Prefix': s3_prefix } ) assert stream assert TEST_FIREHOSE_NAME in firehose.list_delivery_streams()['DeliveryStreamNames'] # create target S3 bucket s3_resource.create_bucket(Bucket=TEST_BUCKET_NAME) # put records firehose.put_record( DeliveryStreamName=TEST_FIREHOSE_NAME, Record={ 'Data': to_bytes(test_data) } ) # check records in target bucket all_objects = testutil.list_all_s3_objects() testutil.assert_objects(json.loads(to_str(test_data)), all_objects)
def test_add_lambda_permission(self): iam_client = aws_stack.connect_to_service('iam') # create lambda permission action = 'lambda:InvokeFunction' resp = self.lambda_client.add_permission( FunctionName=TEST_LAMBDA_NAME_PY, Action=action, StatementId='s3', Principal='s3.amazonaws.com', SourceArn=aws_stack.s3_bucket_arn('test-bucket')) self.assertIn('Statement', resp) # fetch lambda policy policy = self.lambda_client.get_policy( FunctionName=TEST_LAMBDA_NAME_PY)['Policy'] self.assertEqual(policy['Statement'][0]['Action'], action) self.assertEqual(policy['Statement'][0]['Resource'], lambda_api.func_arn(TEST_LAMBDA_NAME_PY)) # fetch IAM policy policies = iam_client.list_policies(Scope='Local', MaxItems=500)['Policies'] matching = [ p for p in policies if p['PolicyName'] == 'lambda_policy_%s' % TEST_LAMBDA_NAME_PY ] self.assertEqual(len(matching), 1) self.assertIn(':policy/', matching[0]['Arn']) # remove permission that we just added resp = self.lambda_client.remove_permission( FunctionName=TEST_LAMBDA_NAME_PY, StatementId=resp['Statement'], Qualifier='qual1', RevisionId='r1') self.assertEqual(resp['ResponseMetadata']['HTTPStatusCode'], 200)
def test_firehose_s3(env=ENV_DEV): s3_resource = aws_stack.connect_to_resource('s3', env=env) s3_client = aws_stack.connect_to_service('s3', env=env) firehose = aws_stack.connect_to_service('firehose', env=env) s3_prefix = '/testdata' bucket_name = 'test_bucket' test_data = b'{"test": "data123"}' # create Firehose stream stream = firehose.create_delivery_stream( DeliveryStreamName=TEST_FIREHOSE_NAME, S3DestinationConfiguration={ 'RoleARN': aws_stack.iam_resource_arn('firehose'), 'BucketARN': aws_stack.s3_bucket_arn(bucket_name), 'Prefix': s3_prefix } ) # create target S3 bucket s3_resource.create_bucket(Bucket=bucket_name) # put records firehose.put_record( DeliveryStreamName=TEST_FIREHOSE_NAME, Record={ 'Data': test_data } ) # check records in target bucket all_objects = testutil.list_all_s3_objects() testutil.assert_objects(json.loads(test_data), all_objects)
def test_cfn_handle_events_rule(self): stack_name = 'stack-%s' % short_uid() bucket_name = 'target-%s' % short_uid() rule_prefix = 's3-rule-%s' % short_uid() rule_name = '%s-%s' % (rule_prefix, short_uid()) cfn = aws_stack.connect_to_service('cloudformation') events = aws_stack.connect_to_service('events') _deploy_stack(stack_name=stack_name, template_body=TEST_TEMPLATE_16 % (bucket_name, rule_name)) rs = events.list_rules( NamePrefix=rule_prefix ) self.assertIn(rule_name, [rule['Name'] for rule in rs['Rules']]) target_arn = aws_stack.s3_bucket_arn(bucket_name) rs = events.list_targets_by_rule( Rule=rule_name ) self.assertIn(target_arn, [target['Arn'] for target in rs['Targets']]) cfn.delete_stack(StackName=stack_name) rs = events.list_rules( NamePrefix=rule_prefix ) self.assertNotIn(rule_name, [rule['Name'] for rule in rs['Rules']])
def test_firehose_s3(): s3_resource = aws_stack.connect_to_resource('s3') firehose = aws_stack.connect_to_service('firehose') s3_prefix = '/testdata' test_data = '{"test": "firehose_data_%s"}' % short_uid() # create Firehose stream stream = firehose.create_delivery_stream( DeliveryStreamName=TEST_FIREHOSE_NAME, S3DestinationConfiguration={ 'RoleARN': aws_stack.iam_resource_arn('firehose'), 'BucketARN': aws_stack.s3_bucket_arn(TEST_BUCKET_NAME), 'Prefix': s3_prefix } ) assert stream assert TEST_FIREHOSE_NAME in firehose.list_delivery_streams()['DeliveryStreamNames'] # create target S3 bucket s3_resource.create_bucket(Bucket=TEST_BUCKET_NAME) # put records firehose.put_record( DeliveryStreamName=TEST_FIREHOSE_NAME, Record={ 'Data': to_bytes(test_data) } ) # check records in target bucket all_objects = testutil.list_all_s3_objects() testutil.assert_objects(json.loads(to_str(test_data)), all_objects)
def test_apply_template(self): cloudformation = aws_stack.connect_to_resource('cloudformation') s3 = aws_stack.connect_to_service('s3') sns = aws_stack.connect_to_service('sns') apigateway = aws_stack.connect_to_service('apigateway') template = template_deployer.template_to_json(load_file(TEST_TEMPLATE_1)) # deploy template stack_name = 'stack-%s' % short_uid() cloudformation.create_stack(StackName=stack_name, TemplateBody=template) # wait for deployment to finish def check_stack(): stack = get_stack_details(stack_name) self.assertEqual(stack['StackStatus'], 'CREATE_COMPLETE') retry(check_stack, retries=3, sleep=2) # assert that bucket has been created assert bucket_exists('cf-test-bucket-1') # assert that queue has been created assert queue_exists('cf-test-queue-1') # assert that topic has been created topic_arn = topic_exists('%s-test-topic-1-1' % stack_name) assert topic_arn # assert that stream has been created assert stream_exists('cf-test-stream-1') # assert that queue has been created resource = describe_stack_resource(stack_name, 'SQSQueueNoNameProperty') assert queue_exists(resource['PhysicalResourceId']) # assert that tags have been created tags = s3.get_bucket_tagging(Bucket='cf-test-bucket-1')['TagSet'] self.assertEqual(tags, [{'Key': 'foobar', 'Value': aws_stack.get_sqs_queue_url('cf-test-queue-1')}]) tags = sns.list_tags_for_resource(ResourceArn=topic_arn)['Tags'] self.assertEqual(tags, [ {'Key': 'foo', 'Value': 'cf-test-bucket-1'}, {'Key': 'bar', 'Value': aws_stack.s3_bucket_arn('cf-test-bucket-1')} ]) # assert that subscriptions have been created subs = sns.list_subscriptions()['Subscriptions'] subs = [s for s in subs if (':%s:cf-test-queue-1' % TEST_AWS_ACCOUNT_ID) in s['Endpoint']] self.assertEqual(len(subs), 1) self.assertIn(':%s:%s-test-topic-1-1' % (TEST_AWS_ACCOUNT_ID, stack_name), subs[0]['TopicArn']) # assert that Gateway responses have been created test_api_name = 'test-api' api = [a for a in apigateway.get_rest_apis()['items'] if a['name'] == test_api_name][0] responses = apigateway.get_gateway_responses(restApiId=api['id'])['items'] self.assertEqual(len(responses), 2) types = [r['responseType'] for r in responses] self.assertEqual(set(types), set(['UNAUTHORIZED', 'DEFAULT_5XX']))
def test_firehose_kinesis_to_s3(self): kinesis = aws_stack.connect_to_service('kinesis') s3_resource = aws_stack.connect_to_resource('s3') firehose = aws_stack.connect_to_service('firehose') aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True) s3_prefix = '/testdata' test_data = '{"test": "firehose_data_%s"}' % short_uid() # create Firehose stream stream = firehose.create_delivery_stream( DeliveryStreamType='KinesisStreamAsSource', KinesisStreamSourceConfiguration={ 'RoleARN': aws_stack.iam_resource_arn('firehose'), 'KinesisStreamARN': aws_stack.kinesis_stream_arn(TEST_STREAM_NAME) }, DeliveryStreamName=TEST_FIREHOSE_NAME, S3DestinationConfiguration={ 'RoleARN': aws_stack.iam_resource_arn('firehose'), 'BucketARN': aws_stack.s3_bucket_arn(TEST_BUCKET_NAME), 'Prefix': s3_prefix } ) self.assertTrue(stream) self.assertIn(TEST_FIREHOSE_NAME, firehose.list_delivery_streams()['DeliveryStreamNames']) # create target S3 bucket s3_resource.create_bucket(Bucket=TEST_BUCKET_NAME) # put records kinesis.put_record( Data=to_bytes(test_data), PartitionKey='testId', StreamName=TEST_STREAM_NAME ) time.sleep(3) # check records in target bucket all_objects = testutil.list_all_s3_objects() testutil.assert_objects(json.loads(to_str(test_data)), all_objects)
def test_firehose_kinesis_to_s3(self): kinesis = aws_stack.create_external_boto_client("kinesis") s3_resource = aws_stack.connect_to_resource("s3") firehose = aws_stack.create_external_boto_client("firehose") aws_stack.create_kinesis_stream(TEST_STREAM_NAME, delete=True) s3_prefix = "/testdata" test_data = '{"test": "firehose_data_%s"}' % short_uid() # create Firehose stream stream = firehose.create_delivery_stream( DeliveryStreamType="KinesisStreamAsSource", KinesisStreamSourceConfiguration={ "RoleARN": aws_stack.iam_resource_arn("firehose"), "KinesisStreamARN": aws_stack.kinesis_stream_arn(TEST_STREAM_NAME), }, DeliveryStreamName=TEST_FIREHOSE_NAME, S3DestinationConfiguration={ "RoleARN": aws_stack.iam_resource_arn("firehose"), "BucketARN": aws_stack.s3_bucket_arn(TEST_BUCKET_NAME), "Prefix": s3_prefix, }, ) self.assertTrue(stream) self.assertIn(TEST_FIREHOSE_NAME, firehose.list_delivery_streams()["DeliveryStreamNames"]) # create target S3 bucket s3_resource.create_bucket(Bucket=TEST_BUCKET_NAME) # put records kinesis.put_record(Data=to_bytes(test_data), PartitionKey="testId", StreamName=TEST_STREAM_NAME) time.sleep(3) # check records in target bucket all_objects = testutil.list_all_s3_objects() testutil.assert_objects(json.loads(to_str(test_data)), all_objects)
def test_firehose_s3(self): s3_resource = aws_stack.connect_to_resource('s3') firehose = aws_stack.connect_to_service('firehose') s3_prefix = '/testdata' test_data = '{"test": "firehose_data_%s"}' % short_uid() # create Firehose stream stream = firehose.create_delivery_stream( DeliveryStreamName=TEST_FIREHOSE_NAME, S3DestinationConfiguration={ 'RoleARN': aws_stack.iam_resource_arn('firehose'), 'BucketARN': aws_stack.s3_bucket_arn(TEST_BUCKET_NAME), 'Prefix': s3_prefix }, Tags=TEST_TAGS ) self.assertTrue(stream) self.assertIn(TEST_FIREHOSE_NAME, firehose.list_delivery_streams()['DeliveryStreamNames']) tags = firehose.list_tags_for_delivery_stream(DeliveryStreamName=TEST_FIREHOSE_NAME) self.assertEquals(TEST_TAGS, tags['Tags']) # create target S3 bucket s3_resource.create_bucket(Bucket=TEST_BUCKET_NAME) # put records firehose.put_record( DeliveryStreamName=TEST_FIREHOSE_NAME, Record={ 'Data': to_bytes(test_data) } ) # check records in target bucket all_objects = testutil.list_all_s3_objects() testutil.assert_objects(json.loads(to_str(test_data)), all_objects) # check file layout in target bucket all_objects = testutil.map_all_s3_objects(buckets=[TEST_BUCKET_NAME]) for key in all_objects.keys(): self.assertRegexpMatches(key, r'.*/\d{4}/\d{2}/\d{2}/\d{2}/.*\-\d{4}\-\d{2}\-\d{2}\-\d{2}.*')
def create_sqs_bucket_notification( s3_client: "S3Client", sqs_client: "SQSClient", bucket_name: str, queue_url: str, events: List["EventType"], ): """A NotificationFactory.""" queue_arn = get_queue_arn(sqs_client, queue_url) assert queue_arn bucket_arn = aws_stack.s3_bucket_arn(bucket_name) policy = { "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": "*", "Action": "sqs:SendMessage", "Resource": queue_arn, "Condition": { "ArnEquals": { "aws:SourceArn": bucket_arn } }, }], } sqs_client.set_queue_attributes(QueueUrl=queue_url, Attributes={"Policy": json.dumps(policy)}) s3_client.put_bucket_notification_configuration( Bucket=bucket_name, NotificationConfiguration=dict( QueueConfigurations=[dict( QueueArn=queue_arn, Events=events, )]), )
def create_sns_bucket_notification( s3_client: "S3Client", sns_client: "SNSClient", bucket_name: str, topic_arn: str, events: List["EventType"], ): """A NotificationFactory.""" bucket_arn = aws_stack.s3_bucket_arn(bucket_name) policy = { "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": "*", "Action": "sns:Publish", "Resource": topic_arn, "Condition": { "ArnEquals": { "aws:SourceArn": bucket_arn } }, }], } sns_client.set_topic_attributes(TopicArn=topic_arn, AttributeName="Policy", AttributeValue=json.dumps(policy)) s3_client.put_bucket_notification_configuration( Bucket=bucket_name, NotificationConfiguration=dict( TopicConfigurations=[dict( TopicArn=topic_arn, Events=events, )]), )
def test_put_events_with_target_firehose(self): s3_bucket = 's3-{}'.format(short_uid()) s3_prefix = 'testeventdata' stream_name = 'firehose-{}'.format(short_uid()) rule_name = 'rule-{}'.format(short_uid()) target_id = 'target-{}'.format(short_uid()) # create firehose target bucket s3_client = aws_stack.connect_to_service('s3') s3_client.create_bucket(Bucket=s3_bucket) # create firehose delivery stream to s3 firehose_client = aws_stack.connect_to_service('firehose') stream = firehose_client.create_delivery_stream( DeliveryStreamName=stream_name, S3DestinationConfiguration={ 'RoleARN': aws_stack.iam_resource_arn('firehose'), 'BucketARN': aws_stack.s3_bucket_arn(s3_bucket), 'Prefix': s3_prefix }) stream_arn = stream['DeliveryStreamARN'] self.events_client.create_event_bus(Name=TEST_EVENT_BUS_NAME) self.events_client.put_rule( Name=rule_name, EventBusName=TEST_EVENT_BUS_NAME, EventPattern=json.dumps(TEST_EVENT_PATTERN)) rs = self.events_client.put_targets(Rule=rule_name, EventBusName=TEST_EVENT_BUS_NAME, Targets=[{ 'Id': target_id, 'Arn': stream_arn }]) self.assertIn('FailedEntryCount', rs) self.assertIn('FailedEntries', rs) self.assertEqual(rs['FailedEntryCount'], 0) self.assertEqual(rs['FailedEntries'], []) self.events_client.put_events( Entries=[{ 'EventBusName': TEST_EVENT_BUS_NAME, 'Source': TEST_EVENT_PATTERN['Source'], 'DetailType': TEST_EVENT_PATTERN['DetailType'], 'Detail': TEST_EVENT_PATTERN['Detail'] }]) # run tests bucket_contents = s3_client.list_objects(Bucket=s3_bucket)['Contents'] self.assertEqual(len(bucket_contents), 1) key = bucket_contents[0]['Key'] s3_object = s3_client.get_object(Bucket=s3_bucket, Key=key) self.assertEqual((s3_object['Body'].read()).decode(), str(TEST_EVENT_PATTERN['Detail'])) # clean up firehose_client.delete_delivery_stream(DeliveryStreamName=stream_name) # empty and delete bucket s3_client.delete_object(Bucket=s3_bucket, Key=key) s3_client.delete_bucket(Bucket=s3_bucket) self.events_client.remove_targets(Rule=rule_name, EventBusName=TEST_EVENT_BUS_NAME, Ids=[target_id], Force=True) self.events_client.delete_rule(Name=rule_name, EventBusName=TEST_EVENT_BUS_NAME, Force=True) self.events_client.delete_event_bus(Name=TEST_EVENT_BUS_NAME)
def test_put_events_with_target_firehose(self, events_client, s3_client, firehose_client): s3_bucket = "s3-{}".format(short_uid()) s3_prefix = "testeventdata" stream_name = "firehose-{}".format(short_uid()) rule_name = "rule-{}".format(short_uid()) target_id = "target-{}".format(short_uid()) bus_name = "bus-{}".format(short_uid()) # create firehose target bucket s3_client.create_bucket(Bucket=s3_bucket) # create firehose delivery stream to s3 stream = firehose_client.create_delivery_stream( DeliveryStreamName=stream_name, S3DestinationConfiguration={ "RoleARN": aws_stack.iam_resource_arn("firehose"), "BucketARN": aws_stack.s3_bucket_arn(s3_bucket), "Prefix": s3_prefix, }, ) stream_arn = stream["DeliveryStreamARN"] events_client.create_event_bus(Name=bus_name) events_client.put_rule( Name=rule_name, EventBusName=bus_name, EventPattern=json.dumps(TEST_EVENT_PATTERN), ) rs = events_client.put_targets( Rule=rule_name, EventBusName=bus_name, Targets=[{"Id": target_id, "Arn": stream_arn}], ) assert "FailedEntryCount" in rs assert "FailedEntries" in rs assert rs["FailedEntryCount"] == 0 assert rs["FailedEntries"] == [] events_client.put_events( Entries=[ { "EventBusName": bus_name, "Source": TEST_EVENT_PATTERN["source"][0], "DetailType": TEST_EVENT_PATTERN["detail-type"][0], "Detail": json.dumps(EVENT_DETAIL), } ] ) # run tests bucket_contents = s3_client.list_objects(Bucket=s3_bucket)["Contents"] assert len(bucket_contents) == 1 key = bucket_contents[0]["Key"] s3_object = s3_client.get_object(Bucket=s3_bucket, Key=key) actual_event = json.loads(s3_object["Body"].read().decode()) self.assert_valid_event(actual_event) assert actual_event["detail"] == EVENT_DETAIL # clean up firehose_client.delete_delivery_stream(DeliveryStreamName=stream_name) # empty and delete bucket s3_client.delete_object(Bucket=s3_bucket, Key=key) s3_client.delete_bucket(Bucket=s3_bucket) self.cleanup(bus_name, rule_name, target_id)
def S3_Bucket_get_cfn_attribute(self, attribute_name): if attribute_name in ['Arn']: return aws_stack.s3_bucket_arn(self.name) return S3_Bucket_get_cfn_attribute_orig(self, attribute_name)
def test_create_delete_stack(self): cloudformation = aws_stack.connect_to_resource('cloudformation') cf_client = aws_stack.connect_to_service('cloudformation') s3 = aws_stack.connect_to_service('s3') sns = aws_stack.connect_to_service('sns') sqs = aws_stack.connect_to_service('sqs') apigateway = aws_stack.connect_to_service('apigateway') template = template_deployer.template_to_json( load_file(TEST_TEMPLATE_1)) # deploy template stack_name = 'stack-%s' % short_uid() cloudformation.create_stack(StackName=stack_name, TemplateBody=template) # wait for deployment to finish def check_stack(): stack = get_stack_details(stack_name) self.assertEqual(stack['StackStatus'], 'CREATE_COMPLETE') retry(check_stack, retries=3, sleep=2) # assert that resources have been created assert bucket_exists('cf-test-bucket-1') queue_url = queue_exists('cf-test-queue-1') assert queue_url topic_arn = topic_exists('%s-test-topic-1-1' % stack_name) assert topic_arn assert stream_exists('cf-test-stream-1') resource = describe_stack_resource(stack_name, 'SQSQueueNoNameProperty') assert queue_exists(resource['PhysicalResourceId']) assert ssm_param_exists('cf-test-param-1') # assert that tags have been created tags = s3.get_bucket_tagging(Bucket='cf-test-bucket-1')['TagSet'] self.assertEqual( tags, [{ 'Key': 'foobar', 'Value': aws_stack.get_sqs_queue_url('cf-test-queue-1') }]) tags = sns.list_tags_for_resource(ResourceArn=topic_arn)['Tags'] self.assertEqual( tags, [{ 'Key': 'foo', 'Value': 'cf-test-bucket-1' }, { 'Key': 'bar', 'Value': aws_stack.s3_bucket_arn('cf-test-bucket-1') }]) queue_tags = sqs.list_queue_tags(QueueUrl=queue_url) self.assertIn('Tags', queue_tags) self.assertEqual(queue_tags['Tags'], { 'key1': 'value1', 'key2': 'value2' }) # assert that bucket notifications have been created notifs = s3.get_bucket_notification_configuration( Bucket='cf-test-bucket-1') self.assertIn('QueueConfigurations', notifs) self.assertIn('LambdaFunctionConfigurations', notifs) self.assertEqual(notifs['QueueConfigurations'][0]['QueueArn'], 'aws:arn:sqs:test:testqueue') self.assertEqual(notifs['QueueConfigurations'][0]['Events'], ['s3:ObjectDeleted:*']) self.assertEqual( notifs['LambdaFunctionConfigurations'][0]['LambdaFunctionArn'], 'aws:arn:lambda:test:testfunc') self.assertEqual(notifs['LambdaFunctionConfigurations'][0]['Events'], ['s3:ObjectCreated:*']) # assert that subscriptions have been created subs = sns.list_subscriptions()['Subscriptions'] subs = [ s for s in subs if (':%s:cf-test-queue-1' % TEST_AWS_ACCOUNT_ID) in s['Endpoint'] ] self.assertEqual(len(subs), 1) self.assertIn( ':%s:%s-test-topic-1-1' % (TEST_AWS_ACCOUNT_ID, stack_name), subs[0]['TopicArn']) # assert that subscription attributes are added properly attrs = sns.get_subscription_attributes( SubscriptionArn=subs[0]['SubscriptionArn'])['Attributes'] self.assertEqual( attrs, { 'Endpoint': subs[0]['Endpoint'], 'Protocol': 'sqs', 'SubscriptionArn': subs[0]['SubscriptionArn'], 'TopicArn': subs[0]['TopicArn'], 'FilterPolicy': json.dumps({'eventType': ['created']}) }) # assert that Gateway responses have been created test_api_name = 'test-api' api = [ a for a in apigateway.get_rest_apis()['items'] if a['name'] == test_api_name ][0] responses = apigateway.get_gateway_responses( restApiId=api['id'])['items'] self.assertEqual(len(responses), 2) types = [r['responseType'] for r in responses] self.assertEqual(set(types), set(['UNAUTHORIZED', 'DEFAULT_5XX'])) # delete the stack cf_client.delete_stack(StackName=stack_name) # assert that resources have been deleted assert not bucket_exists('cf-test-bucket-1') assert not queue_exists('cf-test-queue-1') assert not topic_exists('%s-test-topic-1-1' % stack_name) retry(lambda: self.assertFalse(stream_exists('cf-test-stream-1')))
def test_delivery_stream_with_kinesis_as_source( self, firehose_client, kinesis_client, s3_client, s3_bucket, kinesis_create_stream, ): bucket_arn = aws_stack.s3_bucket_arn(s3_bucket) stream_name = f"test-stream-{short_uid()}" log_group_name = f"group{short_uid()}" role_arn = "arn:aws:iam::000000000000:role/Firehose-Role" delivery_stream_name = f"test-delivery-stream-{short_uid()}" kinesis_create_stream(StreamName=stream_name, ShardCount=2) stream_arn = kinesis_client.describe_stream( StreamName=stream_name)["StreamDescription"]["StreamARN"] response = firehose_client.create_delivery_stream( DeliveryStreamName=delivery_stream_name, DeliveryStreamType="KinesisStreamAsSource", KinesisStreamSourceConfiguration={ "KinesisStreamARN": stream_arn, "RoleARN": role_arn, }, ExtendedS3DestinationConfiguration={ "BucketARN": bucket_arn, "RoleARN": role_arn, "BufferingHints": { "IntervalInSeconds": 60, "SizeInMBs": 64 }, "DynamicPartitioningConfiguration": { "Enabled": True }, "ProcessingConfiguration": { "Enabled": True, "Processors": [ { "Type": "MetadataExtraction", "Parameters": [ { "ParameterName": "MetadataExtractionQuery", "ParameterValue": "{s3Prefix: .tableName}", }, { "ParameterName": "JsonParsingEngine", "ParameterValue": "JQ-1.6" }, ], }, ], }, "DataFormatConversionConfiguration": { "Enabled": True }, "CompressionFormat": "GZIP", "Prefix": "firehoseTest/!{partitionKeyFromQuery:s3Prefix}/!{partitionKeyFromLambda:companyId}/!{partitionKeyFromLambda:year}/!{partitionKeyFromLambda:month}/", "ErrorOutputPrefix": "firehoseTest-errors/!{firehose:error-output-type}/", "CloudWatchLoggingOptions": { "Enabled": True, "LogGroupName": log_group_name, }, }, ) assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 firehose_client.delete_delivery_stream( DeliveryStreamName=delivery_stream_name) kinesis_client.delete_stream(StreamName=stream_name) s3_client.delete_bucket(Bucket=s3_bucket)
def test_kinesis_firehose_opensearch_s3_backup( self, firehose_client, kinesis_client, opensearch_client, s3_client, s3_bucket, kinesis_create_stream, monkeypatch, opensearch_endpoint_strategy, ): domain_name = f"test-domain-{short_uid()}" stream_name = f"test-stream-{short_uid()}" role_arn = "arn:aws:iam::000000000000:role/Firehose-Role" delivery_stream_name = f"test-delivery-stream-{short_uid()}" monkeypatch.setattr(config, "OPENSEARCH_ENDPOINT_STRATEGY", opensearch_endpoint_strategy) try: opensearch_create_response = opensearch_client.create_domain( DomainName=domain_name) opensearch_url = f"http://{opensearch_create_response['DomainStatus']['Endpoint']}" opensearch_arn = opensearch_create_response["DomainStatus"]["ARN"] # create s3 backup bucket arn bucket_arn = aws_stack.s3_bucket_arn(s3_bucket) # create kinesis stream kinesis_create_stream(StreamName=stream_name, ShardCount=2) stream_arn = kinesis_client.describe_stream( StreamName=stream_name)["StreamDescription"]["StreamARN"] kinesis_stream_source_def = { "KinesisStreamARN": stream_arn, "RoleARN": role_arn, } opensearch_destination_configuration = { "RoleARN": role_arn, "DomainARN": opensearch_arn, "IndexName": "activity", "TypeName": "activity", "S3BackupMode": "AllDocuments", "S3Configuration": { "RoleARN": role_arn, "BucketARN": bucket_arn, }, } firehose_client.create_delivery_stream( DeliveryStreamName=delivery_stream_name, DeliveryStreamType="KinesisStreamAsSource", KinesisStreamSourceConfiguration=kinesis_stream_source_def, AmazonopensearchserviceDestinationConfiguration= opensearch_destination_configuration, ) # wait for opensearch cluster to be ready def check_domain_state(): result = opensearch_client.describe_domain( DomainName=domain_name)["DomainStatus"]["Processing"] return not result assert poll_condition(check_domain_state, 30, 1) # put kinesis stream record kinesis_record = {"target": "hello"} kinesis_client.put_record(StreamName=stream_name, Data=to_bytes( json.dumps(kinesis_record)), PartitionKey="1") firehose_record = {"target": "world"} firehose_client.put_record( DeliveryStreamName=delivery_stream_name, Record={"Data": to_bytes(json.dumps(firehose_record))}, ) def assert_opensearch_contents(): response = requests.get(f"{opensearch_url}/activity/_search") response_bod = response.json() assert "hits" in response_bod response_bod_hits = response_bod["hits"] assert "hits" in response_bod_hits result = response_bod_hits["hits"] assert len(result) == 2 sources = [item["_source"] for item in result] assert firehose_record in sources assert kinesis_record in sources retry(assert_opensearch_contents) def assert_s3_contents(): result = s3_client.list_objects(Bucket=s3_bucket) contents = [] for o in result.get("Contents"): data = s3_client.get_object(Bucket=s3_bucket, Key=o.get("Key")) content = data["Body"].read() contents.append(content) assert len(contents) == 2 assert to_bytes(json.dumps(firehose_record)) in contents assert to_bytes(json.dumps(kinesis_record)) in contents retry(assert_s3_contents) finally: firehose_client.delete_delivery_stream( DeliveryStreamName=delivery_stream_name) opensearch_client.delete_domain(DomainName=domain_name)