def queue_exists(name):
sqs_client = aws_stack.connect_to_service('sqs')
queues = sqs_client.list_queues()
url = name if '://' in name else aws_stack.get_sqs_queue_url(name)
for queue_url in queues['QueueUrls']:
if queue_url == url:
return True
def func(*args):
rule_name = data.get('Name')
client = aws_stack.connect_to_service('events')
targets = client.list_targets_by_rule(Rule=rule_name)['Targets']
if targets:
LOG.debug(
'Notifying %s targets in response to triggered Events rule %s'
% (len(targets), rule_name))
for target in targets:
arn = target.get('Arn')
event = json.loads(target.get('Input') or '{}')
if ':lambda:' in arn:
lambda_api.run_lambda(event=event, context={}, func_arn=arn)
elif ':sns:' in arn:
sns_client = aws_stack.connect_to_service('sns')
sns_client.publish(TopicArn=arn, Message=json.dumps(event))
elif ':sqs:' in arn:
sqs_client = aws_stack.connect_to_service('sqs')
queue_url = aws_stack.get_sqs_queue_url(arn)
sqs_client.send_message(QueueUrl=queue_url,
MessageBody=json.dumps(event))
elif ':states' in arn:
stepfunctions_client = aws_stack.connect_to_service(
'stepfunctions')
stepfunctions_client.start_execution(stateMachineArn=arn,
input=json.dumps(event))
else:
LOG.info('Unsupported Events rule target ARN "%s"' % arn)
def _send_to_dead_letter_queue(source_type, source_arn, dlq_arn, event, error):
if not dlq_arn:
return
LOG.info('Sending failed execution %s to dead letter queue %s' %
(source_arn, dlq_arn))
message = json.dumps(event)
message_attrs = {
'RequestID': {
'DataType': 'String',
'StringValue': str(uuid.uuid4())
},
'ErrorCode': {
'DataType': 'String',
'StringValue': '200'
},
'ErrorMessage': {
'DataType': 'String',
'StringValue': str(error)
}
}
if ':sqs:' in dlq_arn:
queue_url = aws_stack.get_sqs_queue_url(dlq_arn)
sqs_client = aws_stack.connect_to_service('sqs')
sqs_client.send_message(QueueUrl=queue_url,
MessageBody=message,
MessageAttributes=message_attrs)
elif ':sns:' in dlq_arn:
sns_client = aws_stack.connect_to_service('sns')
sns_client.publish(TopicArn=dlq_arn,
Message=message,
MessageAttributes=message_attrs)
else:
LOG.warning('Unsupported dead letter queue type: %s' % dlq_arn)
return dlq_arn
def test_list_stack_resources_returns_queue_urls(self):
cloudformation = aws_stack.connect_to_resource('cloudformation')
template = template_deployer.template_to_json(
load_file(TEST_TEMPLATE_2))
cloudformation.create_stack(StackName=TEST_STACK_NAME_2,
TemplateBody=template)
def check_stack():
stack = get_stack_details(TEST_STACK_NAME_2)
assert stack['StackStatus'] == 'CREATE_COMPLETE'
retry(check_stack, retries=3, sleep=2)
list_stack_summaries = list_stack_resources(TEST_STACK_NAME_2)
queue_urls = get_queue_urls()
topic_arns = get_topic_arns()
stack_queues = [
r for r in list_stack_summaries
if r['ResourceType'] == 'AWS::SQS::Queue'
]
for resource in stack_queues:
url = aws_stack.get_sqs_queue_url(resource['PhysicalResourceId'])
self.assertIn(url, queue_urls)
stack_topics = [
r for r in list_stack_summaries
if r['ResourceType'] == 'AWS::SNS::Topic'
]
for resource in stack_topics:
self.assertIn(resource['PhysicalResourceId'], topic_arns)
def test_queue_handler_deployed(self):
function_name = "sls-test-local-queueHandler"
queue_name = "sls-test-local-CreateQueue"
lambda_client = aws_stack.create_external_boto_client("lambda")
sqs_client = aws_stack.create_external_boto_client("sqs")
resp = lambda_client.list_functions()
function = [
fn for fn in resp["Functions"]
if fn["FunctionName"] == function_name
][0]
self.assertEqual("handler.createQueue", function["Handler"])
resp = lambda_client.list_event_source_mappings(
FunctionName=function_name)
events = resp["EventSourceMappings"]
self.assertEqual(1, len(events))
event_source_arn = events[0]["EventSourceArn"]
self.assertEqual(event_source_arn, aws_stack.sqs_queue_arn(queue_name))
result = sqs_client.get_queue_attributes(
QueueUrl=aws_stack.get_sqs_queue_url(queue_name),
AttributeNames=[
"RedrivePolicy",
],
)
redrive_policy = json.loads(result["Attributes"]["RedrivePolicy"])
self.assertEqual(3, redrive_policy["maxReceiveCount"])
def _send_to_dead_letter_queue(source_type, source_arn, dlq_arn, event, error):
if not dlq_arn:
return
LOG.info("Sending failed execution %s to dead letter queue %s" % (source_arn, dlq_arn))
messages = _prepare_messages_to_dlq(source_arn, event, error)
if ":sqs:" in dlq_arn:
queue_url = aws_stack.get_sqs_queue_url(dlq_arn)
sqs_client = aws_stack.connect_to_service("sqs")
error = None
result_code = None
try:
result = sqs_client.send_message_batch(QueueUrl=queue_url, Entries=messages)
result_code = result.get("ResponseMetadata", {}).get("HTTPStatusCode")
except Exception as e:
error = e
if error or not result_code or result_code >= 400:
msg = "Unable to send message to dead letter queue %s (code %s): %s" % (
queue_url,
result_code,
error,
)
LOG.info(msg)
raise Exception(msg)
elif ":sns:" in dlq_arn:
sns_client = aws_stack.connect_to_service("sns")
for message in messages:
sns_client.publish(
TopicArn=dlq_arn,
Message=message["MessageBody"],
MessageAttributes=message["MessageAttributes"],
)
else:
LOG.warning("Unsupported dead letter queue type: %s" % dlq_arn)
return dlq_arn
def publish_message(topic_arn, req_data, subscription_arn=None):
message = req_data['Message'][0]
sqs_client = aws_stack.connect_to_service('sqs')
for subscriber in SNS_SUBSCRIPTIONS.get(topic_arn, []):
if subscription_arn not in [None, subscriber['SubscriptionArn']]:
continue
filter_policy = json.loads(subscriber.get('FilterPolicy') or '{}')
message_attributes = get_message_attributes(req_data)
if not check_filter_policy(filter_policy, message_attributes):
continue
if subscriber['Protocol'] == 'sqs':
endpoint = subscriber['Endpoint']
if 'sqs_queue_url' in subscriber:
queue_url = subscriber.get('sqs_queue_url')
elif '://' in endpoint:
queue_url = endpoint
else:
queue_name = endpoint.split(':')[5]
queue_url = aws_stack.get_sqs_queue_url(queue_name)
subscriber['sqs_queue_url'] = queue_url
try:
sqs_client.send_message(
QueueUrl=queue_url,
MessageBody=create_sns_message_body(subscriber, req_data),
MessageAttributes=create_sqs_message_attributes(
subscriber, message_attributes))
except Exception as exc:
return make_error(message=str(exc), code=400)
elif subscriber['Protocol'] == 'lambda':
lambda_api.process_sns_notification(subscriber['Endpoint'],
topic_arn,
subscriber['SubscriptionArn'],
message,
message_attributes,
subject=req_data.get(
'Subject', [None])[0])
elif subscriber['Protocol'] in ['http', 'https']:
msg_type = (req_data.get('Type') or ['Notification'])[0]
try:
message_body = create_sns_message_body(subscriber, req_data)
except Exception as exc:
return make_error(message=str(exc), code=400)
requests.post(
subscriber['Endpoint'],
headers={
'Content-Type': 'text/plain',
# AWS headers according to
# https://docs.aws.amazon.com/sns/latest/dg/sns-message-and-json-formats.html#http-header
'x-amz-sns-message-type': msg_type,
'x-amz-sns-topic-arn': subscriber['TopicArn'],
'x-amz-sns-subscription-arn':
subscriber['SubscriptionArn'],
'User-Agent': 'Amazon Simple Notification Service Agent'
},
data=message_body,
verify=False)
else:
LOGGER.warning('Unexpected protocol "%s" for SNS subscription' %
subscriber['Protocol'])
def update_sns(method,
path,
data,
headers,
response=None,
return_forward_info=False):
if return_forward_info:
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(data)
topic_arn = req_data.get('TargetArn') or req_data.get('TopicArn')
if topic_arn:
topic_arn = topic_arn[0]
if topic_arn not in SNS_SUBSCRIPTIONS:
SNS_SUBSCRIPTIONS[topic_arn] = []
if 'Subscribe' in req_data['Action']:
subscription = {
'topic_arn': topic_arn,
'endpoint': req_data['Endpoint'][0],
'protocol': req_data['Protocol'][0]
}
SNS_SUBSCRIPTIONS[topic_arn].append(subscription)
elif 'Publish' in req_data['Action']:
message = req_data['Message'][0]
sqs_client = aws_stack.connect_to_service('sqs')
for subscriber in SNS_SUBSCRIPTIONS[topic_arn]:
if subscriber['protocol'] == 'sqs':
queue_name = subscriber['endpoint'].split(':')[5]
queue_url = subscriber.get('sqs_queue_url')
if not queue_url:
queue_url = aws_stack.get_sqs_queue_url(queue_name)
subscriber['sqs_queue_url'] = queue_url
sqs_client.send_message(QueueUrl=queue_url,
MessageBody=message)
elif subscriber['protocol'] == 'lambda':
lambda_api.process_sns_notification(
subscriber['endpoint'],
topic_arn,
message,
subject=req_data.get('Subject'))
elif subscriber['protocol'] == 'http':
requests.post(subscriber['endpoint'],
headers={'Content-Type': 'text/plain'},
data=json.dumps({
'Type': 'Notification',
'Message': message,
}))
else:
LOGGER.warning(
'Unexpected protocol "%s" for SNS subscription' %
subscriber['protocol'])
# return response here because we do not want the request to be forwarded to SNS
response = Response()
response._content = """<PublishResponse xmlns="http://sns.amazonaws.com/doc/2010-03-31/">
<PublishResult><MessageId>n/a</MessageId></PublishResult>
<ResponseMetadata><RequestId>n/a</RequestId></ResponseMetadata></PublishResponse>"""
response.status_code = 200
return response
return True
def sqs_error_to_dead_letter_queue(queue_arn, event, error):
client = aws_stack.connect_to_service('sqs')
queue_url = aws_stack.get_sqs_queue_url(queue_arn)
attrs = client.get_queue_attributes(QueueUrl=queue_url, AttributeNames=['RedrivePolicy'])
attrs = attrs.get('Attributes', {})
policy = json.loads(attrs.get('RedrivePolicy') or '{}')
target_arn = policy.get('deadLetterTargetArn')
if not target_arn:
return
return _send_to_dead_letter_queue('SQS', queue_arn, target_arn, event, error)
def get_physical_resource_id(self, attribute=None, **kwargs):
queue_url = None
props = self.props
try:
queue_url = aws_stack.get_sqs_queue_url(props.get('QueueName'))
except Exception as e:
if 'NonExistentQueue' in str(e):
raise DependencyNotYetSatisfied(resource_ids=self.resource_id, message='Unable to get queue: %s' % e)
if attribute == 'Arn':
return aws_stack.sqs_queue_arn(props.get('QueueName'))
return queue_url
def receive_messages():
sqs = connect_to_service('sqs')
queue_url = get_sqs_queue_url(QUEUE_NAME)
messages = []
start = datetime.now()
while len(messages) < NUM_MESSAGES:
result = sqs.receive_message(QueueUrl=queue_url)
messages.extend(result.get('Messages') or [])
print_duration(start, len(messages), action='Received')
print('All %s messages received' % len(messages))
def receive_messages():
sqs = create_external_boto_client("sqs")
queue_url = get_sqs_queue_url(QUEUE_NAME)
messages = []
start = datetime.now()
while len(messages) < NUM_MESSAGES:
result = sqs.receive_message(QueueUrl=queue_url)
messages.extend(result.get("Messages") or [])
print_duration(start, len(messages), action="Received")
print("All %s messages received" % len(messages))
def test_apply_template(self):
cloudformation = aws_stack.connect_to_resource('cloudformation')
s3 = aws_stack.connect_to_service('s3')
sns = aws_stack.connect_to_service('sns')
apigateway = aws_stack.connect_to_service('apigateway')
template = template_deployer.template_to_json(load_file(TEST_TEMPLATE_1))
# deploy template
stack_name = 'stack-%s' % short_uid()
cloudformation.create_stack(StackName=stack_name, TemplateBody=template)
# wait for deployment to finish
def check_stack():
stack = get_stack_details(stack_name)
self.assertEqual(stack['StackStatus'], 'CREATE_COMPLETE')
retry(check_stack, retries=3, sleep=2)
# assert that bucket has been created
assert bucket_exists('cf-test-bucket-1')
# assert that queue has been created
assert queue_exists('cf-test-queue-1')
# assert that topic has been created
topic_arn = topic_exists('%s-test-topic-1-1' % stack_name)
assert topic_arn
# assert that stream has been created
assert stream_exists('cf-test-stream-1')
# assert that queue has been created
resource = describe_stack_resource(stack_name, 'SQSQueueNoNameProperty')
assert queue_exists(resource['PhysicalResourceId'])
# assert that tags have been created
tags = s3.get_bucket_tagging(Bucket='cf-test-bucket-1')['TagSet']
self.assertEqual(tags, [{'Key': 'foobar', 'Value': aws_stack.get_sqs_queue_url('cf-test-queue-1')}])
tags = sns.list_tags_for_resource(ResourceArn=topic_arn)['Tags']
self.assertEqual(tags, [
{'Key': 'foo', 'Value': 'cf-test-bucket-1'},
{'Key': 'bar', 'Value': aws_stack.s3_bucket_arn('cf-test-bucket-1')}
])
# assert that subscriptions have been created
subs = sns.list_subscriptions()['Subscriptions']
subs = [s for s in subs if (':%s:cf-test-queue-1' % TEST_AWS_ACCOUNT_ID) in s['Endpoint']]
self.assertEqual(len(subs), 1)
self.assertIn(':%s:%s-test-topic-1-1' % (TEST_AWS_ACCOUNT_ID, stack_name), subs[0]['TopicArn'])
# assert that Gateway responses have been created
test_api_name = 'test-api'
api = [a for a in apigateway.get_rest_apis()['items'] if a['name'] == test_api_name][0]
responses = apigateway.get_gateway_responses(restApiId=api['id'])['items']
self.assertEqual(len(responses), 2)
types = [r['responseType'] for r in responses]
self.assertEqual(set(types), set(['UNAUTHORIZED', 'DEFAULT_5XX']))
def publish_message(topic_arn, req_data):
message = req_data['Message'][0]
sqs_client = aws_stack.connect_to_service('sqs')
for subscriber in SNS_SUBSCRIPTIONS.get(topic_arn, []):
filter_policy = json.loads(subscriber.get('FilterPolicy', '{}'))
message_attributes = get_message_attributes(req_data)
if not check_filter_policy(filter_policy, message_attributes):
continue
if subscriber['Protocol'] == 'sqs':
endpoint = subscriber['Endpoint']
if 'sqs_queue_url' in subscriber:
queue_url = subscriber.get('sqs_queue_url')
elif '://' in endpoint:
queue_url = endpoint
else:
queue_name = endpoint.split(':')[5]
queue_url = aws_stack.get_sqs_queue_url(queue_name)
subscriber['sqs_queue_url'] = queue_url
try:
sqs_client.send_message(
QueueUrl=queue_url,
MessageBody=create_sns_message_body(subscriber, req_data),
MessageAttributes=create_sqs_message_attributes(subscriber, message_attributes)
)
except Exception as exc:
return make_error(message=str(exc), code=400)
elif subscriber['Protocol'] == 'lambda':
lambda_api.process_sns_notification(
subscriber['Endpoint'],
topic_arn,
subscriber['SubscriptionArn'],
message,
message_attributes,
subject=req_data.get('Subject', [None])[0]
)
elif subscriber['Protocol'] in ['http', 'https']:
try:
message_body = create_sns_message_body(subscriber, req_data)
except Exception as exc:
return make_error(message=str(exc), code=400)
requests.post(
subscriber['Endpoint'],
headers={
'Content-Type': 'text/plain',
'x-amz-sns-message-type': 'Notification'
},
data=message_body
)
else:
LOGGER.warning('Unexpected protocol "%s" for SNS subscription' % subscriber['Protocol'])
def test_apply_template(self):
cloudformation = aws_stack.connect_to_resource('cloudformation')
s3 = aws_stack.connect_to_service('s3')
sns = aws_stack.connect_to_service('sns')
template = template_deployer.template_to_json(
load_file(TEST_TEMPLATE_1))
# deploy template
cloudformation.create_stack(StackName=TEST_STACK_NAME,
TemplateBody=template)
# wait for deployment to finish
def check_stack():
stack = get_stack_details(TEST_STACK_NAME)
self.assertEqual(stack['StackStatus'], 'CREATE_COMPLETE')
retry(check_stack, retries=3, sleep=2)
# assert that bucket has been created
assert bucket_exists('cf-test-bucket-1')
# assert that queue has been created
assert queue_exists('cf-test-queue-1')
# assert that topic has been created
assert topic_exists('cf-test-topic-1-1')
# assert that stream has been created
assert stream_exists('cf-test-stream-1')
# assert that queue has been created
resource = describe_stack_resource(TEST_STACK_NAME,
'SQSQueueNoNameProperty')
assert queue_exists(resource['PhysicalResourceId'])
# assert that topic tags have been created
tags = s3.get_bucket_tagging(Bucket='cf-test-bucket-1')['TagSet']
self.assertEqual(
tags, [{
'Key': 'foobar',
'Value': aws_stack.get_sqs_queue_url('cf-test-queue-1')
}])
# assert that subscriptions have been created
subs = sns.list_subscriptions()['Subscriptions']
subs = [
s for s in subs
if (':%s:cf-test-queue-1' % TEST_AWS_ACCOUNT_ID) in s['Endpoint']
]
self.assertEqual(len(subs), 1)
self.assertIn(':%s:cf-test-topic-1-1' % TEST_AWS_ACCOUNT_ID,
subs[0]['TopicArn'])
def _send_to_dead_letter_queue(source_type, source_arn, dlq_arn, event, error):
if not dlq_arn:
return
LOG.info('Sending failed execution %s to dead letter queue %s' %
(source_arn, dlq_arn))
message = json.dumps(event)
message_attrs = {
'RequestID': {
'DataType': 'String',
'StringValue': str(uuid.uuid4())
},
'ErrorCode': {
'DataType': 'String',
'StringValue': '200'
},
'ErrorMessage': {
'DataType': 'String',
'StringValue': str(error)
}
}
if ':sqs:' in dlq_arn:
queue_url = aws_stack.get_sqs_queue_url(dlq_arn)
sqs_client = aws_stack.connect_to_service('sqs')
error = None
result_code = None
try:
result = sqs_client.send_message(QueueUrl=queue_url,
MessageBody=message,
MessageAttributes=message_attrs)
result_code = result.get('ResponseMetadata',
{}).get('HTTPStatusCode')
except Exception as e:
error = e
if error or not result_code or result_code >= 400:
msg = 'Unable to send message to dead letter queue %s (code %s): %s' % (
queue_url, result_code, error)
LOG.info(msg)
raise Exception(msg)
elif ':sns:' in dlq_arn:
sns_client = aws_stack.connect_to_service('sns')
sns_client.publish(TopicArn=dlq_arn,
Message=message,
MessageAttributes=message_attrs)
else:
LOG.warning('Unsupported dead letter queue type: %s' % dlq_arn)
return dlq_arn
def sqs_error_to_dead_letter_queue(queue_arn: str, event: Dict, error):
client = aws_stack.connect_to_service("sqs")
queue_url = aws_stack.get_sqs_queue_url(queue_arn)
attrs = client.get_queue_attributes(QueueUrl=queue_url,
AttributeNames=["RedrivePolicy"])
attrs = attrs.get("Attributes", {})
try:
policy = json.loads(attrs.get("RedrivePolicy") or "{}")
except JSONDecodeError:
LOG.warning("Parsing RedrivePolicy {} failed, Queue: {}".format(
attrs.get("RedrivePolicy"), queue_arn))
return
target_arn = policy.get("deadLetterTargetArn")
if not target_arn:
return
return _send_to_dead_letter_queue("SQS", queue_arn, target_arn, event,
error)
def sqs_error_to_dead_letter_queue(queue_arn, event, error):
client = aws_stack.connect_to_service('sqs')
queue_url = aws_stack.get_sqs_queue_url(queue_arn)
attrs = client.get_queue_attributes(QueueUrl=queue_url,
AttributeNames=['RedrivePolicy'])
attrs = attrs.get('Attributes', {})
try:
policy = json.loads(attrs.get('RedrivePolicy') or '{}')
except JSONDecodeError:
LOG.warning('Parsing RedrivePolicy {} failed, Queue: {}'.format(
attrs.get('RedrivePolicy'), queue_arn))
return
target_arn = policy.get('deadLetterTargetArn')
if not target_arn:
return
return _send_to_dead_letter_queue('SQS', queue_arn, target_arn, event,
error)
def update_sns(method, path, data, headers, response=None, return_forward_info=False):
if return_forward_info:
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(data)
topic_arn = req_data.get('TopicArn')
if topic_arn:
topic_arn = topic_arn[0]
if topic_arn not in SNS_SUBSCRIPTIONS:
SNS_SUBSCRIPTIONS[topic_arn] = []
if 'Subscribe' in req_data['Action']:
subscription = {
'topic_arn': topic_arn,
'endpoint': req_data['Endpoint'][0],
'protocol': req_data['Protocol'][0]
}
SNS_SUBSCRIPTIONS[topic_arn].append(subscription)
elif 'Publish' in req_data['Action']:
message = req_data['Message'][0]
sqs_client = aws_stack.connect_to_service('sqs', client=True)
for subscriber in SNS_SUBSCRIPTIONS[topic_arn]:
if subscriber['protocol'] == 'sqs':
queue_name = subscriber['endpoint'].split(':')[5]
queue_url = subscriber.get('sqs_queue_url')
if not queue_url:
queue_url = aws_stack.get_sqs_queue_url(queue_name)
subscriber['sqs_queue_url'] = queue_url
sqs_client.send_message(QueueUrl=queue_url, MessageBody=message)
else:
LOGGER.warning('Unexpected protocol "%s" for SNS subscription' % subscriber['protocol'])
# return response here because we do not want the request to be forwarded to SNS
response = Response()
response._content = """<PublishResponse xmlns="http://sns.amazonaws.com/doc/2010-03-31/">
<PublishResult><MessageId>n/a</MessageId></PublishResult>
<ResponseMetadata><RequestId>n/a</RequestId></ResponseMetadata></PublishResponse>"""
response.status_code = 200
return response
return True
def publish_message(topic_arn, req_data, subscription_arn=None):
message = req_data['Message'][0]
sqs_client = aws_stack.connect_to_service('sqs')
LOG.debug('Publishing message to TopicArn: %s | Message: %s' % (topic_arn, message))
subscriptions = SNS_SUBSCRIPTIONS.get(topic_arn, [])
for subscriber in list(subscriptions):
if subscription_arn not in [None, subscriber['SubscriptionArn']]:
continue
filter_policy = json.loads(subscriber.get('FilterPolicy') or '{}')
message_attributes = get_message_attributes(req_data)
if not check_filter_policy(filter_policy, message_attributes):
continue
if subscriber['Protocol'] == 'sqs':
queue_url = None
try:
endpoint = subscriber['Endpoint']
if 'sqs_queue_url' in subscriber:
queue_url = subscriber.get('sqs_queue_url')
elif '://' in endpoint:
queue_url = endpoint
else:
queue_name = endpoint.split(':')[5]
queue_url = aws_stack.get_sqs_queue_url(queue_name)
subscriber['sqs_queue_url'] = queue_url
sqs_client.send_message(
QueueUrl=queue_url,
MessageBody=create_sns_message_body(subscriber, req_data),
MessageAttributes=create_sqs_message_attributes(subscriber, message_attributes)
)
except Exception as exc:
sns_error_to_dead_letter_queue(subscriber['SubscriptionArn'], req_data, str(exc))
if 'NonExistentQueue' in str(exc):
LOG.info('Removing non-existent queue "%s" subscribed to topic "%s"' % (queue_url, topic_arn))
subscriptions.remove(subscriber)
elif subscriber['Protocol'] == 'lambda':
try:
external_url = external_service_url('sns')
unsubscribe_url = '%s/?Action=Unsubscribe&SubscriptionArn=%s' % (external_url,
subscriber['SubscriptionArn'])
response = lambda_api.process_sns_notification(
subscriber['Endpoint'],
topic_arn,
subscriber['SubscriptionArn'],
message,
message_attributes,
unsubscribe_url,
subject=req_data.get('Subject', [None])[0]
)
if isinstance(response, FlaskResponse):
response.raise_for_status()
except Exception as exc:
LOG.warning('Unable to run Lambda function on SNS message: %s %s' % (exc, traceback.format_exc()))
sns_error_to_dead_letter_queue(subscriber['SubscriptionArn'], req_data, str(exc))
elif subscriber['Protocol'] in ['http', 'https']:
msg_type = (req_data.get('Type') or ['Notification'])[0]
try:
message_body = create_sns_message_body(subscriber, req_data)
except Exception:
continue
try:
response = requests.post(
subscriber['Endpoint'],
headers={
'Content-Type': 'text/plain',
# AWS headers according to
# https://docs.aws.amazon.com/sns/latest/dg/sns-message-and-json-formats.html#http-header
'x-amz-sns-message-type': msg_type,
'x-amz-sns-topic-arn': subscriber['TopicArn'],
'x-amz-sns-subscription-arn': subscriber['SubscriptionArn'],
'User-Agent': 'Amazon Simple Notification Service Agent'
},
data=message_body,
verify=False
)
response.raise_for_status()
except Exception as exc:
sns_error_to_dead_letter_queue(subscriber['SubscriptionArn'], req_data, str(exc))
else:
LOG.warning('Unexpected protocol "%s" for SNS subscription' % subscriber['Protocol'])
def SQS_Queue_physical_resource_id(self):
result = SQS_Queue_physical_resource_id_orig.fget(self)
if '://' not in result:
# convert ID to queue URL
return aws_stack.get_sqs_queue_url(result)
return result
async def message_to_subscriber(
message_id,
message,
topic_arn,
req_data,
headers,
subscription_arn,
skip_checks,
sns_backend,
subscriber,
subscriptions,
):
if subscription_arn not in [None, subscriber["SubscriptionArn"]]:
return
filter_policy = json.loads(subscriber.get("FilterPolicy") or "{}")
message_attributes = get_message_attributes(req_data)
if not skip_checks and not check_filter_policy(filter_policy,
message_attributes):
LOG.info("SNS filter policy %s does not match attributes %s" %
(filter_policy, message_attributes))
return
if subscriber["Protocol"] == "sms":
event = {
"topic_arn": topic_arn,
"endpoint": subscriber["Endpoint"],
"message_content": req_data["Message"][0],
}
sns_backend.sms_messages.append(event)
LOG.info(
"Delivering SMS message to %s: %s",
subscriber["Endpoint"],
req_data["Message"][0],
)
return
elif subscriber["Protocol"] == "sqs":
queue_url = None
try:
endpoint = subscriber["Endpoint"]
if "sqs_queue_url" in subscriber:
queue_url = subscriber.get("sqs_queue_url")
elif "://" in endpoint:
queue_url = endpoint
else:
queue_name = endpoint.split(":")[5]
queue_url = aws_stack.get_sqs_queue_url(queue_name)
subscriber["sqs_queue_url"] = queue_url
message_group_id = (req_data.get("MessageGroupId")[0]
if req_data.get("MessageGroupId") else "")
sqs_client = aws_stack.connect_to_service("sqs")
# TODO remove this kwargs if we stop using ElasticMQ entirely
kwargs = {
"MessageGroupId": message_group_id
} if SQS_BACKEND_IMPL == "moto" else {}
sqs_client.send_message(
QueueUrl=queue_url,
MessageBody=create_sns_message_body(subscriber, req_data,
message_id),
MessageAttributes=create_sqs_message_attributes(
subscriber, message_attributes),
MessageSystemAttributes=create_sqs_system_attributes(headers),
**kwargs,
)
except Exception as exc:
LOG.info("Unable to forward SNS message to SQS: %s %s" %
(exc, traceback.format_exc()))
sns_error_to_dead_letter_queue(subscriber["SubscriptionArn"],
req_data, str(exc))
if "NonExistentQueue" in str(exc):
LOG.info(
'Removing non-existent queue "%s" subscribed to topic "%s"'
% (queue_url, topic_arn))
subscriptions.remove(subscriber)
return
elif subscriber["Protocol"] == "lambda":
try:
external_url = external_service_url("sns")
unsubscribe_url = "%s/?Action=Unsubscribe&SubscriptionArn=%s" % (
external_url,
subscriber["SubscriptionArn"],
)
response = lambda_api.process_sns_notification(
subscriber["Endpoint"],
topic_arn,
subscriber["SubscriptionArn"],
message,
message_id,
message_attributes,
unsubscribe_url,
subject=req_data.get("Subject", [None])[0],
)
if isinstance(response, Response):
response.raise_for_status()
elif isinstance(response, FlaskResponse):
if response.status_code >= 400:
raise Exception("Error response (code %s): %s" %
(response.status_code, response.data))
except Exception as exc:
LOG.info("Unable to run Lambda function on SNS message: %s %s" %
(exc, traceback.format_exc()))
sns_error_to_dead_letter_queue(subscriber["SubscriptionArn"],
req_data, str(exc))
return
elif subscriber["Protocol"] in ["http", "https"]:
msg_type = (req_data.get("Type") or ["Notification"])[0]
try:
message_body = create_sns_message_body(subscriber, req_data,
message_id)
except Exception:
return
try:
response = requests.post(
subscriber["Endpoint"],
headers={
"Content-Type": "text/plain",
# AWS headers according to
# https://docs.aws.amazon.com/sns/latest/dg/sns-message-and-json-formats.html#http-header
"x-amz-sns-message-type": msg_type,
"x-amz-sns-topic-arn": subscriber["TopicArn"],
"x-amz-sns-subscription-arn":
subscriber["SubscriptionArn"],
"User-Agent": "Amazon Simple Notification Service Agent",
},
data=message_body,
verify=False,
)
response.raise_for_status()
except Exception as exc:
LOG.info(
"Received error on sending SNS message, putting to DLQ (if configured): %s"
% exc)
sns_error_to_dead_letter_queue(subscriber["SubscriptionArn"],
req_data, str(exc))
return
elif subscriber["Protocol"] == "application":
try:
sns_client = aws_stack.connect_to_service("sns")
sns_client.publish(TargetArn=subscriber["Endpoint"],
Message=message)
except Exception as exc:
LOG.warning(
"Unable to forward SNS message to SNS platform app: %s %s" %
(exc, traceback.format_exc()))
sns_error_to_dead_letter_queue(subscriber["SubscriptionArn"],
req_data, str(exc))
return
elif subscriber["Protocol"] == "email":
ses_client = aws_stack.connect_to_service("ses")
if subscriber.get("Endpoint"):
ses_client.verify_email_address(
EmailAddress=subscriber.get("Endpoint"))
ses_client.verify_email_address(
EmailAddress="*****@*****.**")
ses_client.send_email(
Source="*****@*****.**",
Message={
"Body": {
"Text": {
"Data": message
}
},
"Subject": {
"Data": "SNS-Subscriber-Endpoint"
},
},
Destination={"ToAddresses": [subscriber.get("Endpoint")]},
)
else:
LOG.warning('Unexpected protocol "%s" for SNS subscription' %
subscriber["Protocol"])
async def message_to_subscriber(
message_id,
message,
topic_arn,
req_data,
headers,
subscription_arn,
skip_checks,
sns_backend,
subscriber,
subscriptions,
message_attributes,
):
if subscription_arn not in [None, subscriber["SubscriptionArn"]]:
return
filter_policy = json.loads(subscriber.get("FilterPolicy") or "{}")
if not message_attributes:
message_attributes = get_message_attributes(req_data)
if not skip_checks and not check_filter_policy(filter_policy,
message_attributes):
LOG.info("SNS filter policy %s does not match attributes %s",
filter_policy, message_attributes)
return
if subscriber["Protocol"] == "sms":
event = {
"topic_arn": topic_arn,
"endpoint": subscriber["Endpoint"],
"message_content": req_data["Message"][0],
}
sns_backend.sms_messages.append(event)
LOG.info(
"Delivering SMS message to %s: %s",
subscriber["Endpoint"],
req_data["Message"][0],
)
# MOCK DATA
delivery = {
"phoneCarrier": "Mock Carrier",
"mnc": 270,
"priceInUSD": 0.00645,
"smsType": "Transactional",
"mcc": 310,
"providerResponse": "Message has been accepted by phone carrier",
"dwellTimeMsUntilDeviceAck": 200,
}
store_delivery_log(subscriber, True, message, message_id, delivery)
return
elif subscriber["Protocol"] == "sqs":
queue_url = None
try:
endpoint = subscriber["Endpoint"]
if "sqs_queue_url" in subscriber:
queue_url = subscriber.get("sqs_queue_url")
elif "://" in endpoint:
queue_url = endpoint
else:
queue_name = endpoint.split(":")[5]
queue_url = aws_stack.get_sqs_queue_url(queue_name)
subscriber["sqs_queue_url"] = queue_url
message_group_id = (req_data.get("MessageGroupId")
if req_data.get("MessageGroupId") else "")
if isinstance(message_group_id, list):
message_group_id = message_group_id[0]
message_deduplication_id = (
req_data.get("MessageDeduplicationId")[0]
if req_data.get("MessageDeduplicationId") else "")
sqs_client = aws_stack.connect_to_service("sqs")
kwargs = {}
if message_group_id:
kwargs["MessageGroupId"] = message_group_id
if message_deduplication_id:
kwargs["MessageDeduplicationId"] = message_deduplication_id
sqs_client.send_message(
QueueUrl=queue_url,
MessageBody=create_sns_message_body(subscriber, req_data,
message_id),
MessageAttributes=create_sqs_message_attributes(
subscriber, message_attributes),
MessageSystemAttributes=create_sqs_system_attributes(headers),
**kwargs,
)
store_delivery_log(subscriber, True, message, message_id)
except Exception as exc:
LOG.info("Unable to forward SNS message to SQS: %s %s", exc,
traceback.format_exc())
store_delivery_log(subscriber, False, message, message_id)
sns_error_to_dead_letter_queue(subscriber["SubscriptionArn"],
req_data, str(exc))
if "NonExistentQueue" in str(exc):
LOG.info(
'Removing non-existent queue "%s" subscribed to topic "%s"',
queue_url,
topic_arn,
)
subscriptions.remove(subscriber)
return
elif subscriber["Protocol"] == "lambda":
try:
external_url = external_service_url("sns")
unsubscribe_url = "%s/?Action=Unsubscribe&SubscriptionArn=%s" % (
external_url,
subscriber["SubscriptionArn"],
)
response = lambda_api.process_sns_notification(
subscriber["Endpoint"],
topic_arn,
subscriber["SubscriptionArn"],
message,
message_id,
message_attributes,
unsubscribe_url,
subject=req_data.get("Subject", [None])[0],
)
if response is not None:
delivery = {
"statusCode": response.status_code,
"providerResponse": response.get_data(),
}
store_delivery_log(subscriber, True, message, message_id,
delivery)
if isinstance(response, Response):
response.raise_for_status()
elif isinstance(response, FlaskResponse):
if response.status_code >= 400:
raise Exception("Error response (code %s): %s" %
(response.status_code, response.data))
except Exception as exc:
LOG.info("Unable to run Lambda function on SNS message: %s %s",
exc, traceback.format_exc())
store_delivery_log(subscriber, False, message, message_id)
sns_error_to_dead_letter_queue(subscriber["SubscriptionArn"],
req_data, str(exc))
return
elif subscriber["Protocol"] in ["http", "https"]:
msg_type = (req_data.get("Type") or ["Notification"])[0]
try:
message_body = create_sns_message_body(subscriber, req_data,
message_id)
except Exception:
return
try:
response = requests.post(
subscriber["Endpoint"],
headers={
"Content-Type": "text/plain",
# AWS headers according to
# https://docs.aws.amazon.com/sns/latest/dg/sns-message-and-json-formats.html#http-header
"x-amz-sns-message-type": msg_type,
"x-amz-sns-topic-arn": subscriber["TopicArn"],
"x-amz-sns-subscription-arn":
subscriber["SubscriptionArn"],
"User-Agent": "Amazon Simple Notification Service Agent",
},
data=message_body,
verify=False,
)
delivery = {
"statusCode": response.status_code,
"providerResponse": response.content.decode("utf-8"),
}
store_delivery_log(subscriber, True, message, message_id, delivery)
response.raise_for_status()
except Exception as exc:
LOG.info(
"Received error on sending SNS message, putting to DLQ (if configured): %s",
exc)
store_delivery_log(subscriber, False, message, message_id)
sns_error_to_dead_letter_queue(subscriber["SubscriptionArn"],
req_data, str(exc))
return
elif subscriber["Protocol"] == "application":
try:
sns_client = aws_stack.connect_to_service("sns")
sns_client.publish(TargetArn=subscriber["Endpoint"],
Message=message)
store_delivery_log(subscriber, True, message, message_id)
except Exception as exc:
LOG.warning(
"Unable to forward SNS message to SNS platform app: %s %s",
exc,
traceback.format_exc(),
)
store_delivery_log(subscriber, False, message, message_id)
sns_error_to_dead_letter_queue(subscriber["SubscriptionArn"],
req_data, str(exc))
return
elif subscriber["Protocol"] in ["email", "email-json"]:
ses_client = aws_stack.connect_to_service("ses")
if subscriber.get("Endpoint"):
ses_client.verify_email_address(
EmailAddress=subscriber.get("Endpoint"))
ses_client.verify_email_address(
EmailAddress="*****@*****.**")
ses_client.send_email(
Source="*****@*****.**",
Message={
"Body": {
"Text": {
"Data":
create_sns_message_body(subscriber=subscriber,
req_data=req_data,
message_id=message_id) if
subscriber["Protocol"] == "email-json" else message
}
},
"Subject": {
"Data": "SNS-Subscriber-Endpoint"
},
},
Destination={"ToAddresses": [subscriber.get("Endpoint")]},
)
store_delivery_log(subscriber, True, message, message_id)
else:
LOG.warning('Unexpected protocol "%s" for SNS subscription',
subscriber["Protocol"])
def send_event_to_sqs(event, arn):
region = arn.split(':')[3]
queue_url = aws_stack.get_sqs_queue_url(arn)
sqs_client = aws_stack.connect_to_service('sqs', region_name=region)
sqs_client.send_message(QueueUrl=queue_url, MessageBody=json.dumps(event))
def message_to_subscribers(message_id,
message,
topic_arn,
req_data,
subscription_arn=None,
skip_checks=False):
subscriptions = SNS_SUBSCRIPTIONS.get(topic_arn, [])
for subscriber in list(subscriptions):
if subscription_arn not in [None, subscriber['SubscriptionArn']]:
continue
filter_policy = json.loads(subscriber.get('FilterPolicy') or '{}')
message_attributes = get_message_attributes(req_data)
if not skip_checks and not check_filter_policy(filter_policy,
message_attributes):
LOG.info('SNS filter policy %s does not match attributes %s' %
(filter_policy, message_attributes))
continue
if subscriber['Protocol'] == 'sms':
event = {
'topic_arn': topic_arn,
'endpoint': subscriber['Endpoint'],
'message_content': req_data['Message'][0]
}
SMS_MESSAGES.append(event)
LOG.info('Delivering SMS message to %s: %s',
subscriber['Endpoint'], req_data['Message'][0])
elif subscriber['Protocol'] == 'sqs':
queue_url = None
try:
endpoint = subscriber['Endpoint']
if 'sqs_queue_url' in subscriber:
queue_url = subscriber.get('sqs_queue_url')
elif '://' in endpoint:
queue_url = endpoint
else:
queue_name = endpoint.split(':')[5]
queue_url = aws_stack.get_sqs_queue_url(queue_name)
subscriber['sqs_queue_url'] = queue_url
sqs_client = aws_stack.connect_to_service('sqs')
sqs_client.send_message(
QueueUrl=queue_url,
MessageBody=create_sns_message_body(
subscriber, req_data, message_id),
MessageAttributes=create_sqs_message_attributes(
subscriber, message_attributes))
except Exception as exc:
LOG.warning('Unable to forward SNS message to SQS: %s %s' %
(exc, traceback.format_exc()))
sns_error_to_dead_letter_queue(subscriber['SubscriptionArn'],
req_data, str(exc))
if 'NonExistentQueue' in str(exc):
LOG.info(
'Removing non-existent queue "%s" subscribed to topic "%s"'
% (queue_url, topic_arn))
subscriptions.remove(subscriber)
elif subscriber['Protocol'] == 'lambda':
try:
external_url = external_service_url('sns')
unsubscribe_url = '%s/?Action=Unsubscribe&SubscriptionArn=%s' % (
external_url, subscriber['SubscriptionArn'])
response = lambda_api.process_sns_notification(
subscriber['Endpoint'],
topic_arn,
subscriber['SubscriptionArn'],
message,
message_id,
message_attributes,
unsubscribe_url,
subject=req_data.get('Subject', [None])[0])
if isinstance(response, Response):
response.raise_for_status()
elif isinstance(response, FlaskResponse):
if response.status_code >= 400:
raise Exception('Error response (code %s): %s' %
(response.status_code, response.data))
except Exception as exc:
LOG.warning(
'Unable to run Lambda function on SNS message: %s %s' %
(exc, traceback.format_exc()))
sns_error_to_dead_letter_queue(subscriber['SubscriptionArn'],
req_data, str(exc))
elif subscriber['Protocol'] in ['http', 'https']:
msg_type = (req_data.get('Type') or ['Notification'])[0]
try:
message_body = create_sns_message_body(subscriber, req_data,
message_id)
except Exception:
continue
try:
response = requests.post(
subscriber['Endpoint'],
headers={
'Content-Type': 'text/plain',
# AWS headers according to
# https://docs.aws.amazon.com/sns/latest/dg/sns-message-and-json-formats.html#http-header
'x-amz-sns-message-type': msg_type,
'x-amz-sns-topic-arn': subscriber['TopicArn'],
'x-amz-sns-subscription-arn':
subscriber['SubscriptionArn'],
'User-Agent':
'Amazon Simple Notification Service Agent'
},
data=message_body,
verify=False)
response.raise_for_status()
except Exception as exc:
LOG.info(
'Received error on sending SNS message, putting to DLQ (if configured): %s'
% exc)
sns_error_to_dead_letter_queue(subscriber['SubscriptionArn'],
req_data, str(exc))
elif subscriber['Protocol'] == 'application':
try:
sns_client = aws_stack.connect_to_service('sns')
sns_client.publish(TargetArn=subscriber['Endpoint'],
Message=message)
except Exception as exc:
LOG.warning(
'Unable to forward SNS message to SNS platform app: %s %s'
% (exc, traceback.format_exc()))
sns_error_to_dead_letter_queue(subscriber['SubscriptionArn'],
req_data, str(exc))
else:
LOG.warning('Unexpected protocol "%s" for SNS subscription' %
subscriber['Protocol'])
def send_event_to_sqs(event, arn):
queue_url = aws_stack.get_sqs_queue_url(arn)
sqs_client = aws_stack.connect_to_service('sqs')
sqs_client.send_message(QueueUrl=queue_url, MessageBody=json.dumps(event))
def test_create_delete_stack(self):
cloudformation = aws_stack.connect_to_resource('cloudformation')
cf_client = aws_stack.connect_to_service('cloudformation')
s3 = aws_stack.connect_to_service('s3')
sns = aws_stack.connect_to_service('sns')
sqs = aws_stack.connect_to_service('sqs')
apigateway = aws_stack.connect_to_service('apigateway')
template = template_deployer.template_to_json(
load_file(TEST_TEMPLATE_1))
# deploy template
stack_name = 'stack-%s' % short_uid()
cloudformation.create_stack(StackName=stack_name,
TemplateBody=template)
# wait for deployment to finish
def check_stack():
stack = get_stack_details(stack_name)
self.assertEqual(stack['StackStatus'], 'CREATE_COMPLETE')
retry(check_stack, retries=3, sleep=2)
# assert that resources have been created
assert bucket_exists('cf-test-bucket-1')
queue_url = queue_exists('cf-test-queue-1')
assert queue_url
topic_arn = topic_exists('%s-test-topic-1-1' % stack_name)
assert topic_arn
assert stream_exists('cf-test-stream-1')
resource = describe_stack_resource(stack_name,
'SQSQueueNoNameProperty')
assert queue_exists(resource['PhysicalResourceId'])
assert ssm_param_exists('cf-test-param-1')
# assert that tags have been created
tags = s3.get_bucket_tagging(Bucket='cf-test-bucket-1')['TagSet']
self.assertEqual(
tags, [{
'Key': 'foobar',
'Value': aws_stack.get_sqs_queue_url('cf-test-queue-1')
}])
tags = sns.list_tags_for_resource(ResourceArn=topic_arn)['Tags']
self.assertEqual(
tags, [{
'Key': 'foo',
'Value': 'cf-test-bucket-1'
}, {
'Key': 'bar',
'Value': aws_stack.s3_bucket_arn('cf-test-bucket-1')
}])
queue_tags = sqs.list_queue_tags(QueueUrl=queue_url)
self.assertIn('Tags', queue_tags)
self.assertEqual(queue_tags['Tags'], {
'key1': 'value1',
'key2': 'value2'
})
# assert that bucket notifications have been created
notifs = s3.get_bucket_notification_configuration(
Bucket='cf-test-bucket-1')
self.assertIn('QueueConfigurations', notifs)
self.assertIn('LambdaFunctionConfigurations', notifs)
self.assertEqual(notifs['QueueConfigurations'][0]['QueueArn'],
'aws:arn:sqs:test:testqueue')
self.assertEqual(notifs['QueueConfigurations'][0]['Events'],
['s3:ObjectDeleted:*'])
self.assertEqual(
notifs['LambdaFunctionConfigurations'][0]['LambdaFunctionArn'],
'aws:arn:lambda:test:testfunc')
self.assertEqual(notifs['LambdaFunctionConfigurations'][0]['Events'],
['s3:ObjectCreated:*'])
# assert that subscriptions have been created
subs = sns.list_subscriptions()['Subscriptions']
subs = [
s for s in subs
if (':%s:cf-test-queue-1' % TEST_AWS_ACCOUNT_ID) in s['Endpoint']
]
self.assertEqual(len(subs), 1)
self.assertIn(
':%s:%s-test-topic-1-1' % (TEST_AWS_ACCOUNT_ID, stack_name),
subs[0]['TopicArn'])
# assert that subscription attributes are added properly
attrs = sns.get_subscription_attributes(
SubscriptionArn=subs[0]['SubscriptionArn'])['Attributes']
self.assertEqual(
attrs, {
'Endpoint': subs[0]['Endpoint'],
'Protocol': 'sqs',
'SubscriptionArn': subs[0]['SubscriptionArn'],
'TopicArn': subs[0]['TopicArn'],
'FilterPolicy': json.dumps({'eventType': ['created']})
})
# assert that Gateway responses have been created
test_api_name = 'test-api'
api = [
a for a in apigateway.get_rest_apis()['items']
if a['name'] == test_api_name
][0]
responses = apigateway.get_gateway_responses(
restApiId=api['id'])['items']
self.assertEqual(len(responses), 2)
types = [r['responseType'] for r in responses]
self.assertEqual(set(types), set(['UNAUTHORIZED', 'DEFAULT_5XX']))
# delete the stack
cf_client.delete_stack(StackName=stack_name)
# assert that resources have been deleted
assert not bucket_exists('cf-test-bucket-1')
assert not queue_exists('cf-test-queue-1')
assert not topic_exists('%s-test-topic-1-1' % stack_name)
retry(lambda: self.assertFalse(stream_exists('cf-test-stream-1')))
async def message_to_subscriber(message_id, message, topic_arn, req_data,
headers, subscription_arn, skip_checks,
sns_backend, subscriber, subscriptions):
if subscription_arn not in [None, subscriber['SubscriptionArn']]:
return
filter_policy = json.loads(subscriber.get('FilterPolicy') or '{}')
message_attributes = get_message_attributes(req_data)
if not skip_checks and not check_filter_policy(filter_policy,
message_attributes):
LOG.info('SNS filter policy %s does not match attributes %s' %
(filter_policy, message_attributes))
return
if subscriber['Protocol'] == 'sms':
event = {
'topic_arn': topic_arn,
'endpoint': subscriber['Endpoint'],
'message_content': req_data['Message'][0]
}
sns_backend.sms_messages.append(event)
LOG.info('Delivering SMS message to %s: %s', subscriber['Endpoint'],
req_data['Message'][0])
return
elif subscriber['Protocol'] == 'sqs':
queue_url = None
try:
endpoint = subscriber['Endpoint']
if 'sqs_queue_url' in subscriber:
queue_url = subscriber.get('sqs_queue_url')
elif '://' in endpoint:
queue_url = endpoint
else:
queue_name = endpoint.split(':')[5]
queue_url = aws_stack.get_sqs_queue_url(queue_name)
subscriber['sqs_queue_url'] = queue_url
message_group_id = req_data.get(
'MessageGroupId')[0] if req_data.get('MessageGroupId') else ''
sqs_client = aws_stack.connect_to_service('sqs')
# TODO remove this kwargs if we stop using ElasticMQ entirely
kwargs = {
'MessageGroupId': message_group_id
} if SQS_BACKEND_IMPL == 'moto' else {}
sqs_client.send_message(
QueueUrl=queue_url,
MessageBody=create_sns_message_body(subscriber, req_data,
message_id),
MessageAttributes=create_sqs_message_attributes(
subscriber, message_attributes),
MessageSystemAttributes=create_sqs_system_attributes(headers),
**kwargs)
except Exception as exc:
LOG.info('Unable to forward SNS message to SQS: %s %s' %
(exc, traceback.format_exc()))
sns_error_to_dead_letter_queue(subscriber['SubscriptionArn'],
req_data, str(exc))
if 'NonExistentQueue' in str(exc):
LOG.info(
'Removing non-existent queue "%s" subscribed to topic "%s"'
% (queue_url, topic_arn))
subscriptions.remove(subscriber)
return
elif subscriber['Protocol'] == 'lambda':
try:
external_url = external_service_url('sns')
unsubscribe_url = '%s/?Action=Unsubscribe&SubscriptionArn=%s' % (
external_url, subscriber['SubscriptionArn'])
response = lambda_api.process_sns_notification(
subscriber['Endpoint'],
topic_arn,
subscriber['SubscriptionArn'],
message,
message_id,
message_attributes,
unsubscribe_url,
subject=req_data.get('Subject', [None])[0])
if isinstance(response, Response):
response.raise_for_status()
elif isinstance(response, FlaskResponse):
if response.status_code >= 400:
raise Exception('Error response (code %s): %s' %
(response.status_code, response.data))
except Exception as exc:
LOG.info('Unable to run Lambda function on SNS message: %s %s' %
(exc, traceback.format_exc()))
sns_error_to_dead_letter_queue(subscriber['SubscriptionArn'],
req_data, str(exc))
return
elif subscriber['Protocol'] in ['http', 'https']:
msg_type = (req_data.get('Type') or ['Notification'])[0]
try:
message_body = create_sns_message_body(subscriber, req_data,
message_id)
except Exception:
return
try:
response = requests.post(
subscriber['Endpoint'],
headers={
'Content-Type': 'text/plain',
# AWS headers according to
# https://docs.aws.amazon.com/sns/latest/dg/sns-message-and-json-formats.html#http-header
'x-amz-sns-message-type': msg_type,
'x-amz-sns-topic-arn': subscriber['TopicArn'],
'x-amz-sns-subscription-arn':
subscriber['SubscriptionArn'],
'User-Agent': 'Amazon Simple Notification Service Agent'
},
data=message_body,
verify=False)
response.raise_for_status()
except Exception as exc:
LOG.info(
'Received error on sending SNS message, putting to DLQ (if configured): %s'
% exc)
sns_error_to_dead_letter_queue(subscriber['SubscriptionArn'],
req_data, str(exc))
return
elif subscriber['Protocol'] == 'application':
try:
sns_client = aws_stack.connect_to_service('sns')
sns_client.publish(TargetArn=subscriber['Endpoint'],
Message=message)
except Exception as exc:
LOG.warning(
'Unable to forward SNS message to SNS platform app: %s %s' %
(exc, traceback.format_exc()))
sns_error_to_dead_letter_queue(subscriber['SubscriptionArn'],
req_data, str(exc))
return
elif subscriber['Protocol'] == 'email':
ses_client = aws_stack.connect_to_service('ses')
if subscriber.get('Endpoint'):
ses_client.verify_email_address(
EmailAddress=subscriber.get('Endpoint'))
ses_client.verify_email_address(
EmailAddress='*****@*****.**')
ses_client.send_email(
Source='*****@*****.**',
Message={
'Body': {
'Text': {
'Data': message
}
},
'Subject': {
'Data': 'SNS-Subscriber-Endpoint'
}
},
Destination={'ToAddresses': [subscriber.get('Endpoint')]})
else:
LOG.warning('Unexpected protocol "%s" for SNS subscription' %
subscriber['Protocol'])
def forward_request(self, method, path, data, headers):
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(to_str(data))
req_action = req_data['Action'][0]
topic_arn = req_data.get('TargetArn') or req_data.get('TopicArn')
if topic_arn:
topic_arn = topic_arn[0]
do_create_topic(topic_arn)
if req_action == 'SetSubscriptionAttributes':
sub = get_subscription_by_arn(req_data['SubscriptionArn'][0])
if not sub:
return make_error(message='Unable to find subscription for given ARN', code=400)
attr_name = req_data['AttributeName'][0]
attr_value = req_data['AttributeValue'][0]
sub[attr_name] = attr_value
return make_response(req_action)
elif req_action == 'GetSubscriptionAttributes':
sub = get_subscription_by_arn(req_data['SubscriptionArn'][0])
if not sub:
return make_error(message='Unable to find subscription for given ARN', code=400)
content = '<Attributes>'
for key, value in sub.items():
content += '<entry><key>%s</key><value>%s</value></entry>\n' % (key, value)
content += '</Attributes>'
return make_response(req_action, content=content)
elif req_action == 'Subscribe':
if 'Endpoint' not in req_data:
return make_error(message='Endpoint not specified in subscription', code=400)
elif req_action == 'Unsubscribe':
if 'SubscriptionArn' not in req_data:
return make_error(message='SubscriptionArn not specified in unsubscribe request', code=400)
do_unsubscribe(req_data.get('SubscriptionArn')[0])
elif req_action == 'Publish':
message = req_data['Message'][0]
sqs_client = aws_stack.connect_to_service('sqs')
for subscriber in SNS_SUBSCRIPTIONS[topic_arn]:
filter_policy = json.loads(subscriber.get('FilterPolicy', '{}'))
message_attributes = get_message_attributes(req_data)
if check_filter_policy(filter_policy, message_attributes):
if subscriber['Protocol'] == 'sqs':
endpoint = subscriber['Endpoint']
if 'sqs_queue_url' in subscriber:
queue_url = subscriber.get('sqs_queue_url')
elif '://' in endpoint:
queue_url = endpoint
else:
queue_name = endpoint.split(':')[5]
queue_url = aws_stack.get_sqs_queue_url(queue_name)
subscriber['sqs_queue_url'] = queue_url
try:
sqs_client.send_message(
QueueUrl=queue_url,
MessageBody=create_sns_message_body(subscriber, req_data)
)
except Exception as exc:
return make_error(message=str(exc), code=400)
elif subscriber['Protocol'] == 'lambda':
lambda_api.process_sns_notification(
subscriber['Endpoint'],
topic_arn, message, subject=req_data.get('Subject', [None])[0]
)
elif subscriber['Protocol'] in ['http', 'https']:
try:
message_body = create_sns_message_body(subscriber, req_data)
except Exception as exc:
return make_error(message=str(exc), code=400)
requests.post(
subscriber['Endpoint'],
headers={
'Content-Type': 'text/plain',
'x-amz-sns-message-type': 'Notification'
},
data=message_body
)
else:
LOGGER.warning('Unexpected protocol "%s" for SNS subscription' % subscriber['Protocol'])
# return response here because we do not want the request to be forwarded to SNS
return make_response(req_action)
return True
def send_event_to_target(
target_arn: str,
event: Dict,
target_attributes: Dict = None,
asynchronous: bool = True,
target: Dict = {},
):
region = target_arn.split(":")[3]
if ":lambda:" in target_arn:
from localstack.services.awslambda import lambda_api
lambda_api.run_lambda(func_arn=target_arn,
event=event,
context={},
asynchronous=asynchronous)
elif ":sns:" in target_arn:
sns_client = connect_to_service("sns", region_name=region)
sns_client.publish(TopicArn=target_arn, Message=json.dumps(event))
elif ":sqs:" in target_arn:
sqs_client = connect_to_service("sqs", region_name=region)
queue_url = get_sqs_queue_url(target_arn)
msg_group_id = dict_utils.get_safe(target_attributes,
"$.SqsParameters.MessageGroupId")
kwargs = {"MessageGroupId": msg_group_id} if msg_group_id else {}
sqs_client.send_message(QueueUrl=queue_url,
MessageBody=json.dumps(event),
**kwargs)
elif ":states:" in target_arn:
stepfunctions_client = connect_to_service("stepfunctions",
region_name=region)
stepfunctions_client.start_execution(stateMachineArn=target_arn,
input=json.dumps(event))
elif ":firehose:" in target_arn:
delivery_stream_name = firehose_name(target_arn)
firehose_client = connect_to_service("firehose", region_name=region)
firehose_client.put_record(
DeliveryStreamName=delivery_stream_name,
Record={"Data": to_bytes(json.dumps(event))},
)
elif ":events:" in target_arn:
if ":api-destination/" in target_arn or ":destination/" in target_arn:
send_event_to_api_destination(target_arn, event,
target.get("HttpParameters"))
else:
events_client = connect_to_service("events", region_name=region)
eventbus_name = target_arn.split(":")[-1].split("/")[-1]
events_client.put_events(
Entries=[{
"EventBusName": eventbus_name,
"Source": event.get("source"),
"DetailType": event.get("detail-type"),
"Detail": event.get("detail"),
}])
elif ":kinesis:" in target_arn:
partition_key_path = dict_utils.get_safe(
target_attributes,
"$.KinesisParameters.PartitionKeyPath",
default_value="$.id",
)
stream_name = target_arn.split("/")[-1]
partition_key = dict_utils.get_safe(event, partition_key_path,
event["id"])
kinesis_client = connect_to_service("kinesis", region_name=region)
kinesis_client.put_record(
StreamName=stream_name,
Data=to_bytes(json.dumps(event)),
PartitionKey=partition_key,
)
elif ":logs:" in target_arn:
log_group_name = target_arn.split(":")[-1]
logs_client = connect_to_service("logs", region_name=region)
log_stream_name = str(uuid.uuid4())
logs_client.create_log_stream(logGroupName=log_group_name,
logStreamName=log_stream_name)
logs_client.put_log_events(
logGroupName=log_group_name,
logStreamName=log_stream_name,
logEvents=[{
"timestamp": now_utc(millis=True),
"message": json.dumps(event)
}],
)
else:
LOG.warning('Unsupported Events rule target ARN: "%s"', target_arn)
def update_sns(method,
path,
data,
headers,
response=None,
return_forward_info=False):
if return_forward_info:
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(data)
req_action = req_data['Action'][0]
topic_arn = req_data.get('TargetArn') or req_data.get('TopicArn')
if topic_arn:
topic_arn = topic_arn[0]
if topic_arn not in SNS_SUBSCRIPTIONS:
SNS_SUBSCRIPTIONS[topic_arn] = []
if req_action == 'SetSubscriptionAttributes':
sub = get_subscription_by_arn(req_data['SubscriptionArn'][0])
if not sub:
return make_error(
message='Unable to find subscription for given ARN',
code=400)
attr_name = req_data['AttributeName'][0]
attr_value = req_data['AttributeValue'][0]
sub[attr_name] = attr_value
return make_response(req_action)
elif req_action == 'GetSubscriptionAttributes':
sub = get_subscription_by_arn(req_data['SubscriptionArn'][0])
if not sub:
return make_error(
message='Unable to find subscription for given ARN',
code=400)
content = '<Attributes>'
for key, value in sub.items():
content += '<entry><key>%s</key><value>%s</value></entry>\n' % (
key, value)
content += '</Attributes>'
return make_response(req_action, content=content)
elif req_action == 'Subscribe':
if 'Endpoint' not in req_data:
return make_error(
message='Endpoint not specified in subscription',
code=400)
elif req_action == 'Publish':
message = req_data['Message'][0]
sqs_client = aws_stack.connect_to_service('sqs')
for subscriber in SNS_SUBSCRIPTIONS[topic_arn]:
if subscriber['Protocol'] == 'sqs':
queue_name = subscriber['Endpoint'].split(':')[5]
queue_url = subscriber.get('sqs_queue_url')
if not queue_url:
queue_url = aws_stack.get_sqs_queue_url(queue_name)
subscriber['sqs_queue_url'] = queue_url
sqs_client.send_message(
QueueUrl=queue_url,
MessageBody=create_sns_message_body(
subscriber, req_data))
elif subscriber['Protocol'] == 'lambda':
lambda_api.process_sns_notification(
subscriber['Endpoint'],
topic_arn,
message,
subject=req_data.get('Subject'))
elif subscriber['Protocol'] == 'http':
requests.post(subscriber['Endpoint'],
headers={
'Content-Type': 'text/plain',
'x-amz-sns-message-type':
'Notification'
},
data=json.dumps({
'Type': 'Notification',
'Message': message,
}))
else:
LOGGER.warning(
'Unexpected protocol "%s" for SNS subscription' %
subscriber['Protocol'])
# return response here because we do not want the request to be forwarded to SNS
return make_response(req_action)
return True
else:
# This branch is executed by the proxy after we've already received a
# response from the backend, hence we can utilize the "reponse" variable here
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(data)
req_action = req_data['Action'][0]
if req_action == 'Subscribe' and response.status_code < 400:
response_data = xmltodict.parse(response.content)
topic_arn = (req_data.get('TargetArn')
or req_data.get('TopicArn'))[0]
sub_arn = response_data['SubscribeResponse'][
'SubscribeResult']['SubscriptionArn']
subscription = {
# http://docs.aws.amazon.com/cli/latest/reference/sns/get-subscription-attributes.html
'TopicArn': topic_arn,
'Endpoint': req_data['Endpoint'][0],
'Protocol': req_data['Protocol'][0],
'SubscriptionArn': sub_arn,
'RawMessageDelivery': 'false'
}
SNS_SUBSCRIPTIONS[topic_arn].append(subscription)
def forward_request(self, method, path, data, headers):
if method == 'POST' and path == '/':
req_data = urlparse.parse_qs(to_str(data))
req_action = req_data['Action'][0]
topic_arn = req_data.get('TargetArn') or req_data.get('TopicArn')
if topic_arn:
topic_arn = topic_arn[0]
do_create_topic(topic_arn)
if req_action == 'SetSubscriptionAttributes':
sub = get_subscription_by_arn(req_data['SubscriptionArn'][0])
if not sub:
return make_error(
message='Unable to find subscription for given ARN',
code=400)
attr_name = req_data['AttributeName'][0]
attr_value = req_data['AttributeValue'][0]
sub[attr_name] = attr_value
return make_response(req_action)
elif req_action == 'GetSubscriptionAttributes':
sub = get_subscription_by_arn(req_data['SubscriptionArn'][0])
if not sub:
return make_error(
message='Unable to find subscription for given ARN',
code=400)
content = '<Attributes>'
for key, value in sub.items():
content += '<entry><key>%s</key><value>%s</value></entry>\n' % (
key, value)
content += '</Attributes>'
return make_response(req_action, content=content)
elif req_action == 'Subscribe':
if 'Endpoint' not in req_data:
return make_error(
message='Endpoint not specified in subscription',
code=400)
elif req_action == 'Unsubscribe':
if 'SubscriptionArn' not in req_data:
return make_error(
message=
'SubscriptionArn not specified in unsubscribe request',
code=400)
do_unsubscribe(req_data.get('SubscriptionArn')[0])
elif req_action == 'Publish':
message = req_data['Message'][0]
sqs_client = aws_stack.connect_to_service('sqs')
for subscriber in SNS_SUBSCRIPTIONS[topic_arn]:
if subscriber['Protocol'] == 'sqs':
endpoint = subscriber['Endpoint']
if 'sqs_queue_url' in subscriber:
queue_url = subscriber.get('sqs_queue_url')
elif '://' in endpoint:
queue_url = endpoint
else:
queue_name = endpoint.split(':')[5]
queue_url = aws_stack.get_sqs_queue_url(queue_name)
subscriber['sqs_queue_url'] = queue_url
sqs_client.send_message(
QueueUrl=queue_url,
MessageBody=create_sns_message_body(
subscriber, req_data))
elif subscriber['Protocol'] == 'lambda':
lambda_api.process_sns_notification(
subscriber['Endpoint'],
topic_arn,
message,
subject=req_data.get('Subject'))
elif subscriber['Protocol'] in ['http', 'https']:
requests.post(subscriber['Endpoint'],
headers={
'Content-Type': 'text/plain',
'x-amz-sns-message-type':
'Notification'
},
data=create_sns_message_body(
subscriber, req_data))
else:
LOGGER.warning(
'Unexpected protocol "%s" for SNS subscription' %
subscriber['Protocol'])
# return response here because we do not want the request to be forwarded to SNS
return make_response(req_action)
return True
