def get_event_message(event_name, bucket_name, file_name='testfile.txt', file_size=1024):
# Based on: http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
return {
'Records': [{
'eventVersion': '2.0',
'eventSource': 'aws:s3',
'awsRegion': DEFAULT_REGION,
'eventTime': timestamp(format=TIMESTAMP_FORMAT_MILLIS),
'eventName': event_name,
'userIdentity': {
'principalId': 'AIDAJDPLRKLG7UEXAMPLE'
},
's3': {
's3SchemaVersion': '1.0',
'configurationId': 'testConfigRule',
'bucket': {
'name': bucket_name,
'ownerIdentity': {
'principalId': 'A3NL1KOZZKExample'
},
'arn': 'arn:aws:s3:::%s' % bucket_name
},
'object': {
'key': file_name,
'size': file_size,
'eTag': 'd41d8cd98f00b204e9800998ecf8427e',
'versionId': '096fKKXTRTtl3on89fVO.nfljtsv6qko',
'sequencer': '0055AED6DCD90281E5'
}
}
}]
}
def append_cors_headers(bucket_name, request_method, request_headers, response):
cors = BUCKET_CORS.get(bucket_name)
if not cors:
return
origin = request_headers.get('Origin', '')
rules = cors['CORSConfiguration']['CORSRule']
if not isinstance(rules, list):
rules = [rules]
for rule in rules:
# add allow-origin header
allowed_methods = rule.get('AllowedMethod', [])
if request_method in allowed_methods:
allowed_origins = rule.get('AllowedOrigin', [])
for allowed in allowed_origins:
if origin in allowed or re.match(allowed.replace('*', '.*'), origin):
response.headers['Access-Control-Allow-Origin'] = origin
break
# add additional headers
exposed_headers = rule.get('ExposeHeader', [])
for header in exposed_headers:
if header.lower() == 'date':
response.headers[header] = timestamp(format='%a, %d %b %Y %H:%M:%S +0000')
elif header.lower() == 'etag':
response.headers[header] = md5(response._content)
elif header.lower() in ('server', 'x-amz-id-2', 'x-amz-request-id'):
response.headers[header] = short_uid()
elif header.lower() == 'x-amz-delete-marker':
response.headers[header] = 'false'
elif header.lower() == 'x-amz-version-id':
# TODO: check whether bucket versioning is enabled and return proper version id
response.headers[header] = 'null'
def append_cors_headers(bucket_name, request_method, request_headers, response):
cors = BUCKET_CORS.get(bucket_name)
if not cors:
return
origin = request_headers.get('Origin', '')
rules = cors['CORSConfiguration']['CORSRule']
if not isinstance(rules, list):
rules = [rules]
for rule in rules:
# add allow-origin header
allowed_methods = rule.get('AllowedMethod', [])
if request_method in allowed_methods:
allowed_origins = rule.get('AllowedOrigin', [])
for allowed in allowed_origins:
if origin in allowed or re.match(allowed.replace('*', '.*'), origin):
response.headers['Access-Control-Allow-Origin'] = origin
break
# add additional headers
exposed_headers = rule.get('ExposeHeader', [])
for header in exposed_headers:
if header.lower() == 'date':
response.headers[header] = timestamp(format='%a, %d %b %Y %H:%M:%S +0000')
elif header.lower() == 'etag':
response.headers[header] = md5(response._content)
elif header.lower() in ('server', 'x-amz-id-2', 'x-amz-request-id'):
response.headers[header] = short_uid()
elif header.lower() == 'x-amz-delete-marker':
response.headers[header] = 'false'
elif header.lower() == 'x-amz-version-id':
# TODO: check whether bucket versioning is enabled and return proper version id
response.headers[header] = 'null'
def _add_missing_record_attributes(self, records: List[Dict]) -> None:
def _get_entry(obj, key):
return obj.get(key) or obj.get(first_char_to_lower(key))
for record in records:
if not _get_entry(record, "ApproximateArrivalTimestamp"):
record["ApproximateArrivalTimestamp"] = int(
now_utc(millis=True))
if not _get_entry(record, "KinesisRecordMetadata"):
record["kinesisRecordMetadata"] = {
"shardId":
"shardId-000000000000",
# not really documented what AWS is using internally - simply using a random UUID here
"partitionKey":
str(uuid.uuid4()),
"approximateArrivalTimestamp":
timestamp(
float(_get_entry(
record, "ApproximateArrivalTimestamp")) / 1000,
format=TIMESTAMP_FORMAT_MICROS,
),
"sequenceNumber":
next_sequence_number(),
"subsequenceNumber":
"",
}
def process_sns_notification(
func_arn,
topic_arn,
subscriptionArn,
message,
message_attributes,
subject='',
):
try:
event = {
'Records': [{
'EventSource': 'localstack:sns',
'EventVersion': '1.0',
'EventSubscriptionArn': subscriptionArn,
'Sns': {
'Type': 'Notification',
'TopicArn': topic_arn,
'Subject': subject,
'Message': message,
'Timestamp': timestamp(format=TIMESTAMP_FORMAT_MILLIS),
'MessageAttributes': message_attributes
}
}]
}
return run_lambda(event=event,
context={},
func_arn=func_arn,
asynchronous=True)
except Exception as e:
LOG.warning('Unable to run Lambda function on SNS message: %s %s' %
(e, traceback.format_exc()))
def __init__(self, **kwargs):
self.t = kwargs.get('timestamp') or kwargs.get('t') or timestamp()
self.m_id = kwargs.get('machine_id') or kwargs.get(
'm_id') or get_machine_id()
self.p_id = kwargs.get('process_id') or kwargs.get(
'p_id') or get_process_id()
self.e_t = kwargs.get('event_type') or kwargs.get('e_t')
self.p = kwargs.get('payload') or kwargs.get('p')
def get_s3_object_path(stream_name, prefix):
# See https://aws.amazon.com/kinesis/data-firehose/faqs/#Data_delivery
# Path prefix pattern: myApp/YYYY/MM/DD/HH/
# Object name pattern: DeliveryStreamName-DeliveryStreamVersion-YYYY-MM-DD-HH-MM-SS-RandomString
prefix = '%s%s' % (prefix, '' if prefix.endswith('/') else '/')
pattern = '{pre}%Y/%m/%d/%H/{name}-%Y-%m-%d-%H-%M-%S-{rand}'
path = pattern.format(pre=prefix, name=stream_name, rand=str(uuid.uuid4()))
path = timestamp(format=path)
return path
def _get_s3_object_path(self, stream_name, prefix):
# See https://aws.amazon.com/kinesis/data-firehose/faqs/#Data_delivery
# Path prefix pattern: myApp/YYYY/MM/DD/HH/
# Object name pattern: DeliveryStreamName-DeliveryStreamVersion-YYYY-MM-DD-HH-MM-SS-RandomString
if not prefix.endswith("/") and prefix != "":
prefix = prefix + "/"
pattern = "{pre}%Y/%m/%d/%H/{name}-%Y-%m-%d-%H-%M-%S-{rand}"
path = pattern.format(pre=prefix, name=stream_name, rand=str(uuid.uuid4()))
path = timestamp(format=path)
return path
def __init__(self, **kwargs):
self.t = kwargs.get("timestamp") or kwargs.get("t") or timestamp()
self.m_id = kwargs.get("machine_id") or kwargs.get(
"m_id") or get_machine_id()
self.p_id = kwargs.get("process_id") or kwargs.get(
"p_id") or get_process_id()
self.p = kwargs.get("payload") if kwargs.get(
"payload") is not None else kwargs.get("p")
self.k = kwargs.get("api_key") or kwargs.get(
"k") or read_api_key_safe()
self.e_t = kwargs.get("event_type") or kwargs.get("e_t")
def __init__(self, **kwargs):
self.t = kwargs.get('timestamp') or kwargs.get('t') or timestamp()
self.m_id = kwargs.get('machine_id') or kwargs.get(
'm_id') or get_machine_id()
self.p_id = kwargs.get('process_id') or kwargs.get(
'p_id') or get_process_id()
self.p = kwargs.get('payload') if kwargs.get(
'payload') is not None else kwargs.get('p')
self.k = kwargs.get('api_key') or kwargs.get(
'k') or read_api_key_safe()
self.e_t = kwargs.get('event_type') or kwargs.get('e_t')
def create_sns_message_body(subscriber, req_data):
message = req_data['Message'][0]
subject = req_data.get('Subject', [None])[0]
protocol = subscriber['Protocol']
if six.PY2 and type(message).__name__ == 'unicode':
# fix non-ascii unicode characters under Python 2
message = message.encode('raw-unicode-escape')
if is_raw_message_delivery(subscriber):
return message
if req_data.get('MessageStructure') == ['json']:
message = json.loads(message)
try:
message = message.get(protocol, message['default'])
except KeyError:
raise Exception("Unable to find 'default' key in message payload")
data = {
'Type':
req_data.get('Type', ['Notification'])[0],
'MessageId':
str(uuid.uuid4()),
'Token':
req_data.get('Token', [None])[0],
'TopicArn':
subscriber['TopicArn'],
'Message':
message,
'SubscribeURL':
req_data.get('SubscribeURL', [None])[0],
'Timestamp':
timestamp(format=TIMESTAMP_FORMAT_MILLIS),
'SignatureVersion':
'1',
# TODO Add a more sophisticated solution with an actual signature
# Hardcoded
'Signature':
'EXAMPLEpH+..',
'SigningCertURL':
'https://sns.us-east-1.amazonaws.com/SimpleNotificationService-0000000000000000000000.pem'
}
if subject is not None:
data['Subject'] = subject
attributes = get_message_attributes(req_data)
if attributes:
data['MessageAttributes'] = attributes
return json.dumps(data)
def process_sns_notification(func_arn, topic_arn, message, subject=''):
try:
event = {
'Records': [{
'Sns': {
'Type': 'Notification',
'TopicArn': topic_arn,
'Subject': subject,
'Message': message,
'Timestamp': timestamp(format=TIMESTAMP_FORMAT_MILLIS)
}
}]
}
return run_lambda(event=event, context={}, func_arn=func_arn, async=True)
except Exception as e:
LOG.warning('Unable to run Lambda function on SNS message: %s %s' % (e, traceback.format_exc()))
def process_sns_notification(func_arn, topic_arn, message, subject=''):
try:
event = {
'Records': [{
'Sns': {
'Type': 'Notification',
'TopicArn': topic_arn,
'Subject': subject,
'Message': message,
'Timestamp': timestamp(format=TIMESTAMP_FORMAT_MILLIS)
}
}]
}
return run_lambda(event=event, context={}, func_arn=func_arn, async=True)
except Exception as e:
LOG.warning('Unable to run Lambda function on SNS message: %s %s' % (e, traceback.format_exc()))
def get_event_message(event_name,
bucket_name,
file_name='testfile.txt',
file_size=1024):
LOGGER.debug('get_event_message - event_name: "%s"' % event_name)
LOGGER.debug('get_event_message - bucket_name: "%s"' % bucket_name)
LOGGER.debug('get_event_message - file_name: "%s"' % file_name)
# Based on: http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
return {
'Records': [{
'eventVersion': '2.0',
'eventSource': 'aws:s3',
'awsRegion': DEFAULT_REGION,
'eventTime': timestamp(format=TIMESTAMP_FORMAT_MILLIS),
'eventName': event_name,
'userIdentity': {
'principalId': 'AIDAJDPLRKLG7UEXAMPLE'
},
'requestParameters': {
'sourceIPAddress': '127.0.0.1' # TODO determine real source IP
},
'responseElements': {
'x-amz-request-id': short_uid(),
'x-amz-id-2':
'eftixk72aD6Ap51TnqcoF8eFidJG9Z/2' # Amazon S3 host that processed the request
},
's3': {
's3SchemaVersion': '1.0',
'configurationId': 'testConfigRule',
'bucket': {
'name': bucket_name,
'ownerIdentity': {
'principalId': 'A3NL1KOZZKExample'
},
'arn': 'arn:aws:s3:::%s' % bucket_name
},
'object': {
'key': file_name,
'size': file_size,
'eTag': 'd41d8cd98f00b204e9800998ecf8427e',
'versionId': '096fKKXTRTtl3on89fVO.nfljtsv6qko',
'sequencer': '0055AED6DCD90281E5'
}
}
}]
}
def get_event_message(event_name, bucket_name, file_name='testfile.txt', file_size=1024):
# Based on: http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
return {
'Records': [{
'eventVersion': '2.0',
'eventSource': 'aws:s3',
'awsRegion': DEFAULT_REGION,
'eventTime': timestamp(format=TIMESTAMP_FORMAT_MILLIS),
'eventName': event_name,
'userIdentity': {
'principalId': 'AIDAJDPLRKLG7UEXAMPLE'
},
'requestParameters': {
'sourceIPAddress': '127.0.0.1' # TODO determine real source IP
},
'responseElements': {
'x-amz-request-id': short_uid(),
'x-amz-id-2': 'eftixk72aD6Ap51TnqcoF8eFidJG9Z/2' # Amazon S3 host that processed the request
},
's3': {
's3SchemaVersion': '1.0',
'configurationId': 'testConfigRule',
'bucket': {
'name': bucket_name,
'ownerIdentity': {
'principalId': 'A3NL1KOZZKExample'
},
'arn': 'arn:aws:s3:::%s' % bucket_name
},
'object': {
'key': file_name,
'size': file_size,
'eTag': 'd41d8cd98f00b204e9800998ecf8427e',
'versionId': '096fKKXTRTtl3on89fVO.nfljtsv6qko',
'sequencer': '0055AED6DCD90281E5'
}
}
}]
}
def create_sns_message_body(subscriber, req_data):
message = req_data['Message'][0]
subject = req_data.get('Subject', [None])[0]
protocol = subscriber['Protocol']
if six.PY2 and type(message).__name__ == 'unicode':
# fix non-ascii unicode characters under Python 2
message = message.encode('raw-unicode-escape')
if subscriber.get('RawMessageDelivery') in ('true', 'True', True):
return message
if req_data.get('MessageStructure') == ['json']:
message = json.loads(message)
try:
message = message.get(protocol, message['default'])
except KeyError:
raise Exception("Unable to find 'default' key in message payload")
data = {}
data['MessageId'] = str(uuid.uuid4())
data['Type'] = req_data.get('Type', ['Notification'])[0]
data['Timestamp'] = timestamp(format=TIMESTAMP_FORMAT_MILLIS)
data['Message'] = message
data['TopicArn'] = subscriber['TopicArn']
if subject is not None:
data['Subject'] = subject
attributes = get_message_attributes(req_data)
if attributes:
data['MessageAttributes'] = attributes
for attr in ['Token', 'SubscribeURL']:
value = req_data.get(attr, [None])[0]
if value:
data[attr] = value
result = json.dumps(data)
return result
def fix_hardcoded_creation_date(response):
# TODO: remove once this is fixed upstream
search = r'<CreationTime>\s*(2011-05-23T15:47:44Z)?\s*</CreationTime>'
replace = r'<CreationTime>%s</CreationTime>' % timestamp()
response._content = re.sub(search, replace, to_str(response._content or ''))
response.headers['Content-Length'] = str(len(response._content))
def dynamodb_stream_arn(table_name, account_id=None):
account_id = get_account_id(account_id)
return ('arn:aws:dynamodb:%s:%s:table/%s/stream/%s' %
(get_local_region(), account_id, table_name, timestamp()))
def fix_hardcoded_creation_date(response):
search = '<CreationTime>2011-05-23T15:47:44Z</CreationTime>'
replace = '<CreationTime>%s</CreationTime>' % timestamp()
response._content = to_str(response._content
or '').replace(search, replace)
response.headers['Content-Length'] = str(len(response._content))
def dynamodb_stream_arn(table_name, account_id=None):
account_id = get_account_id(account_id)
return ("arn:aws:dynamodb:%s:%s:table/%s/stream/%s" %
(DEFAULT_REGION, account_id, table_name, timestamp()))
def dynamodb_stream_arn(table_name, account_id=None):
account_id = get_account_id(account_id)
return ('arn:aws:dynamodb:%s:%s:table/%s/stream/%s' %
(get_local_region(), account_id, table_name, timestamp()))
def __init__(self, **kwargs):
self.t = kwargs.get('timestamp') or kwargs.get('t') or timestamp()
self.m_id = kwargs.get('machine_id') or kwargs.get('m_id') or get_machine_id()
self.p_id = kwargs.get('process_id') or kwargs.get('p_id') or get_process_id()
self.e_t = kwargs.get('event_type') or kwargs.get('e_t')
self.p = kwargs.get('payload') if kwargs.get('payload') is not None else kwargs.get('p')