def get_event_message(event_name, bucket_name, file_name='testfile.txt', file_size=1024): # Based on: http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html return { 'Records': [{ 'eventVersion': '2.0', 'eventSource': 'aws:s3', 'awsRegion': DEFAULT_REGION, 'eventTime': timestamp(format=TIMESTAMP_FORMAT_MILLIS), 'eventName': event_name, 'userIdentity': { 'principalId': 'AIDAJDPLRKLG7UEXAMPLE' }, 's3': { 's3SchemaVersion': '1.0', 'configurationId': 'testConfigRule', 'bucket': { 'name': bucket_name, 'ownerIdentity': { 'principalId': 'A3NL1KOZZKExample' }, 'arn': 'arn:aws:s3:::%s' % bucket_name }, 'object': { 'key': file_name, 'size': file_size, 'eTag': 'd41d8cd98f00b204e9800998ecf8427e', 'versionId': '096fKKXTRTtl3on89fVO.nfljtsv6qko', 'sequencer': '0055AED6DCD90281E5' } } }] }
def append_cors_headers(bucket_name, request_method, request_headers, response): cors = BUCKET_CORS.get(bucket_name) if not cors: return origin = request_headers.get('Origin', '') rules = cors['CORSConfiguration']['CORSRule'] if not isinstance(rules, list): rules = [rules] for rule in rules: # add allow-origin header allowed_methods = rule.get('AllowedMethod', []) if request_method in allowed_methods: allowed_origins = rule.get('AllowedOrigin', []) for allowed in allowed_origins: if origin in allowed or re.match(allowed.replace('*', '.*'), origin): response.headers['Access-Control-Allow-Origin'] = origin break # add additional headers exposed_headers = rule.get('ExposeHeader', []) for header in exposed_headers: if header.lower() == 'date': response.headers[header] = timestamp(format='%a, %d %b %Y %H:%M:%S +0000') elif header.lower() == 'etag': response.headers[header] = md5(response._content) elif header.lower() in ('server', 'x-amz-id-2', 'x-amz-request-id'): response.headers[header] = short_uid() elif header.lower() == 'x-amz-delete-marker': response.headers[header] = 'false' elif header.lower() == 'x-amz-version-id': # TODO: check whether bucket versioning is enabled and return proper version id response.headers[header] = 'null'
def append_cors_headers(bucket_name, request_method, request_headers, response): cors = BUCKET_CORS.get(bucket_name) if not cors: return origin = request_headers.get('Origin', '') rules = cors['CORSConfiguration']['CORSRule'] if not isinstance(rules, list): rules = [rules] for rule in rules: # add allow-origin header allowed_methods = rule.get('AllowedMethod', []) if request_method in allowed_methods: allowed_origins = rule.get('AllowedOrigin', []) for allowed in allowed_origins: if origin in allowed or re.match(allowed.replace('*', '.*'), origin): response.headers['Access-Control-Allow-Origin'] = origin break # add additional headers exposed_headers = rule.get('ExposeHeader', []) for header in exposed_headers: if header.lower() == 'date': response.headers[header] = timestamp(format='%a, %d %b %Y %H:%M:%S +0000') elif header.lower() == 'etag': response.headers[header] = md5(response._content) elif header.lower() in ('server', 'x-amz-id-2', 'x-amz-request-id'): response.headers[header] = short_uid() elif header.lower() == 'x-amz-delete-marker': response.headers[header] = 'false' elif header.lower() == 'x-amz-version-id': # TODO: check whether bucket versioning is enabled and return proper version id response.headers[header] = 'null'
def _add_missing_record_attributes(self, records: List[Dict]) -> None: def _get_entry(obj, key): return obj.get(key) or obj.get(first_char_to_lower(key)) for record in records: if not _get_entry(record, "ApproximateArrivalTimestamp"): record["ApproximateArrivalTimestamp"] = int( now_utc(millis=True)) if not _get_entry(record, "KinesisRecordMetadata"): record["kinesisRecordMetadata"] = { "shardId": "shardId-000000000000", # not really documented what AWS is using internally - simply using a random UUID here "partitionKey": str(uuid.uuid4()), "approximateArrivalTimestamp": timestamp( float(_get_entry( record, "ApproximateArrivalTimestamp")) / 1000, format=TIMESTAMP_FORMAT_MICROS, ), "sequenceNumber": next_sequence_number(), "subsequenceNumber": "", }
def process_sns_notification( func_arn, topic_arn, subscriptionArn, message, message_attributes, subject='', ): try: event = { 'Records': [{ 'EventSource': 'localstack:sns', 'EventVersion': '1.0', 'EventSubscriptionArn': subscriptionArn, 'Sns': { 'Type': 'Notification', 'TopicArn': topic_arn, 'Subject': subject, 'Message': message, 'Timestamp': timestamp(format=TIMESTAMP_FORMAT_MILLIS), 'MessageAttributes': message_attributes } }] } return run_lambda(event=event, context={}, func_arn=func_arn, asynchronous=True) except Exception as e: LOG.warning('Unable to run Lambda function on SNS message: %s %s' % (e, traceback.format_exc()))
def __init__(self, **kwargs): self.t = kwargs.get('timestamp') or kwargs.get('t') or timestamp() self.m_id = kwargs.get('machine_id') or kwargs.get( 'm_id') or get_machine_id() self.p_id = kwargs.get('process_id') or kwargs.get( 'p_id') or get_process_id() self.e_t = kwargs.get('event_type') or kwargs.get('e_t') self.p = kwargs.get('payload') or kwargs.get('p')
def get_s3_object_path(stream_name, prefix): # See https://aws.amazon.com/kinesis/data-firehose/faqs/#Data_delivery # Path prefix pattern: myApp/YYYY/MM/DD/HH/ # Object name pattern: DeliveryStreamName-DeliveryStreamVersion-YYYY-MM-DD-HH-MM-SS-RandomString prefix = '%s%s' % (prefix, '' if prefix.endswith('/') else '/') pattern = '{pre}%Y/%m/%d/%H/{name}-%Y-%m-%d-%H-%M-%S-{rand}' path = pattern.format(pre=prefix, name=stream_name, rand=str(uuid.uuid4())) path = timestamp(format=path) return path
def _get_s3_object_path(self, stream_name, prefix): # See https://aws.amazon.com/kinesis/data-firehose/faqs/#Data_delivery # Path prefix pattern: myApp/YYYY/MM/DD/HH/ # Object name pattern: DeliveryStreamName-DeliveryStreamVersion-YYYY-MM-DD-HH-MM-SS-RandomString if not prefix.endswith("/") and prefix != "": prefix = prefix + "/" pattern = "{pre}%Y/%m/%d/%H/{name}-%Y-%m-%d-%H-%M-%S-{rand}" path = pattern.format(pre=prefix, name=stream_name, rand=str(uuid.uuid4())) path = timestamp(format=path) return path
def __init__(self, **kwargs): self.t = kwargs.get("timestamp") or kwargs.get("t") or timestamp() self.m_id = kwargs.get("machine_id") or kwargs.get( "m_id") or get_machine_id() self.p_id = kwargs.get("process_id") or kwargs.get( "p_id") or get_process_id() self.p = kwargs.get("payload") if kwargs.get( "payload") is not None else kwargs.get("p") self.k = kwargs.get("api_key") or kwargs.get( "k") or read_api_key_safe() self.e_t = kwargs.get("event_type") or kwargs.get("e_t")
def __init__(self, **kwargs): self.t = kwargs.get('timestamp') or kwargs.get('t') or timestamp() self.m_id = kwargs.get('machine_id') or kwargs.get( 'm_id') or get_machine_id() self.p_id = kwargs.get('process_id') or kwargs.get( 'p_id') or get_process_id() self.p = kwargs.get('payload') if kwargs.get( 'payload') is not None else kwargs.get('p') self.k = kwargs.get('api_key') or kwargs.get( 'k') or read_api_key_safe() self.e_t = kwargs.get('event_type') or kwargs.get('e_t')
def create_sns_message_body(subscriber, req_data): message = req_data['Message'][0] subject = req_data.get('Subject', [None])[0] protocol = subscriber['Protocol'] if six.PY2 and type(message).__name__ == 'unicode': # fix non-ascii unicode characters under Python 2 message = message.encode('raw-unicode-escape') if is_raw_message_delivery(subscriber): return message if req_data.get('MessageStructure') == ['json']: message = json.loads(message) try: message = message.get(protocol, message['default']) except KeyError: raise Exception("Unable to find 'default' key in message payload") data = { 'Type': req_data.get('Type', ['Notification'])[0], 'MessageId': str(uuid.uuid4()), 'Token': req_data.get('Token', [None])[0], 'TopicArn': subscriber['TopicArn'], 'Message': message, 'SubscribeURL': req_data.get('SubscribeURL', [None])[0], 'Timestamp': timestamp(format=TIMESTAMP_FORMAT_MILLIS), 'SignatureVersion': '1', # TODO Add a more sophisticated solution with an actual signature # Hardcoded 'Signature': 'EXAMPLEpH+..', 'SigningCertURL': 'https://sns.us-east-1.amazonaws.com/SimpleNotificationService-0000000000000000000000.pem' } if subject is not None: data['Subject'] = subject attributes = get_message_attributes(req_data) if attributes: data['MessageAttributes'] = attributes return json.dumps(data)
def process_sns_notification(func_arn, topic_arn, message, subject=''): try: event = { 'Records': [{ 'Sns': { 'Type': 'Notification', 'TopicArn': topic_arn, 'Subject': subject, 'Message': message, 'Timestamp': timestamp(format=TIMESTAMP_FORMAT_MILLIS) } }] } return run_lambda(event=event, context={}, func_arn=func_arn, async=True) except Exception as e: LOG.warning('Unable to run Lambda function on SNS message: %s %s' % (e, traceback.format_exc()))
def process_sns_notification(func_arn, topic_arn, message, subject=''): try: event = { 'Records': [{ 'Sns': { 'Type': 'Notification', 'TopicArn': topic_arn, 'Subject': subject, 'Message': message, 'Timestamp': timestamp(format=TIMESTAMP_FORMAT_MILLIS) } }] } return run_lambda(event=event, context={}, func_arn=func_arn, async=True) except Exception as e: LOG.warning('Unable to run Lambda function on SNS message: %s %s' % (e, traceback.format_exc()))
def get_event_message(event_name, bucket_name, file_name='testfile.txt', file_size=1024): LOGGER.debug('get_event_message - event_name: "%s"' % event_name) LOGGER.debug('get_event_message - bucket_name: "%s"' % bucket_name) LOGGER.debug('get_event_message - file_name: "%s"' % file_name) # Based on: http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html return { 'Records': [{ 'eventVersion': '2.0', 'eventSource': 'aws:s3', 'awsRegion': DEFAULT_REGION, 'eventTime': timestamp(format=TIMESTAMP_FORMAT_MILLIS), 'eventName': event_name, 'userIdentity': { 'principalId': 'AIDAJDPLRKLG7UEXAMPLE' }, 'requestParameters': { 'sourceIPAddress': '127.0.0.1' # TODO determine real source IP }, 'responseElements': { 'x-amz-request-id': short_uid(), 'x-amz-id-2': 'eftixk72aD6Ap51TnqcoF8eFidJG9Z/2' # Amazon S3 host that processed the request }, 's3': { 's3SchemaVersion': '1.0', 'configurationId': 'testConfigRule', 'bucket': { 'name': bucket_name, 'ownerIdentity': { 'principalId': 'A3NL1KOZZKExample' }, 'arn': 'arn:aws:s3:::%s' % bucket_name }, 'object': { 'key': file_name, 'size': file_size, 'eTag': 'd41d8cd98f00b204e9800998ecf8427e', 'versionId': '096fKKXTRTtl3on89fVO.nfljtsv6qko', 'sequencer': '0055AED6DCD90281E5' } } }] }
def get_event_message(event_name, bucket_name, file_name='testfile.txt', file_size=1024): # Based on: http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html return { 'Records': [{ 'eventVersion': '2.0', 'eventSource': 'aws:s3', 'awsRegion': DEFAULT_REGION, 'eventTime': timestamp(format=TIMESTAMP_FORMAT_MILLIS), 'eventName': event_name, 'userIdentity': { 'principalId': 'AIDAJDPLRKLG7UEXAMPLE' }, 'requestParameters': { 'sourceIPAddress': '127.0.0.1' # TODO determine real source IP }, 'responseElements': { 'x-amz-request-id': short_uid(), 'x-amz-id-2': 'eftixk72aD6Ap51TnqcoF8eFidJG9Z/2' # Amazon S3 host that processed the request }, 's3': { 's3SchemaVersion': '1.0', 'configurationId': 'testConfigRule', 'bucket': { 'name': bucket_name, 'ownerIdentity': { 'principalId': 'A3NL1KOZZKExample' }, 'arn': 'arn:aws:s3:::%s' % bucket_name }, 'object': { 'key': file_name, 'size': file_size, 'eTag': 'd41d8cd98f00b204e9800998ecf8427e', 'versionId': '096fKKXTRTtl3on89fVO.nfljtsv6qko', 'sequencer': '0055AED6DCD90281E5' } } }] }
def create_sns_message_body(subscriber, req_data): message = req_data['Message'][0] subject = req_data.get('Subject', [None])[0] protocol = subscriber['Protocol'] if six.PY2 and type(message).__name__ == 'unicode': # fix non-ascii unicode characters under Python 2 message = message.encode('raw-unicode-escape') if subscriber.get('RawMessageDelivery') in ('true', 'True', True): return message if req_data.get('MessageStructure') == ['json']: message = json.loads(message) try: message = message.get(protocol, message['default']) except KeyError: raise Exception("Unable to find 'default' key in message payload") data = {} data['MessageId'] = str(uuid.uuid4()) data['Type'] = req_data.get('Type', ['Notification'])[0] data['Timestamp'] = timestamp(format=TIMESTAMP_FORMAT_MILLIS) data['Message'] = message data['TopicArn'] = subscriber['TopicArn'] if subject is not None: data['Subject'] = subject attributes = get_message_attributes(req_data) if attributes: data['MessageAttributes'] = attributes for attr in ['Token', 'SubscribeURL']: value = req_data.get(attr, [None])[0] if value: data[attr] = value result = json.dumps(data) return result
def fix_hardcoded_creation_date(response): # TODO: remove once this is fixed upstream search = r'<CreationTime>\s*(2011-05-23T15:47:44Z)?\s*</CreationTime>' replace = r'<CreationTime>%s</CreationTime>' % timestamp() response._content = re.sub(search, replace, to_str(response._content or '')) response.headers['Content-Length'] = str(len(response._content))
def dynamodb_stream_arn(table_name, account_id=None): account_id = get_account_id(account_id) return ('arn:aws:dynamodb:%s:%s:table/%s/stream/%s' % (get_local_region(), account_id, table_name, timestamp()))
def fix_hardcoded_creation_date(response): search = '<CreationTime>2011-05-23T15:47:44Z</CreationTime>' replace = '<CreationTime>%s</CreationTime>' % timestamp() response._content = to_str(response._content or '').replace(search, replace) response.headers['Content-Length'] = str(len(response._content))
def dynamodb_stream_arn(table_name, account_id=None): account_id = get_account_id(account_id) return ("arn:aws:dynamodb:%s:%s:table/%s/stream/%s" % (DEFAULT_REGION, account_id, table_name, timestamp()))
def dynamodb_stream_arn(table_name, account_id=None): account_id = get_account_id(account_id) return ('arn:aws:dynamodb:%s:%s:table/%s/stream/%s' % (get_local_region(), account_id, table_name, timestamp()))
def __init__(self, **kwargs): self.t = kwargs.get('timestamp') or kwargs.get('t') or timestamp() self.m_id = kwargs.get('machine_id') or kwargs.get('m_id') or get_machine_id() self.p_id = kwargs.get('process_id') or kwargs.get('p_id') or get_process_id() self.e_t = kwargs.get('event_type') or kwargs.get('e_t') self.p = kwargs.get('payload') if kwargs.get('payload') is not None else kwargs.get('p')