def test_can_create_event_from_valid_message_string(self):
json_string = json.dumps(valid_message_object())
event = event_from_json(json_string)
self.assertEqual(event.event_id, EVENT_ID)
self.assertEqual(event.event_type, EVENT_TYPE)
self.assertEqual(event.timestamp, TIMESTAMP)
self.assertEqual(event.originating_service, ORIGINATING_SERVICE)
self.assertEqual(event.session_id, SESSION_ID)
self.assertEqual(event.details['session_event_type'], SESSION_EVENT_TYPE)
def test_throws_validation_exception_if_required_element_is_missing(self):
required_elements = [
'eventId',
'eventType',
'timestamp',
'originatingService',
'details',
]
for element in required_elements:
message_object = valid_message_object()
message_object.pop(element)
json_string = json.dumps(message_object)
with self.assertRaises(ValueError) as raised_exception:
event_from_json(json_string)
self.assertEqual(
str(raised_exception.exception),
'Invalid Message. Missing required field "{0}"'.format(element)
)
def test_should_handle_error_message_with_session_id_in_json_string(self):
message_object = valid_error_message_object_with_session_id()
json_string = json.dumps(message_object)
event = event_from_json(json_string)
self.assertEqual(event.event_id, EVENT_ID)
self.assertEqual(event.event_type, 'error_event')
self.assertEqual(event.timestamp, TIMESTAMP)
self.assertEqual(event.originating_service, ORIGINATING_SERVICE)
self.assertEqual(event.session_id, SESSION_ID)
self.assertEqual(event.details['session_event_type'], SESSION_EVENT_TYPE)
def test_ignores_additional_elements_in_json_string(self):
message_object = valid_message_object()
message_object['foo'] = 'bar'
json_string = json.dumps(message_object)
event = event_from_json(json_string)
self.assertEqual(event.event_id, EVENT_ID)
self.assertEqual(event.event_type, EVENT_TYPE)
self.assertEqual(event.timestamp, TIMESTAMP)
self.assertEqual(event.originating_service, ORIGINATING_SERVICE)
self.assertEqual(event.session_id, SESSION_ID)
self.assertEqual(event.details['session_event_type'], SESSION_EVENT_TYPE)
def store_queued_events(_, __):
sqs_client = boto3.client('sqs')
queue_url = os.environ['QUEUE_URL']
db_connection = create_db_connection()
decryption_key = fetch_decryption_key()
while True:
message = fetch_single_message(sqs_client, queue_url)
if message is None:
break
# noinspection PyBroadException
# catch all errors and log them - we never want a single failing message to kill the process.
try:
decrypted_message = decrypt_message(message['Body'], decryption_key)
event = event_from_json(decrypted_message)
write_to_database(event, db_connection)
delete_message(sqs_client, queue_url, message)
except Exception as exception:
logging.getLogger('event-recorder').exception('Failed to store message')
def test_throws_validation_exception_if_string_is_not_valid_json(self):
with self.assertRaises(JSONDecodeError):
event_from_json('not valid')
def store_queued_events(_, __):
sqs_client = boto3.client('sqs')
queue_url = os.environ['QUEUE_URL']
logger = logging.getLogger('event-recorder')
logger.setLevel(logging.INFO)
if 'ENCRYPTION_KEY' in os.environ:
encrypted_decryption_key = os.environ['ENCRYPTION_KEY']
logger.info('Got decryption key from environment variable')
else:
encrypted_decryption_key = fetch_decryption_key()
logger.info('Got decryption key from S3')
decryption_key = decrypt(encrypted_decryption_key)
logger.info('Decrypted key successfully')
dsn = os.environ['DB_CONNECTION_STRING']
database_password = None
if 'ENCRYPTED_DATABASE_PASSWORD' in os.environ:
# boto returns decrypted as b'bytes' so decode to convert to password string
database_password = decrypt(
os.environ['ENCRYPTED_DATABASE_PASSWORD']).decode()
else:
dsn_components = parse_dsn(dsn)
database_password = boto3.client('rds').generate_db_auth_token(
dsn_components['host'], 5432, dsn_components['user'])
db_connection = create_db_connection(dsn, database_password)
logger.info('Created connection to DB')
event_count = 0
while True:
message = fetch_single_message(sqs_client, queue_url)
if message is None:
logger.info('Queue is empty - finishing after {0} events'.format(
event_count))
break
event_count += 1
# noinspection PyBroadException
# catch all errors and log them - we never want a single failing message to kill the process.
event = None
try:
decrypted_message = decrypt_message(message['Body'],
decryption_key)
event = event_from_json(decrypted_message)
logger.info('Decrypted event with ID: {0}'.format(event.event_id))
write_audit_event_to_database(event, db_connection)
logger.info('Stored audit event: {0}'.format(event.event_id))
if event.event_type == 'session_event' and event.details.get(
'session_event_type') == 'idp_authn_succeeded':
write_billing_event_to_database(event, db_connection)
logger.info('Stored billing event: {0}'.format(event.event_id))
if event.event_type == 'session_event' and event.details.get(
'session_event_type') == 'fraud_detected':
write_fraud_event_to_database(event, db_connection)
logger.info('Stored fraud event: {0}'.format(event.event_id))
# really don't want the event system to fail because of Splunk logging
try:
splunk_res = push_event_to_splunk(decrypted_message)
except Exception as e:
splunk_res = False
if splunk_res and splunk_res[0] == 200:
# log successfully pushed events
logger.info('Pushed fraud event to Splunk: {0}'.format(
event.event_id))
elif 'production' in os.environ['QUEUE_URL']:
# log unsuccessful push events as errors but don't raise an exception
# this way, if Splunk was down, the event system still works as expected
logger.error(
'Failed to push fraud event to Splunk: {0}'.format(
event.event_id))
delete_message(sqs_client, queue_url, message)
logger.info('Deleted event from queue with ID: {0}'.format(
event.event_id))
except Exception as exception:
if event:
logger.exception(
'Failed to store event {0}, event type "{1}" from SQS message ID {2}'
.format(event.event_id, event.event_type,
message['MessageId']))
else:
logger.exception(
'Failed to decrypt message, SQS ID = {0}'.format(
message['MessageId']))
def store_queued_events(_, __):
sqs_client = boto3.client('sqs')
queue_url = os.environ['QUEUE_URL']
logger = logging.getLogger('event-recorder')
logger.setLevel(logging.INFO)
if 'ENCRYPTION_KEY' in os.environ:
encrypted_decryption_key = os.environ['ENCRYPTION_KEY']
logger.info('Got decryption key from environment variable')
else:
encrypted_decryption_key = fetch_decryption_key()
logger.info('Got decryption key from S3')
decryption_key = decrypt(encrypted_decryption_key)
logger.info('Decrypted key successfully')
dsn = os.environ['DB_CONNECTION_STRING']
db_connection = create_db_connection(dsn, get_database_password(dsn))
logger.info('Created connection to DB')
event_count = 0
while True:
message = fetch_single_message(sqs_client, queue_url)
if message is None:
logger.info('Queue is empty - finishing after {0} events'.format(event_count))
break
event_count += 1
# noinspection PyBroadException
# catch all errors and log them - we never want a single failing message to kill the process.
event = None
try:
decrypted_message = decrypt_message(message['Body'], decryption_key)
event = event_from_json(decrypted_message)
# Send audit events to this lambda function's CloudWatch log group.
# This is the raw JSON event on a line by its self so Splunk can
# parse it as JSON.
print(decrypted_message)
logger.info('Decrypted event with ID: {0}'.format(event.event_id))
write_audit_event_to_database(event, db_connection)
logger.info('Stored audit event: {0}'.format(event.event_id))
if event.event_type == 'session_event' and event.details.get('session_event_type') == 'idp_authn_succeeded':
write_billing_event_to_database(event, db_connection)
logger.info('Stored billing event: {0}'.format(event.event_id))
if event.event_type == 'session_event' and event.details.get('session_event_type') == 'fraud_detected':
write_fraud_event_to_database(event, db_connection)
logger.info('Stored fraud event: {0}'.format(event.event_id))
delete_message(sqs_client, queue_url, message)
logger.info('Deleted event from queue with ID: {0}'.format(event.event_id))
except Exception:
if event:
logger.exception(
'Failed to store event {0}, event type "{1}" from SQS message ID {2}'.format(event.event_id,
event.event_type,
message['MessageId']))
else:
logger.exception('Failed to decrypt message, SQS ID = {0}'.format(message['MessageId']))