def handler(event, context):
"""Method invoked by Lambda event. Verifies that records were received and,
if so, passes them to be parsed"""
logger.debug('Starting Lambda Execution')
records = event.get('Records')
if records is None:
logger.error('Records block is missing in SQS Message')
raise NoRecordsReceived('Records block missing', event)
elif len(records) < 1:
logger.error('Records block contains no records')
raise NoRecordsReceived('Records block empty', event)
results = parseRecords(records)
logger.info('Successfully invoked lambda')
# This return will be reflected in the CloudWatch logs
# but doesn't actually do anything
return results
def handler(event, context):
"""Central handler invoked by Lambda trigger. Begins processing of kinesis
stream.
"""
logger.debug('Starting Lambda Execution')
records = event.get('Records')
if records is None:
logger.error('Records block is missing in Kinesis Event')
raise NoRecordsReceived('Records block missing', event)
elif len(records) < 1:
logger.error('Records block contains no records')
raise NoRecordsReceived('Records block empty', event)
results = parseRecords(records)
logger.info('Successfully invoked lambda')
# This return will be reflected in the CloudWatch logs
# but doesn't actually do anything
return results
def parseRecord(encodedRec, updater):
"""Handles each individual record by parsing JSON from the base64 encoded
string recieved from the Kinesis stream, creating a database session and
inserting/updating the database to reflect this new data source. It will
rollback changes if an error is encountered
"""
try:
record = json.loads(base64.b64decode(encodedRec['kinesis']['data']))
statusCode = record['status']
if statusCode != 200:
if statusCode == 204:
logger.info('No updates received')
raise NoRecordsReceived(
'No records received from {}'.format(record['source']),
record
)
else:
logger.error('Received error from pipeline')
logger.debug(record)
raise DataError('Received non-200 status code')
except json.decoder.JSONDecodeError as jsonErr:
logger.error('Invalid JSON block received')
logger.error(jsonErr)
raise DataError('Invalid JSON block')
except (UnicodeDecodeError, binascii.Error) as b64Err:
logger.error('Invalid data found in base64 encoded block')
logger.debug(b64Err)
raise DataError('Error in base64 encoding of record')
outRec = None
try:
MANAGER.startSession() # Start transaction
outRec = updater.importRecord(deepcopy(record))
MANAGER.commitChanges()
except OperationalError as opErr:
logger.error('Conflicting updates caused deadlock, retry')
logger.debug(opErr)
OutputManager.putKinesis(
record.get('data'),
os.environ['UPDATE_STREAM'],
recType=record.get('type', 'work'),
)
MANAGER.session.rollback() # Rollback current record only except
except IntegrityError as intErr:
logger.error('Unique constraint violated, retry')
logger.debug(intErr)
OutputManager.putKinesis(
record.get('data'),
os.environ['UPDATE_STREAM'],
recType=record.get('type', 'work'),
)
MANAGER.session.rollback() # Rollback current record only
except Exception as err: # noqa: Q000
# There are a large number of SQLAlchemy errors that can be thrown
# These should be handled elsewhere, but this should catch anything
# and rollback the session if we encounter something unexpected
logger.error('Failed to store record')
logger.debug(err)
logger.debug(traceback.format_exc())
MANAGER.session.rollback() # Rollback current record only
return outRec
def test_NoRecordsReceived(self):
testNoRecs = NoRecordsReceived('testMessage', 'testInvocation')
assert testNoRecs.message == 'testMessage'
assert testNoRecs.invocation == 'testInvocation'