def lambda_handler(*args, **kwargs): event = args[0] context = args[1] aws_lambda_logging.setup(level=os.environ.get('LOGLEVEL', 'INFO'), aws_request_id=context.aws_request_id, boto_level='CRITICAL') received_raw_kinesis_records = event['Records'] for raw_kinesis_records in chunks( iter_deaggregate_records(received_raw_kinesis_records), batch_size): kinesis_records: list = [] for raw_kinesis_record in raw_kinesis_records: kinesis_record = KinesisRecord(raw_kinesis_record) if kinesis_record.is_any_of(event_types): kinesis_records.append(kinesis_record) if kinesis_records: log.info({ "Action": "Processing", "Events": [ kinesis_record.get_type() for kinesis_record in kinesis_records ] }) results = func(kinesis_records, context) if results: log.info({"Results": results})
def lambda_handler(event, context): aws_lambda_logging.setup(level=log_level, aws_request_id=context.aws_request_id) dataframes = [] # download files locally for filename in event: data_file = download_data(filename) # read each file and store as Pandas dataframe with gzip.open(data_file, 'rb') as ndjson_file: records = map(json.loads, ndjson_file) df = pd.DataFrame.from_records(json_normalize(records)) dataframes.append(df) # process the data to get air quality readings parameter_readings = process_data(dataframes) # write to file results_filename = "{}.json.gz".format(context.aws_request_id) parameter_readings.to_json(os.path.join('/tmp', results_filename), compression='gzip') # upload to target S3 bucket upload_intermediate_results(results_filename) # return temp file and number of rows processed. return { "message": "Mapper phase complete.", "processed_file": 'lambda-etl-refarch/temp/{}'.format(results_filename), "rows": len(parameter_readings) }
def handler(event, context): """Main function handler. Args: event: AWS event context: Lambda context """ # Set up logging aws_lambda_logging.setup(level=LOG_LEVEL, boto_level="CRITICAL") logger.debug(event) # Set up S3 client s3 = boto3.client("s3", region_name=MAIL_BUCKET_REGION) attachment_prefixes_serialized = event["Records"][0]["Sns"][ "MessageAttributes"]["attachments"]["Value"] attachment_prefixes = json.loads(attachment_prefixes_serialized) for prefix in attachment_prefixes: try: file_name = prefix.split("/")[-1] file_name = f"/tmp/{file_name}" with open(file_name, "wb") as fp: s3.download_fileobj(MAIL_BUCKET, prefix, fp) except Exception as e: logging.error(f"{repr(e)}") raise e key_map, value_map, block_map = get_kv_map(file_name) # Get Key Value relationship kvs = get_kv_relationship(key_map, value_map, block_map) o = marshal_response(kvs) logging.info(f"{file_name}: {o}")
def test_setup_with_invalid_log_level(root_logger, logger, stdout): from aws_lambda_logging import setup setup('not a valid log level') # writes a log event log_dict = json.loads(stdout.getvalue()) check_log_dict(log_dict)
def handler_example3(event, context): aws_lambda_logging.setup(level='INFO', This_Is_A_Custom_Key="This is the custom keys value") logger.info('This is example number 3') # Example 4 - Basic log entry with varying log levels def handler_example4(event, context): aws_lambda_logging.setup(level='INFO') logger.info('This is example number 4 - INFO level') logger.warning('This is example number 4 - WARNING level') logger.error('This is example number 4 - ERROR level') logger.debug("This is example number 4 - DEBUG level which you won't see in the output") # Example 5 - JSON String log entry def handler_example5(event, context): aws_lambda_logging.setup(level='INFO') logger.info('{"Example 5": [1,2,3]}') # Example 6 - Variant boto logging level def handler_example6(event, context): aws_lambda_logging.setup(level=loglevel, boto_level=botologlevel)
def lambda_handler(*args, **kwargs): event = args[0] context = args[1] aws_lambda_logging.setup(level=os.environ.get('LOGLEVEL', 'INFO'), aws_request_id=context.aws_request_id, boto_level='CRITICAL') received_raw_dynamo_records = event['Records'] for raw_dynamo_records in chunks(received_raw_dynamo_records, batch_size): dynamo_records: list = [] for raw_dynamo_record in raw_dynamo_records: dynamo_record = DynamoRecord(raw_dynamo_record) if dynamo_record.is_any_of(event_types): dynamo_records.append(dynamo_record) if dynamo_records: log.info({ "Action": "Processing", "Event": [ dynamo_record.get_type() for dynamo_record in dynamo_records ] }) result = func(dynamo_records, context) if result: log.info({"Result": result})
def handler(event, context): """Handler for drm-filter-api""" aws_lambda_logging.setup(level=os.environ.get('LOGLEVEL', 'INFO'), env=os.environ.get('ENV')) try: logging.debug(json.dumps({'message': 'logging event', 'event': event})) except: logging.exception(json.dumps({'message': 'logging event'})) raise try: request = json.loads(event['body']) logging.debug( json.dumps({ 'message': "decoding message", "request": request })) except: logging.exception(json.dumps({'message': "decoding message"})) response = { "statusCode": 400, "body": json.dumps( {"error": "Could not decode request: JSON parsing failed"}), 'headers': { 'Content-Type': 'application/json', } } return response try: body = filter_drm(request) logging.debug( json.dumps({ 'message': "filtering response", "body": body })) except: logging.exception(json.dumps({'message': "filtering response"})) response = { "statusCode": 503, "body": json.dumps({"error": "Failed to filter payload"}), 'headers': { 'Content-Type': 'application/json', } } return response response = { "statusCode": 200, "body": json.dumps(body), 'headers': { 'Content-Type': 'application/json', } } logging.info(json.dumps({'message': 'responding', 'response': response})) return response
def main(): aws_lambda_logging.setup(level=os.environ.get('LOGLEVEL', 'INFO'), env=os.environ.get('ENV'), timestamp=int(time.time())) aws_lambda_logging.setup(level=os.environ.get('LOGLEVEL', 'INFO'), env=os.environ.get('ENV'), timestamp=int(time.time())) unittest.main(verbosity=2)
def handler(event, context): aws_lambda_logging.setup(level='DEBUG', aws_request_id=context.aws_request_id) random_number = random.randrange(100) + 1 log.info({"Number": random_number}) return(random_number)
def test_with_dict_message(root_logger, logger, stdout): from aws_lambda_logging import setup setup('DEBUG', another='value') msg = {'x': 'isx'} logger.critical(msg) log_dict = json.loads(stdout.getvalue()) assert msg == log_dict['message']
def handler(event, context): aws_lambda_logging.setup(level="INFO") logger.info("event: " + json.dumps(event)) pp = pprint.PrettyPrinter(indent=4) logger.info("context") logger.info(pp.pprint(context)) sqs = boto3.resource("sqs") queue = sqs.get_queue_by_name(QueueName="ImageSyncJobQueue") logger.info(queue) queue.send_message(MessageBody="testing testing 123") return {"message": "Hello, World!"}
def log_handler(event: dict, context) -> None: aws_lambda_logging.setup(level="INFO") log.info(event) s3_event: S3Event = S3Event.from_dict(event) for record in s3_event.records: bucket = record.s3.bucket.name key = record.s3.object.key log.info(f"Processing {bucket}/{key}") SHIPPER.ship(bucket, key)
def lambda_handler(event, context): """ AWS lambda insertion point. :param event: AWS Lambda event data :param context: AWS Lambda context :return: Service response """ aws_lambda_logging.setup(level='INFO', boto_level='INFO') LOGGER.info(f'Micro Airlines API {__version__}') LOGGER.info({'event': event}) return awsgi.response(app, event, context)
def lambda_handler(event, context): aws_lambda_logging.setup(level=log_level, aws_request_id=context.aws_request_id) log.info(f"Processing data for: {prev_day}") file_names = get_file_inventory() chunks = [ file_names[i:i + chunk_size] for i in range(0, len(file_names), chunk_size) ] return {"chunks": chunks, "message": "Init phase complete"}
def main(): aws_lambda_logging.setup(level=os.environ.get('LOGLEVEL', 'INFO'), boto_level='CRITICAL') methods = [make_errors, make_warns, make_fatals, make_info] scheduler = sched.scheduler(time.time, time.sleep) while True: try: scheduler.enter(0.5, 1, choice(methods)) scheduler.enter(0.5, 1, choice(methods)) scheduler.run() except (KeyboardInterrupt, SystemExit): break
def test_logging_exception_traceback(root_logger, logger, stdout): from aws_lambda_logging import setup setup('DEBUG', request_id='request id!', another='value') try: raise Exception('Boom') except Exception: logger.exception('This is a test') log_dict = json.loads(stdout.getvalue()) check_log_dict(log_dict) assert 'exception' in log_dict
def test_with_unserialisable_value_in_message(root_logger, logger, stdout): from aws_lambda_logging import setup setup('DEBUG', another='value') class X: pass msg = {'x': X()} logger.critical(msg) log_dict = json.loads(stdout.getvalue()) assert log_dict['message']['x'].startswith('<')
def lambda_handler(event, context): aws_lambda_logging.setup(level=log_level, aws_request_id=context.aws_request_id) for item in event: intermediate_files = event['intermediate_files'] # delete from S3 bucket delete_intermediate_results(intermediate_files) return { "message": event["message"], "results": f'Download results from {event["output_file"]}' }
def logger_setup(service: str = "service_undefined", level: str = "INFO", **kwargs): """Setups root logger to format statements in JSON. Includes service name and any additional key=value into logs It also accepts both service name or level explicitly via env vars Environment variables --------------------- POWERTOOLS_SERVICE_NAME : str service name LOG_LEVEL: str logging level (e.g. INFO, DEBUG) Parameters ---------- service : str, optional service name to be appended in logs, by default "service_undefined" level : str, optional logging.level, by default "INFO" Example ------- Setups structured logging in JSON for Lambda functions with explicit service name >>> from lambda_python_powertools.logging import logger_setup >>> logger = logger_setup(service="payment") >>> >>> def handler(event, context): logger.info("Hello") Setups structured logging in JSON for Lambda functions using env vars $ export POWERTOOLS_SERVICE_NAME="payment" >>> from lambda_python_powertools.logging import logger_setup >>> logger = logger_setup() >>> >>> def handler(event, context): logger.info("Hello") """ service = os.getenv("POWERTOOLS_SERVICE_NAME") or service log_level = os.getenv("LOG_LEVEL") or level logger = logging.getLogger(name=service) logger.setLevel(log_level) # Patch logger by structuring its outputs as JSON aws_lambda_logging.setup(level=log_level, service=service, **kwargs) return logger
def handler(event, context): aws_lambda_logging.setup(level="INFO") logger.info("event: " + json.dumps(event)) pp = pprint.PrettyPrinter(indent=4) logger.info(pp.pprint(context)) dynamodb: DynamodbClient = boto3.client("dynamodb") s3: S3Client = boto3.client("s3") s3key = uuid.uuid4() logger.info("buckets") logger.info(pp.pprint(s3.list_buckets())) logger.info("dynamodb") logger.info(pp.pprint(dynamodb.list_tables())) logger.info("s3key: " + s3key) return {"message": "Hello, World!"}
def test_setup_with_valid_log_levels(root_logger, logger, stdout, level): from aws_lambda_logging import setup setup(level, request_id='request id!', another='value') logger.critical('This is a test') log_dict = json.loads(stdout.getvalue()) check_log_dict(log_dict) assert 'CRITICAL' == log_dict['level'] assert 'This is a test' == log_dict['message'] assert 'request id!' == log_dict['request_id'] assert 'exception' not in log_dict
def auth_handler(event, context): aws_lambda_logging.setup(level=os.environ.get('LOGLEVEL', 'INFO'), aws_request_id=context.aws_request_id, boto_level='CRITICAL') authorization_user = '******' authorization_method = 'N/A' try: authorization_header = event['authorizationToken'] if 'TOKEN' == event[ 'type'] else event['headers']['Authorization'] if authorization_header: authorization_method, authorization_value = authorization_header.split( ' ') authorization_user, authorization_password = base64.b64decode( authorization_value).decode('utf-8').split(':') parameter = get_ssm_param(authorization_user) if parameter['Value'] == authorization_password: log.info({ 'Method': authorization_method, 'User': authorization_user, 'Effect': 'Allow' }) effective_resource = re.match( '(arn:aws:execute-api:[^:]+:[^:]+:[^/]+/[^/]+).*', event['methodArn'])[1] policy = { 'principalId': authorization_user, 'policyDocument': { 'Version': '2012-10-17', 'Statement': [{ 'Action': 'execute-api:Invoke', 'Effect': 'Allow', 'Resource': f'{effective_resource}/*' }] } } log.info({'Authorization': 'Success', 'Policy': policy}) return policy except Exception as e: log.error(e) # authorization failed log.info({ 'Method': authorization_method, 'User': authorization_user, 'Effect': 'Deny' }) raise Exception('Unauthorized')
def handler(event, context): """ Logs every record in the stream """ aws_lambda_logging.setup(level=os.environ.get('LOGLEVEL', 'INFO'), aws_request_id=context.aws_request_id, boto_level='CRITICAL') raw_kinesis_records = event['Records'] for kinesis_record in iter_deaggregate_records(raw_kinesis_records): try: kinesis_record['kinesis']['data'] = json.loads( base64.b64decode(kinesis_record['kinesis']['data'])) except json.JSONDecodeError: pass log.info(kinesis_record)
def handler(event: dict, context): """ Listen to CloudTrail Events and publishes them to Kinesis """ aws_lambda_logging.setup(level=os.environ.get('LOGLEVEL', 'INFO'), aws_request_id=context.aws_request_id, boto_level='CRITICAL') log.info(event) source = event.get('source') if source: expr = source_path_map.get(source) if expr: values = [match.value for match in expr.find(event)] if values: log.debug(values[0])
def handler(event, context): """ Minimalistic almost-passthrough Firehose record processor """ aws_lambda_logging.setup(level=os.environ.get('LOGLEVEL', 'INFO'), aws_request_id=context.aws_request_id, boto_level='CRITICAL') received_raw_firehose_records = event['records'] log.info(f'Received {len(received_raw_firehose_records)} events') return { 'records': [ transform(firehose_record) for firehose_record in received_raw_firehose_records ] }
def test_connectivity(event, context): loglevel = os.environ.get('LOGLEVEL', 'DEBUG') correlation_id = context.aws_request_id aws_lambda_logging.setup(level=loglevel, correlation_id=correlation_id) try: aws_lambda_logging.setup(env=os.environ.get('ENV')) except: pass logging.debug(json.dumps({'event': event})) url = os.environ['CENTREON_URL'] useralias = os.environ['CENTREON_USERALIAS'] password = os.environ['CENTREON_PASSWORD'] jar = get_login(url, useralias, password, correlation_id) logout(url, jar, correlation_id)
def __init__(self, connection=None, loglevel="INFO"): self.logger = logging.getLogger() # loglevel = "INFO" logging.basicConfig(level=logging.ERROR) aws_lambda_logging.setup(level=loglevel) if connection is None: host = os.environ.get('MYSQL_HOST') username = os.environ.get('MYSQL_USER') password = os.environ.get('MYSQL_PASSWORD') database = os.environ.get('MYSQL_DATABASE') self.connection = pymysql.connect(host, user=username, passwd=password, db=database, cursorclass=pymysql.cursors.DictCursor) self.logger.debug("!!!!!!!!!!new connection created") else: self.connection = connection self.table_name = None
def handler(event, context): aws_lambda_logging.setup( level=os.environ.get('LOGLEVEL', 'INFO'), aws_request_id=context.aws_request_id, boto_level='CRITICAL' ) log.info(event) now = datetime.now() key = now.strftime(OBJECT_PATTERN) try: s3.get_object( Bucket=BUCKET, Key=key ) log.info(f'Found s3://{BUCKET}/{key}') except s3.exceptions.NoSuchKey: log.error(f'Missing object at s3://{BUCKET}/{key}')
def handler_example4(event, context): aws_lambda_logging.setup(level='INFO') logger.info('This is example number 4 - INFO level') logger.warning('This is example number 4 - WARNING level') logger.error('This is example number 4 - ERROR level') logger.debug("This is example number 4 - DEBUG level which you won't see in the output") # Example 5 - JSON String log entry def handler_example5(event, context): aws_lambda_logging.setup(level='INFO') logger.info('{"Example 5": [1,2,3]}') # Example 6 - Variant boto logging level def handler_example6(event, context): aws_lambda_logging.setup(level=loglevel, boto_level=botologlevel)
def wrapped_func(event, context): extra_info = {'aws_request_id': context.aws_request_id} aws_lambda_logging.setup(level=LOG_LEVEL, boto_level=BOTO_LOG_LEVEL, **extra_info) logger = logging.getLogger() logger.debug("{} invoked!".format(context.function_name)) logger.debug({'event': event, 'context': context.__dict__}) try: retval = handler_func(event, context) except Exception: logger.exception("handler failed!") raise logger.debug("{} complete!".format(context.function_name)) return retval