def test_logger_persist_emit():
logger = ServerlessLogger()
logger.persist = True
with logger.context(bind={"foo": "bar"}):
logger.log("TEST")
body = {"logs": logger.events}
emit_logs(body)
def test_logger_persist_events_context():
logger = ServerlessLogger()
logger.persist = True
with logger.context(bind={"foo": "bar"}):
logger.log("TEST")
for e in logger.events:
# {"level": level, "event": event, "context": self._logger._context}
assert e["event"] == "TEST"
assert e["context"] == {"foo": "bar"}
def execute_payload(
payload: Payload,
path_enum: EnumMeta,
paths: dict,
logger,
event,
context,
debug=False,
):
"""Execute functions, paths, and shortcuts in a Path.
Args:
payload (Payload):
path_enum (EnumMeta): An Enum class which define the possible paths available in this lambda.
paths (dict): Keys are path names / enums and values are a list of Action objects
logger:
event: https://docs.aws.amazon.com/lambda/latest/dg/python-handler.html
context: https://docs.aws.amazon.com/lambda/latest/dg/python-handler.html
"""
if not logger:
logger = ServerlessLogger()
ret = None
if not isinstance(payload.path, path_enum):
payload.path = clean_path(path_enum, payload.path)
if isinstance(payload.path, Enum): # PATH
# Allow someone to simplify their definition of a Path to a list of functions.
if all([isinstance(f, FunctionType) for f in paths[payload.path]]):
paths[payload.path] = [Action(functions=action)]
for action in paths[payload.path]:
assert isinstance(action, Action)
# Build action kwargs and validate type hints
try:
dummy = ["logger", "event"]
action_kwargs = build_action_kwargs(action, {
**{k: None
for k in dummy},
**payload.kwargs
})
for k in dummy:
action_kwargs.pop(k, None)
except (TypeError, AssertionError) as e:
raise InvalidPayloadError(
f"Failed to run {payload.path.name} {action} due to {exception_to_str(e)}"
) from e
# Run action functions
for f in action.functions:
assert isinstance(f, FunctionType)
try:
# TODO: if ret, set _last_output
_log_context = {
"path": payload.path.name,
"function": f.__name__
}
with logger.context(bind={
**_log_context, "kwargs": action_kwargs
}):
logger.log("Executing function.")
with logger.context(bind=_log_context):
ret = f(
**{
**action_kwargs,
"logger": logger,
"event": {
"event": event,
"context": context,
"payload": payload,
},
})
if ret:
_payloads = []
try:
if isinstance(ret, Payload):
_payloads.append(ret.validate(path_enum))
elif isinstance(ret, list):
for r in ret:
if isinstance(r, Payload):
_payloads.append(r.validate(path_enum))
except Exception as e:
logger.debug(exception_to_str(e))
raise FailButContinue(
f"Something went wrong while extracting Payloads from a function return value. {ret}"
) from e
for p in _payloads:
logger.debug(
f"Function returned a Payload. Executing. {p}")
try:
ret = execute_payload(p, path_enum, paths,
logger, event, context,
debug)
except Exception as e:
logger.debug(exception_to_str(e))
raise FailButContinue(
f"Failed to execute returned Payload. {p}"
) from e
except LpipeBaseException:
# CAPTURES:
# FailButContinue
# FailCatastrophically
raise
except Exception as e:
logger.error(
f"Skipped {payload.path.name} {f.__name__} due to unhandled Exception. This is very serious; please update your function to handle this. Reason: {exception_to_str(e)}"
)
sentry.capture(e)
if debug:
raise FailCatastrophically() from e
# Run action paths / shortcuts
for path_descriptor in action.paths:
_payload = Payload(
clean_path(path_enum, path_descriptor),
action_kwargs,
payload.event_source,
).validate(path_enum)
ret = execute_payload(_payload, path_enum, paths, logger,
event, context, debug)
elif isinstance(payload.path, Queue): # SHORTCUT
queue = payload.path
assert isinstance(queue.type, QueueType)
with logger.context(
bind={
"path": queue.path,
"queue_type": queue.type,
"queue_name": queue.name,
"kwargs": payload.kwargs,
}):
logger.log("Pushing record.")
put_record(queue=queue,
record={
"path": queue.path,
"kwargs": payload.kwargs
})
else:
logger.info(
f"Path should be a string (path name), Path (path Enum), or Queue: {payload.path})"
)
return ret
def process_event(
event,
context,
paths: dict,
queue_type: QueueType,
path_enum: EnumMeta = None,
default_path=None,
logger=None,
debug=False,
):
"""Process an AWS Lambda event.
Args:
event: https://docs.aws.amazon.com/lambda/latest/dg/python-handler.html
context: https://docs.aws.amazon.com/lambda/latest/dg/python-handler.html
paths (dict): Keys are path names / enums and values are a list of Action objects
queue_type (QueueType): The event source type.
path_enum (EnumMeta): An Enum class which define the possible paths available in this lambda.
default_path: A string or Enum which will be run for every message received.
logger:
debug (bool):
"""
try:
if not logger:
logger = ServerlessLogger(
level=logging.DEBUG if debug else logging.INFO,
process=getattr(context, "function_name",
config("FUNCTION_NAME", default=None)),
)
if debug and isinstance(logger, ServerlessLogger):
logger.persist = True
except Exception as e:
raise InvalidConfigurationError("Failed to initialize logger.") from e
logger.debug(f"Event received. queue: {queue_type}, event: {event}")
try:
assert isinstance(queue_type, QueueType)
except AssertionError as e:
raise InvalidConfigurationError(
f"Invalid queue type '{queue_type}'") from e
if not path_enum:
try:
path_enum = Enum("AutoPath", [k.upper() for k in paths.keys()])
paths = {clean_path(path_enum, k): v for k, v in paths.items()}
except KeyError as e:
raise InvalidConfigurationError from e
successful_records = []
records = get_records_from_event(queue_type, event)
try:
assert isinstance(records, list)
except AssertionError as e:
logger.error(f"'records' is not a list {exception_to_str(e)}")
return build_response(0, 0, logger)
output = []
exceptions = []
for encoded_record in records:
ret = None
try:
try:
_payload = get_payload_from_record(
queue_type=queue_type,
record=encoded_record,
validate=False if default_path else True,
)
_path = default_path if default_path else _payload["path"]
_kwargs = _payload if default_path else _payload["kwargs"]
_event_source = get_event_source(queue_type, encoded_record)
payload = Payload(_path, _kwargs,
_event_source).validate(path_enum)
except AssertionError as e:
raise InvalidPayloadError(
"'path' or 'kwargs' missing from payload.") from e
except TypeError as e:
raise InvalidPayloadError(
f"Bad record provided for queue type {queue_type}. {encoded_record} {exception_to_str(e)}"
) from e
with logger.context(bind={"payload": payload.to_dict()}):
logger.log(f"Record received.")
# Run your path/action/functions against the payload found in this record.
ret = execute_payload(
payload=payload,
path_enum=path_enum,
paths=paths,
logger=logger,
event=event,
context=context,
debug=debug,
)
# Will handle any cleanup necessary for a successful record later.
successful_records.append(encoded_record)
except FailButContinue as e:
# CAPTURES:
# InvalidPayloadError
# InvalidPathError
logger.error(str(e))
sentry.capture(e)
continue # User can say "bad thing happened but keep going." This drops poisoned records on the floor.
except FailCatastrophically as e:
# CAPTURES:
# InvalidConfigurationError
# raise (later)
exceptions.append({"exception": e, "record": encoded_record})
output.append(ret)
response = build_response(n_records=len(records),
n_ok=len(successful_records),
logger=logger)
if exceptions:
# Handle any cleanup necessary for successful records before creating an error state.
advanced_cleanup(queue_type, successful_records, logger)
logger.info(
f"Encountered exceptions while handling one or more records. RESPONSE: {response}"
)
raise FailCatastrophically(exceptions)
if any(output):
response["output"] = output
if debug:
response["debug"] = json.dumps({"records": records}, cls=AutoEncoder)
return response
def test_encode_logger():
logger = ServerlessLogger()
json.dumps(logger, cls=AutoEncoder)
def test_logger_bind_unbind():
logger = ServerlessLogger()
logger.bind(foo="bar")
logger.unbind("foo")
def test_logger_context_action():
logger = ServerlessLogger()
with logger.context(bind={"foo": "bar"}, action="my_action"):
logger.log("TEST")
def test_create_logger():
logger = ServerlessLogger()
assert isinstance(logger, ServerlessLogger)
def test_create_logger_persist():
logger = ServerlessLogger()
logger.persist = True
assert isinstance(logger, ServerlessLogger)
def test_logger_persist_events():
logger = ServerlessLogger()
logger.persist = True
logger.log("TEST")
for e in logger.events:
assert e["event"] == "TEST"
def test_logger_log_critical():
logger = ServerlessLogger()
logger.critical("Test critical.")
def test_logger_log_error():
logger = ServerlessLogger()
logger.error("Test error.")
def test_logger_log_warning():
logger = ServerlessLogger()
logger.warning("Test warning.")
def test_logger_log_info():
logger = ServerlessLogger()
logger.info("Test info.")
def test_logger_log():
logger = ServerlessLogger()
logger.log("Test log.")