def parse_record(state: State,
record: Any,
event_source: str,
default_path: Union[str, Enum] = None) -> Payload:
try:
kwargs = {"event_source": event_source}
if not default_path:
for field in ["path", "kwargs"]:
assert field in record
kwargs.update({"path": record["path"], "kwargs": record["kwargs"]})
else:
kwargs.update({"path": default_path, "kwargs": record})
return Payload(**kwargs).validate(state.path_enum)
except AssertionError as e:
raise lpipe.exceptions.InvalidPayloadError(
"'path' or 'kwargs' missing from payload.") from e
def execute_payload(payload: Payload, state: State) -> Any:
"""Given a Payload, execute Actions in a Path and fire off messages to the payload's Queues.
Args:
payload (Payload):
state (State):
"""
ret = None
if payload.path is not None and not isinstance(payload.path,
state.path_enum):
payload.path = normalize.normalize_path(state.path_enum, payload.path)
if isinstance(payload.path, Enum): # PATH
state.paths[payload.path] = normalize.normalize_actions(
state.paths[payload.path])
for action in state.paths[payload.path]:
ret = execute_action(payload=payload, action=action, state=state)
elif isinstance(payload.queue, Queue): # QUEUE (aka SHORTCUT)
queue = payload.queue
assert isinstance(queue.type, QueueType)
if queue.path:
record = {"path": queue.path, "kwargs": payload.kwargs}
else:
record = payload.kwargs
with state.logger.context(
bind={
"path": queue.path,
"queue_type": queue.type,
"queue_name": queue.name,
"record": record,
}):
state.logger.log("Pushing record.")
put_record(queue=queue, record=record)
else:
state.logger.info(
f"Path should be a string (path name), Path (path Enum), or Queue: {payload.path})"
)
return ret
def process_event(
event,
context,
queue_type: QueueType,
paths: dict = None,
path_enum: EnumMeta = None,
default_path: Union[str, Enum] = None,
call: FunctionType = None,
logger=None,
debug: bool = False,
exception_handler: FunctionType = None,
) -> dict:
"""Process an AWS Lambda event.
Args:
event: https://docs.aws.amazon.com/lambda/latest/dg/python-handler.html
context: https://docs.aws.amazon.com/lambda/latest/dg/python-handler.html
queue_type (QueueType): The event source type.
paths (dict): Keys are path names / enums and values are a list of Action objects
path_enum (EnumMeta): An Enum class which define the possible paths available in this lambda.
default_path (Union[str, Enum]): The path to be run for every message received.
call (FunctionType): A callable which, if set and `paths` is not, will disable directed-graph workflow features and default to calling this
logger:
debug (bool):
exception_handler (FunctionType): A function which will be used to capture exceptions (e.g. contrib.sentry.capture)
"""
logger = lpipe.logging.setup(logger=logger, context=context, debug=debug)
logger.debug(f"Event received. queue: {queue_type}, event: {event}")
try:
assert isinstance(queue_type, QueueType)
except AssertionError as e:
raise lpipe.exceptions.InvalidConfigurationError(
f"Invalid queue type '{queue_type}'") from e
if isinstance(call, FunctionType):
if not paths:
default_path = "AUTO_PATH"
paths = {default_path: [call]}
else:
raise lpipe.exceptions.InvalidConfigurationError(
"If you initialize lpipe with a function/callable, you cannot define paths, as you have disabled the directed-graph interface."
)
paths, path_enum = normalize.normalize_path_enum(path_enum=path_enum,
paths=paths)
successful_records = []
records = get_records_from_event(queue_type, event)
try:
assert isinstance(records, list)
except AssertionError as e:
logger.error(f"'records' is not a list {utils.exception_to_str(e)}")
return build_event_response(0, 0, logger)
_output = []
_exceptions = []
for encoded_record in records:
ret = None
try:
try:
_payload = get_payload_from_record(
queue_type=queue_type,
record=encoded_record,
validate=False if default_path else True,
)
_path = default_path if default_path else _payload["path"]
_kwargs = _payload if default_path else _payload["kwargs"]
_event_source = get_event_source(queue_type, encoded_record)
payload = Payload(
path=_path, kwargs=_kwargs,
event_source=_event_source).validate(path_enum)
except AssertionError as e:
raise lpipe.exceptions.InvalidPayloadError(
"'path' or 'kwargs' missing from payload.") from e
except TypeError as e:
raise lpipe.exceptions.InvalidPayloadError(
f"Bad record provided for queue type {queue_type}. {encoded_record} {utils.exception_to_str(e)}"
) from e
with logger.context(bind={"payload": payload.to_dict()}):
logger.log("Record received.")
# Run your path/action/functions against the payload found in this record.
ret = execute_payload(
payload=payload,
path_enum=path_enum,
paths=paths,
logger=logger,
event=event,
context=context,
debug=debug,
)
# Will handle cleanup for successful records later, if necessary.
successful_records.append(encoded_record)
except lpipe.exceptions.FailButContinue as e:
# CAPTURES:
# lpipe.exceptions.InvalidPayloadError
# lpipe.exceptions.InvalidPathError
logger.error(str(e))
if exception_handler:
exception_handler(e)
continue # User can say "bad thing happened but keep going." This drops poisoned records on the floor.
except lpipe.exceptions.FailCatastrophically as e:
# CAPTURES:
# lpipe.exceptions.InvalidConfigurationError
# raise (later)
if exception_handler:
exception_handler(e)
_exceptions.append({"exception": e, "record": encoded_record})
_output.append(ret)
response = build_event_response(n_records=len(records),
n_ok=len(successful_records),
logger=logger)
if _exceptions:
# Handle cleanup for successful records, if necessary, before creating an error state.
advanced_cleanup(queue_type, successful_records, logger)
logger.info(
f"Encountered exceptions while handling one or more records. RESPONSE: {response}"
)
raise lpipe.exceptions.FailCatastrophically(_exceptions)
if any(_output):
response["output"] = _output
if debug:
response["debug"] = json.dumps({"records": records},
cls=utils.AutoEncoder)
return response
def execute_action(
payload: Payload,
path_enum: EnumMeta,
paths: dict,
action: Action,
logger,
event,
context,
debug: bool = False,
exception_handler: FunctionType = None,
):
"""Execute functions, paths, and queues (shortcuts) in an Action.
Args:
payload (Payload):
path_enum (EnumMeta): An Enum class which define the possible paths available in this lambda.
paths (dict): Keys are path names / enums and values are a list of Action objects
action: (Action):
logger:
event: https://docs.aws.amazon.com/lambda/latest/dg/python-handler.html
context: https://docs.aws.amazon.com/lambda/latest/dg/python-handler.html
debug (bool):
exception_handler (FunctionType): A function which will be used to capture exceptions (e.g. contrib.sentry.capture)
"""
assert isinstance(action, Action)
ret = None
# Build action kwargs and validate type hints
try:
dummy = ["logger", "event"]
action_kwargs = build_action_kwargs(action, {
**{k: None
for k in dummy},
**payload.kwargs
})
for k in dummy:
action_kwargs.pop(k, None)
except (TypeError, AssertionError) as e:
raise lpipe.exceptions.InvalidPayloadError(
f"Failed to run {payload.path.name} {action} due to {utils.exception_to_str(e)}"
) from e
default_kwargs = {
"logger": logger,
"event": PayloadEvent(event=event, context=context, payload=payload),
}
# Run action functions
for f in action.functions:
assert isinstance(f, FunctionType)
try:
# TODO: if ret, set _last_output
_log_context = {"path": payload.path.name, "function": f.__name__}
with logger.context(bind={
**_log_context, "kwargs": action_kwargs
}):
logger.log("Executing function.")
with logger.context(bind=_log_context):
ret = f(**{**action_kwargs, **default_kwargs})
ret = return_handler(
ret=ret,
path_enum=path_enum,
paths=paths,
logger=logger,
event=event,
context=context,
debug=debug,
)
except lpipe.exceptions.LPBaseException:
# CAPTURES:
# lpipe.exceptions.FailButContinue
# lpipe.exceptions.FailCatastrophically
raise
except Exception as e:
logger.error(
f"Skipped {payload.path.name} {f.__name__} due to unhandled Exception. This is very serious; please update your function to handle this. Reason: {utils.exception_to_str(e)}"
)
if exception_handler:
exception_handler(e)
if debug:
raise lpipe.exceptions.FailCatastrophically(
utils.exception_to_str(e)) from e
payloads = []
for _path in action.paths:
payloads.append(
Payload(
path=normalize.normalize_path(path_enum, _path),
kwargs=action_kwargs,
event_source=payload.event_source,
).validate(path_enum))
for _queue in action.queues:
payloads.append(
Payload(queue=_queue,
kwargs=action_kwargs,
event_source=payload.event_source).validate())
for p in payloads:
ret = execute_payload(p, path_enum, paths, logger, event, context,
debug)
return ret
def execute_payload(
payload: Payload,
path_enum: EnumMeta,
paths: dict,
logger,
event,
context,
debug: bool = False,
exception_handler: FunctionType = None,
) -> Any:
"""Given a Payload, execute Actions in a Path and fire off messages to the payload's Queues.
Args:
payload (Payload):
path_enum (EnumMeta): An Enum class which define the possible paths available in this lambda.
paths (dict): Keys are path names / enums and values are a list of Action objects
logger:
event: https://docs.aws.amazon.com/lambda/latest/dg/python-handler.html
context: https://docs.aws.amazon.com/lambda/latest/dg/python-handler.html
debug (bool):
exception_handler (FunctionType): A function which will be used to capture exceptions (e.g. contrib.sentry.capture)
"""
if not logger:
logger = lpipe.logging.LPLogger()
ret = None
if payload.path is not None and not isinstance(payload.path, path_enum):
payload.path = normalize.normalize_path(path_enum, payload.path)
if isinstance(payload.path, Enum): # PATH
paths[payload.path] = normalize.normalize_actions(paths[payload.path])
for action in paths[payload.path]:
ret = execute_action(
payload=payload,
path_enum=path_enum,
paths=paths,
action=action,
logger=logger,
event=event,
context=context,
debug=debug,
exception_handler=exception_handler,
)
elif isinstance(payload.queue, Queue): # QUEUE (aka SHORTCUT)
queue = payload.queue
assert isinstance(queue.type, QueueType)
if queue.path:
record = {"path": queue.path, "kwargs": payload.kwargs}
else:
record = payload.kwargs
with logger.context(
bind={
"path": queue.path,
"queue_type": queue.type,
"queue_name": queue.name,
"record": record,
}):
logger.log("Pushing record.")
put_record(queue=queue, record=record)
else:
logger.info(
f"Path should be a string (path name), Path (path Enum), or Queue: {payload.path})"
)
return ret
def test_queue_payload(self, fixture_name, fixture):
q = Queue(**fixture)
Payload(queue=q, kwargs={"foo": "bar"}).validate()
def test_payload(self, fixture_name, fixture):
Payload(**fixture).validate(Path)
def execute_action(payload: Payload, action: Action, state: State) -> Any:
"""Execute functions, paths, and queues (shortcuts) in an Action.
Args:
payload (Payload):
action: (Action):
state (State):
"""
assert isinstance(action, Action)
ret = None
# Build action kwargs and validate type hints
try:
if RESERVED_KEYWORDS & set(payload.kwargs):
state.logger.warning(
f"Payload contains a reserved argument name. Please update your function use a different argument name. Reserved keywords: {RESERVED_KEYWORDS}"
)
action_kwargs = build_action_kwargs(action, {
**{k: None
for k in RESERVED_KEYWORDS},
**payload.kwargs
})
for k in RESERVED_KEYWORDS:
action_kwargs.pop(k, None)
except (TypeError, AssertionError) as e:
raise lpipe.exceptions.InvalidPayloadError(
f"Failed to run {payload.path.name} {action} due to {utils.exception_to_str(e)}"
) from e
default_kwargs = {
"logger": state.logger,
"state": state,
"payload": payload
}
# Run action functions
for f in action.functions:
assert isinstance(f, FunctionType)
try:
# TODO: if ret, evaluate viability of passing to next in sequence
_log_context = {"path": payload.path.name, "function": f.__name__}
with state.logger.context(bind={
**_log_context, "kwargs": action_kwargs
}):
state.logger.log("Executing function.")
with state.logger.context(bind=_log_context):
ret = f(**{**action_kwargs, **default_kwargs})
ret = return_handler(ret=ret, state=state)
except lpipe.exceptions.LPBaseException:
# CAPTURES:
# lpipe.exceptions.FailButContinue
# lpipe.exceptions.FailCatastrophically
raise
except Exception as e:
state.logger.error(
f"Skipped {payload.path.name} {f.__name__} due to unhandled Exception {e.__class__.__name__}. This is very serious; please update your function to handle this."
)
log_exception(state, e)
payloads = []
for _path in action.paths:
payloads.append(
Payload(
path=normalize.normalize_path(state.path_enum, _path),
kwargs=action_kwargs,
event_source=payload.event_source,
).validate(state.path_enum))
for _queue in action.queues:
payloads.append(
Payload(queue=_queue,
kwargs=action_kwargs,
event_source=payload.event_source).validate())
for p in payloads:
ret = execute_payload(payload=p, state=state)
return ret