def _find_stream_datatype_handlers(
self, event_name: str,
event_info: EventDescriptor) -> Dict[str, type]:
"""
Computes a dictionary of `{datatype name: datatype class}` that event steps
can be handle when consuming from an stream.
"""
base_event, _ = event_and_step(event_name)
impl = find_event_handler(app_config=self.app_config,
event_name=base_event,
event_info=event_info)
all_steps = extract_module_steps(impl)
steps = effective_steps(event_name, all_steps)
datatypes = {}
for _, _, step in steps:
_, datatype, _, _ = step
if hasattr(datatype, '__stream_event__'):
datatypes[
f"{datatype.__module__}.{datatype.__qualname__}"] = datatype
elif datatype is DataObject:
for type_name in event_info.dataobjects:
datatype = find_datobject_type(type_name)
datatypes[
f"{datatype.__module__}.{datatype.__qualname__}"] = datatype
if len(datatypes) == 0:
raise NotImplementedError(
f"No data types found to read from stream in event={event_name}. "
"Dataclasses must be decorated with `@dataobject` to be used in streams"
)
return datatypes
def test_effective_steps_shuffle_event():
impl = importlib.import_module('mock_app.mock_shuffle_event')
f1 = getattr(impl, 'produce_messages')
f2 = getattr(impl, 'consume_stream')
f3 = getattr(impl, 'generate_default')
module_steps = [
('produce_messages', (f1, str, Spawn[MockData], True)),
(SHUFFLE, None),
('consume_stream', (f2, MockData, Optional[MockResult], False)),
('generate_default', (f3, None, MockResult, False))
]
steps = effective_steps('mock_shuffle_event', module_steps)
assert steps == [
(0, 'produce_messages', (f1, str, Spawn[MockData], True))
]
steps = effective_steps('mock_shuffle_event$consume_stream', module_steps)
assert steps == [
(0, 'consume_stream', (f2, MockData, Optional[MockResult], False)),
(1, 'generate_default', (f3, None, MockResult, False))
]
def test_effective_steps():
impl = importlib.import_module('mock_app.mock_event')
f1 = getattr(impl, 'entry_point')
f2 = getattr(impl, 'handle_ok_case')
f3 = getattr(impl, 'handle_special_case')
module_steps = [
('entry_point', (f1, None, Union[MockData, str], False)),
('handle_ok_case', (f2, MockData, str, False)),
('handle_special_case', (f3, str, str, False))
]
steps = effective_steps('mock_event', module_steps)
assert steps == [
(0, 'entry_point', (f1, None, Union[MockData, str], False)),
(1, 'handle_ok_case', (f2, MockData, str, False)),
(2, 'handle_special_case', (f3, str, str, False))
]
def load_modules(self, effective_events: Dict[str, EventDescriptor]):
for event_name, event_info in effective_events.items():
base_event, _ = event_and_step(event_name)
module = find_event_handler(app_config=self.app_config,
event_name=base_event)
steps = extract_module_steps(module)
self.modules[base_event] = (module, False, steps)
self.preprocess_handlers[base_event] = extract_preprocess_handler(
module)
self.postprocess_handlers[
base_event] = extract_postprocess_handler(module)
setup_app_logger(module,
app_config=self.app_config,
name=base_event,
event_info=event_info)
self.steps[event_name] = effective_steps(event_name, steps)
def load_modules(self, effective_events: Dict[str, EventDescriptor]):
"""
Load and setup event implementation modules. Set logger and load settings.
"""
for event_name, event_info in effective_events.items():
base_event, _ = event_and_step(event_name)
module = find_event_handler(app_config=self.app_config,
event_name=base_event,
event_info=event_info)
steps = extract_module_steps(module)
self.modules[base_event] = (module, False, steps)
self.preprocess_handlers[base_event] = extract_preprocess_handler(
module)
self.postprocess_handlers[
base_event] = extract_postprocess_handler(module)
event_settings = get_event_settings(self.settings, event_name)
setup_app_logger(module,
app_config=self.app_config,
name=base_event,
event_settings=event_settings)
self.steps[event_name] = effective_steps(event_name, steps)
def _find_stream_datatype_handlers(self,
event_name: str) -> Dict[str, type]:
"""
Computes a dictionary of `{datatype name: datatype class}` that event steps
can be handle when consuming from an stream.
"""
base_event, _ = event_and_step(event_name)
impl = find_event_handler(app_config=self.app_config,
event_name=base_event)
all_steps = extract_module_steps(impl)
steps = effective_steps(event_name, all_steps)
datatypes = {}
for _, step in steps.items():
_, datatype, _ = step
if hasattr(datatype, '__stream_event__'):
datatypes[datatype.__name__] = datatype
if len(datatypes) == 0:
raise NotImplementedError(
f"No data types found to read from stream in even={event_name}. "
"Dataclasses must be decorated with `@dataobject` to be used in streams"
)
return datatypes