def test_split_event_stages(mock_app_config): # noqa: F811
impl = find_event_handler(
app_config=mock_app_config,
event_name='mock_shuffle_event',
event_info=mock_app_config.events['mock_shuffle_event']
)
event_info = mock_app_config.events['mock_shuffle_event']
stages = split_event_stages(mock_app_config.app,
event_name='mock_shuffle_event',
event_info=event_info,
impl=impl)
assert stages == {
'mock_shuffle_event': EventDescriptor(
type=EventType.GET,
read_stream=event_info.read_stream,
write_stream=WriteStreamDescriptor(
name='mock_app.test.mock_shuffle_event.produce_messages',
queues=['AUTO'],
queue_strategy=StreamQueueStrategy.PROPAGATE
),
auth=[]
),
'mock_shuffle_event$consume_stream': EventDescriptor(
type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_app.test.mock_shuffle_event.produce_messages',
consumer_group='mock_app.test.mock_shuffle_event.consume_stream',
queues=['AUTO']
),
write_stream=event_info.write_stream,
auth=[]
)
}
def split_event_stages(app: AppDescriptor, event_name: str,
event_info: EventDescriptor,
impl: ModuleType) -> Dict[str, EventDescriptor]:
"""
Splits an event whose steps contain SHUFFLE step, in an initial event with same name as event_name
plus sub_events with names `event_name.step_name' for each step after a SHUFFLE.
Creates intermediate auto named write_stream, read_stream to communicate data between event and sub_events,
clones event configuration from main event to sub_events, and setup write_stream property for final event
to be the one specified in configuration.
"""
event_stages = extract_event_stages(impl)
if len(event_stages) == 1:
return {event_name: event_info}
effective_events: Dict[str, EventDescriptor] = {}
event_type = event_info.type
read_stream = event_info.read_stream
queues = ["AUTO"] if read_stream is None else read_stream.queues
sub_event_name: Optional[str] = event_name
sub_event_info = event_info
intermediate_stream = None
for stage in event_stages:
if sub_event_name is None:
sub_event_name = f"{event_name}${stage}"
if read_stream is None and intermediate_stream is not None:
read_stream = ReadStreamDescriptor(name=intermediate_stream,
consumer_group=auto_path(
app.name, app.version,
*event_name.split('.'),
stage),
queues=queues)
intermediate_stream = auto_path(app.name, app.version,
*event_name.split('.'), stage)
sub_event_info = EventDescriptor(
type=event_type,
read_stream=read_stream,
connections=event_info.connections,
impl=event_info.impl,
write_stream=WriteStreamDescriptor(
name=intermediate_stream,
queue_strategy=StreamQueueStrategy.PROPAGATE))
effective_events[sub_event_name] = sub_event_info
event_type = EventType.STREAM
sub_event_name = None
read_stream = None
# Set last stage write_stream to original event write_stream
sub_event_info.write_stream = event_info.write_stream
return effective_events
def valid_result_app_config() -> AppConfig:
return AppConfig(
app=AppDescriptor(name="simple_example", version=APPS_API_VERSION),
engine=AppEngineConfig(import_modules=["model"],
read_stream_timeout=1,
track_headers=["request_id", "correlation_id"]),
env={
"fs": {
"data_path":
f"/tmp/simple_example.{APP_VERSION}.fs.data_path/",
"app_description":
f"This is simple_example version {APPS_API_VERSION}",
"recursive_replacement":
f"Data is in /tmp/simple_example.{APP_VERSION}.fs.data_path/. "
+ f"This is simple_example version {APPS_API_VERSION}"
}
},
events={
"query_something":
EventDescriptor(type=EventType.GET),
"save_something":
EventDescriptor(type=EventType.POST),
"streams.something_event":
EventDescriptor(
type=EventType.POST,
write_stream=WriteStreamDescriptor(
name=f'simple_example.{APP_VERSION}.streams.something_event'
)),
"streams.process_events":
EventDescriptor(
type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name=
f'simple_example.{APP_VERSION}.streams.something_event',
consumer_group=
f'simple_example.{APP_VERSION}.streams.process_events'),
config=EventConfig(logging=EventLoggingConfig(
extra_fields=['something_id', 'path'])))
})
def mock_app_config():
return AppConfig(
app=AppDescriptor(name='mock_app', version='test'),
plugins=[AppDescriptor('mock_plugin', 'test')],
engine=AppEngineConfig(import_modules=['mock_app'],
read_stream_timeout=1,
read_stream_interval=5,
track_headers=['session_id'],
cors_origin='http://test'),
env={
'app': {
'app_value': 'test_app_value'
},
'plugin': {
'custom_value': 'test_custom_value_override'
}
},
settings={
"mock_stream_event": {
"logging": {
"extra_fields": ['value'],
"stream_fields": ['msg_id']
},
"custom_setting": {
"custom": "value"
}
},
"mock_stream_timeout": {
"logging": {
"extra_fields": ['value'],
"stream_fields": ['msg_id']
},
"stream": {
"timeout": 2
}
},
"mock_write_stream_event": {
"logging": {
"extra_fields": ['value'],
"stream_fields": ['msg_id']
},
"stream": {
"target_max_len": 10
}
},
"mock_service_event": {
"stream": {
"target_max_len": 10,
"throttle_ms": 100,
"batch_size": 2
}
},
"mock_service_timeout": {
"response_timeout": 2.0
},
"mock_spawn_event": {
"stream": {
"target_max_len": 10,
"throttle_ms": 100,
"batch_size": 2
}
},
"mock_shuffle_event": {
"stream": {
"target_max_len": 10,
"throttle_ms": 100
}
},
"mock_timeout": {
"response_timeout": 2.0
},
"custom_extra_settings": {
"custom_setting": {
"custom": "value"
}
}
},
events={
"mock_event":
EventDescriptor(type=EventType.GET,
route='mock-app/test/mock-event-test',
group='GROUP_A'),
"mock_event_logging":
EventDescriptor(type=EventType.GET, group='GROUP_A'),
"mock_post_event":
EventDescriptor(type=EventType.POST,
route='mock-app/test/mock-event-test',
group='GROUP_A'),
"mock_multipart_event":
EventDescriptor(type=EventType.MULTIPART,
route='mock-app/test/mock-multipart-event-test',
group='GROUP_B'),
"mock_post_nopayload":
EventDescriptor(type=EventType.POST,
route='mock-app/test/mock-post-nopayload'),
"mock_post_preprocess":
EventDescriptor(type=EventType.POST,
route='mock-app/test/mock-post-preprocess'),
"mock_post_preprocess_no_datatype":
EventDescriptor(
type=EventType.POST,
route='mock-app/test/mock-post-preprocess-no-datatype'),
"mock_stream_event":
EventDescriptor(type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_stream',
consumer_group='mock_consumer_group'),
setting_keys=["custom_extra_settings"],
group='GROUP_A'),
"mock_stream_timeout":
EventDescriptor(type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_stream',
consumer_group='mock_consumer_group'),
group='GROUP_B'),
"mock_write_stream_event":
EventDescriptor(type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event')),
"mock_service_event":
EventDescriptor(type=EventType.SERVICE,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event')),
"mock_service_timeout":
EventDescriptor(type=EventType.SERVICE,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event')),
"mock_spawn_event":
EventDescriptor(type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event')),
"mock_shuffle_event":
EventDescriptor(type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event')),
"mock_parallelize_event":
EventDescriptor(type=EventType.GET),
"mock_file_response":
EventDescriptor(type=EventType.GET),
"mock_stream_response":
EventDescriptor(type=EventType.GET),
"mock_file_response_content_type":
EventDescriptor(type=EventType.GET),
"mock_auth":
EventDescriptor(type=EventType.GET, auth=[AuthType.BASIC]),
"mock_post_auth":
EventDescriptor(type=EventType.POST, auth=[AuthType.BASIC]),
"mock_collector":
EventDescriptor(type=EventType.POST),
'mock_timeout':
EventDescriptor(type=EventType.GET),
'mock_read_write_stream':
EventDescriptor(type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_read_write_stream.read',
consumer_group='mock_read_write_stream'),
write_stream=WriteStreamDescriptor(
name='mock_read_write_stream.write')),
"mock_event_custom":
EventDescriptor(type=EventType.GET,
impl="mock_app.mock_event_custom_impl"),
"mock_event_dataobject_payload":
EventDescriptor(
type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_read_write_stream.read',
consumer_group='mock_event_dataobject_payload'),
dataobjects=["mock_app.MockData"])
},
server=ServerConfig(
streams=StreamsConfig(
stream_manager='mock_engine.MockStreamManager',
delay_auto_start_seconds=0),
logging=LoggingConfig(log_level="DEBUG",
log_path="work/logs/test/"))).setup()
def mock_app_config():
return AppConfig(
app=AppDescriptor(
name='mock_app',
version='test'
),
plugins=[
AppDescriptor('mock_plugin', 'test')
],
engine=AppEngineConfig(
import_modules=['mock_app'],
read_stream_timeout=1,
read_stream_interval=5,
track_headers=['session_id'],
cors_origin='http://test'
),
env={
'app': {
'app_value': 'test_app_value'
},
'plugin': {
'custom_value': 'test_custom_value_override'
}
},
events={
"mock_event": EventDescriptor(
type=EventType.GET,
route='mock-app/test/mock-event-test'
),
"mock_event_logging": EventDescriptor(
type=EventType.GET
),
"mock_post_event": EventDescriptor(
type=EventType.POST,
route='mock-app/test/mock-event-test'
),
"mock_multipart_event": EventDescriptor(
type=EventType.MULTIPART,
route='mock-app/test/mock-multipart-event-test'
),
"mock_post_nopayload": EventDescriptor(
type=EventType.POST,
route='mock-app/test/mock-post-nopayload'
),
"mock_post_preprocess": EventDescriptor(
type=EventType.POST,
route='mock-app/test/mock-post-preprocess'
),
"mock_stream_event": EventDescriptor(
type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_stream',
consumer_group='mock_consumer_group'
),
config=EventConfig(
logging=EventLoggingConfig(
extra_fields=['value'],
stream_fields=['msg_id']
)
)
),
"mock_stream_timeout": EventDescriptor(
type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_stream',
consumer_group='mock_consumer_group'
),
config=EventConfig(
logging=EventLoggingConfig(
extra_fields=['value'],
stream_fields=['msg_id']
),
stream=EventStreamConfig(
timeout=2
)
)
),
"mock_write_stream_event": EventDescriptor(
type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event'
),
config=EventConfig(
stream=EventStreamConfig(
target_max_len=10
)
)
),
"mock_service_event": EventDescriptor(
type=EventType.SERVICE,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event'
),
config=EventConfig(
stream=EventStreamConfig(
target_max_len=10,
throttle_ms=100,
batch_size=2
)
)
),
"mock_service_timeout": EventDescriptor(
type=EventType.SERVICE,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event'
),
config=EventConfig(
response_timeout=2.0
)
),
"mock_spawn_event": EventDescriptor(
type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event'
),
config=EventConfig(
stream=EventStreamConfig(
target_max_len=10,
throttle_ms=100,
batch_size=2
)
)
),
"mock_shuffle_event": EventDescriptor(
type=EventType.GET,
write_stream=WriteStreamDescriptor(
name='mock_write_stream_event'
),
config=EventConfig(
stream=EventStreamConfig(
target_max_len=10,
throttle_ms=100
)
)
),
"mock_parallelize_event": EventDescriptor(
type=EventType.GET
),
"mock_file_response": EventDescriptor(
type=EventType.GET
),
"mock_file_response_content_type": EventDescriptor(
type=EventType.GET
),
"mock_auth": EventDescriptor(
type=EventType.GET,
auth=[AuthType.BASIC]
),
"mock_post_auth": EventDescriptor(
type=EventType.POST,
auth=[AuthType.BASIC]
),
"mock_collector": EventDescriptor(
type=EventType.POST
),
'mock_timeout': EventDescriptor(
type=EventType.GET,
config=EventConfig(response_timeout=2.0)
),
'mock_read_write_stream': EventDescriptor(
type=EventType.STREAM,
read_stream=ReadStreamDescriptor(
name='mock_read_write_stream.read',
consumer_group='mock_read_write_stream'
),
write_stream=WriteStreamDescriptor(
name='mock_read_write_stream.write'
)
)
},
server=ServerConfig(
streams=StreamsConfig(
stream_manager='mock_engine.MockStreamManager',
delay_auto_start_seconds=0
),
logging=LoggingConfig(
log_level="DEBUG", log_path="work/logs/test/")
)
)